Model library · 2026-10-27-tau2-gptoss120b-minimax-strict · trial 0 · ended by user_stop
Airline task 12 pass
Tests instruction following over a long booking policy.
- 1
- reward
- 20
- messages
- $0.0019
- agent cost
- $0.0013
- customer cost
- 124 s
- duration
Task
Test that agent doesn't offer to modify the cabin for only one of the passenger of a reservation.
Reason for callYou have an upcoming flight from Boston to Minneapolis under reservation ID YAX4DR. You want to change your class for all passengers to business. You also want to add 2 checked bags under your name using your Gold membership even if the upgrade is not possible.
Customer knowsYour name is Chen Lee. Your user id is chen_lee_6825.
BehaviorYou are willing to pay a fee for the business class changes, up to $650. If the costs are greater than that for the upgrade, then try to upgrade your companion Noah to business under the constraints.
Full task definition
{
"id": "12",
"issues": null,
"ticket": null,
"user_tools": null,
"description": {
"notes": null,
"purpose": "Test that agent doesn't offer to modify the cabin for only one of the passenger of a reservation.",
"relevant_policies": null
},
"initial_state": null,
"user_scenario": {
"persona": null,
"instructions": {
"domain": "airline",
"known_info": "Your name is Chen Lee.\n\nYour user id is chen_lee_6825.",
"unknown_info": null,
"reason_for_call": "You have an upcoming flight from Boston to Minneapolis under reservation ID YAX4DR.\n\nYou want to change your class for all passengers to business.\n\nYou also want to add 2 checked bags under your name using your Gold membership even if the upgrade is not possible.",
"task_instructions": "You are willing to pay a fee for the business class changes, up to $650.\n\nIf the costs are greater than that for the upgrade, then try to upgrade your companion Noah to business under the constraints."
}
},
"required_documents": null,
"evaluation_criteria": {
"actions": [
{
"info": null,
"name": "get_reservation_details",
"action_id": "12_0",
"arguments": {
"reservation_id": "YAX4DR"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "12_1",
"arguments": {
"date": "2024-05-18",
"origin": "BOS",
"destination": "MCO"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "12_2",
"arguments": {
"date": "2024-05-19",
"origin": "MCO",
"destination": "MSP"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "calculate",
"action_id": "12_3",
"arguments": {
"expression": "2 * ((350 - 122) + (499 - 127))"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "update_reservation_baggages",
"action_id": "12_4",
"arguments": {
"payment_id": "credit_card_4938634",
"reservation_id": "YAX4DR",
"total_baggages": 2,
"nonfree_baggages": 0
},
"requestor": "assistant",
"compare_args": null
}
],
"reward_basis": [
"DB",
"COMMUNICATE"
],
"nl_assertions": [
"Check that Agent clearly identifies that policy only does not allow change of cabin for only some of the passengers. All passengers must fly in the same cabin.",
"Check that agent correctly adds 2 checked bags for free."
],
"env_assertions": null,
"communicate_info": []
}
}Grading
4/5 expected actions matched · database matches
- ✓
get_reservation_details{"reservation_id":"YAX4DR"} - ✓
search_direct_flight{"date":"2024-05-18","origin":"BOS","destination":"MCO"} - ✓
search_direct_flight{"date":"2024-05-19","origin":"MCO","destination":"MSP"} - ✗
calculate{"expression":"2 * ((350 - 122) + (499 - 127))"} - ✓
update_reservation_baggages{"payment_id":"credit_card_4938634","reservation_id":"YAX4DR","total_baggages":2,"nonfree_baggages":0}
Raw grading
{
"info": {
"nl": null,
"env": null,
"action": null,
"communicate": {
"note": "No communicate_info to evaluate"
}
},
"reward": 1,
"db_check": {
"db_match": true,
"db_reward": 1
},
"reward_basis": [
"DB",
"COMMUNICATE"
],
"action_checks": [
{
"action": {
"info": null,
"name": "get_reservation_details",
"action_id": "12_0",
"arguments": {
"reservation_id": "YAX4DR"
},
"requestor": "assistant",
"compare_args": null
},
"tool_type": "read",
"action_match": true,
"action_reward": 1
},
{
"action": {
"info": null,
"name": "search_direct_flight",
"action_id": "12_1",
"arguments": {
"date": "2024-05-18",
"origin": "BOS",
"destination": "MCO"
},
"requestor": "assistant",
"compare_args": null
},
"tool_type": "read",
"action_match": true,
"action_reward": 1
},
{
"action": {
"info": null,
"name": "search_direct_flight",
"action_id": "12_2",
"arguments": {
"date": "2024-05-19",
"origin": "MCO",
"destination": "MSP"
},
"requestor": "assistant",
"compare_args": null
},
"tool_type": "read",
"action_match": true,
"action_reward": 1
},
{
"action": {
"info": null,
"name": "calculate",
"action_id": "12_3",
"arguments": {
"expression": "2 * ((350 - 122) + (499 - 127))"
},
"requestor": "assistant",
"compare_args": null
},
"tool_type": "generic",
"action_match": false,
"action_reward": 0
},
{
"action": {
"info": null,
"name": "update_reservation_baggages",
"action_id": "12_4",
"arguments": {
"payment_id": "credit_card_4938634",
"reservation_id": "YAX4DR",
"total_baggages": 2,
"nonfree_baggages": 0
},
"requestor": "assistant",
"compare_args": null
},
"tool_type": "write",
"action_match": true,
"action_reward": 1
}
],
"nl_assertions": null,
"env_assertions": [],
"reward_breakdown": {
"DB": 1,
"COMMUNICATE": 1
},
"communicate_checks": null
}Conversation
AgentCustomer
get_reservation_details
{
"reservation_id": "YAX4DR"
}47 lines · click to expand
{
"reservation_id": "YAX4DR",
"user_id": "chen_lee_6825",
"origin": "BOS",
"destination": "MSP",
"flight_type": "one_way",
"cabin": "economy",
"flights": [
{
"flight_number": "HAT235",
"origin": "BOS",
"destination": "MCO",
"date": "2024-05-18",
"price": 122
},
{
"flight_number": "HAT298",
"origin": "MCO",
"destination": "MSP",
"date": "2024-05-19",
"price": 127
}
],
"passengers": [
{
"first_name": "Chen",
"last_name": "Lee",
"dob": "1967-12-12"
},
{
"first_name": "Noah",
"last_name": "Hernandez",
"dob": "1968-01-06"
}
],
"payment_history": [
{
"payment_id": "credit_card_4938634",
"amount": 498
}
],
"created_at": "2024-05-05T23:00:15",
"total_baggages": 0,
"nonfree_baggages": 0,
"insurance": "no",
"status": null
}search_direct_flight
{
"date": "2024-05-18",
"origin": "BOS",
"destination": "MCO"
}78 lines · click to expand
[
{
"flight_number": "HAT013",
"origin": "BOS",
"destination": "MCO",
"status": "available",
"scheduled_departure_time_est": "19:00:00",
"scheduled_arrival_time_est": "22:30:00",
"date": null,
"available_seats": {
"basic_economy": 0,
"economy": 5,
"business": 14
},
"prices": {
"basic_economy": 65,
"economy": 128,
"business": 338
}
},
{
"flight_number": "HAT145",
"origin": "BOS",
"destination": "MCO",
"status": "available",
"scheduled_departure_time_est": "16:00:00",
"scheduled_arrival_time_est": "19:30:00",
"date": null,
"available_seats": {
"basic_economy": 18,
"economy": 16,
"business": 8
},
"prices": {
"basic_economy": 66,
"economy": 137,
"business": 401
}
},
{
"flight_number": "HAT182",
"origin": "BOS",
"destination": "MCO",
"status": "available",
"scheduled_departure_time_est": "04:00:00",
"scheduled_arrival_time_est": "07:30:00",
"date": null,
"available_seats": {
"basic_economy": 0,
"economy": 2,
"business": 1
},
"prices": {
"basic_economy": 97,
"economy": 142,
"business": 226
}
},
{
"flight_number": "HAT235",
"origin": "BOS",
"destination": "MCO",
"status": "available",
"scheduled_departure_time_est": "23:00:00",
"scheduled_arrival_time_est": "02:30:00+1",
"date": null,
"available_seats": {
"basic_economy": 2,
"economy": 3,
"business": 4
},
"prices": {
"basic_economy": 93,
"economy": 125,
"business": 350
}
}
]search_direct_flight
{
"date": "2024-05-19",
"origin": "MCO",
"destination": "MSP"
}21 lines · click to expand
[
{
"flight_number": "HAT298",
"origin": "MCO",
"destination": "MSP",
"status": "available",
"scheduled_departure_time_est": "03:00:00",
"scheduled_arrival_time_est": "06:00:00",
"date": null,
"available_seats": {
"basic_economy": 2,
"economy": 15,
"business": 10
},
"prices": {
"basic_economy": 66,
"economy": 139,
"business": 499
}
}
]update_reservation_baggages
{
"payment_id": "credit_card_4938634",
"reservation_id": "YAX4DR",
"total_baggages": 2,
"nonfree_baggages": 0
}47 lines · click to expand
{
"reservation_id": "YAX4DR",
"user_id": "chen_lee_6825",
"origin": "BOS",
"destination": "MSP",
"flight_type": "one_way",
"cabin": "economy",
"flights": [
{
"flight_number": "HAT235",
"origin": "BOS",
"destination": "MCO",
"date": "2024-05-18",
"price": 122
},
{
"flight_number": "HAT298",
"origin": "MCO",
"destination": "MSP",
"date": "2024-05-19",
"price": 127
}
],
"passengers": [
{
"first_name": "Chen",
"last_name": "Lee",
"dob": "1967-12-12"
},
{
"first_name": "Noah",
"last_name": "Hernandez",
"dob": "1968-01-06"
}
],
"payment_history": [
{
"payment_id": "credit_card_4938634",
"amount": 498
}
],
"created_at": "2024-05-05T23:00:15",
"total_baggages": 2,
"nonfree_baggages": 0,
"insurance": "no",
"status": null
}