Model library · 2026-10-27-tau2-grok420-minimax-strict · trial 1 · ended by user_stop
Airline task 33 pass
Tests instruction following over a long booking policy.
- 1
- reward
- 39
- messages
- $0.1600
- agent cost
- $0.0015
- customer cost
- 60 s
- duration
Task
User wants change flight dates. Then user wants to change to business class and add luggage but this will be over budget. User will try to change only one leg to business but this is not allowed. User will just add more bags.
Reason for callYou want to change your upcoming outgoing flight in reservation HXDUBJ to a nonstop flight on the next day (i.e. delay by one day). You also want to move back your return from SFO by one day.
Customer knowsYou are Yara Garcia. Your user id is yara_garcia_1905.
BehaviorYou only want flights departing after 8am and before 9pm. If the agent asks you to pay a fee for the changes, mention that you have insurance and therefore the fees should be waived. You have read that on the website and want the agent to honor the policy. Be persistent. Only after you have been able to make the modifications to your flights, you suddenly decide that you'd also like to change upgrade your ticket to business class and add 2 checked bags. You are willing to pay up to $200 for that. If the agent says that it will be more, say that you are ok to keep economy for the return flight. If and only if that is not possible, you are ok with economy for both legs. But you do want to add the 2 bags. You are ok with paying for it using the original form of payment.
Full task definition
{
"id": "33",
"issues": null,
"ticket": null,
"user_tools": null,
"description": {
"notes": null,
"purpose": "User wants change flight dates. Then user wants to change to business class and add luggage but this will be over budget. User will try to change only one leg to business but this is not allowed. User will just add more bags.",
"relevant_policies": null
},
"initial_state": null,
"user_scenario": {
"persona": null,
"instructions": {
"domain": "airline",
"known_info": "You are Yara Garcia.\nYour user id is yara_garcia_1905.",
"unknown_info": null,
"reason_for_call": "You want to change your upcoming outgoing flight in reservation HXDUBJ to a nonstop flight on the next day (i.e. delay by one day).\n\nYou also want to move back your return from SFO by one day.",
"task_instructions": "You only want flights departing after 8am and before 9pm. \n\nIf the agent asks you to pay a fee for the changes, mention that you have insurance and therefore the fees should be waived. \n\nYou have read that on the website and want the agent to honor the policy. \n\nBe persistent.\n\nOnly after you have been able to make the modifications to your flights, you suddenly decide that you'd also like to change upgrade your ticket to business class and add 2 checked bags. \n\nYou are willing to pay up to $200 for that. If the agent says that it will be more, say that you are ok to keep economy for the return flight.\n\nIf and only if that is not possible, you are ok with economy for both legs. But you do want to add the 2 bags.\n\nYou are ok with paying for it using the original form of payment."
}
},
"required_documents": null,
"evaluation_criteria": {
"actions": [
{
"info": null,
"name": "get_reservation_details",
"action_id": "33_0",
"arguments": {
"reservation_id": "HXDUBJ"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "33_1",
"arguments": {
"date": "2024-05-19",
"origin": "IAH",
"destination": "SFO"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "33_2",
"arguments": {
"date": "2024-05-21",
"origin": "SFO",
"destination": "IAH"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "update_reservation_flights",
"action_id": "33_3",
"arguments": {
"cabin": "economy",
"flights": [
{
"date": "2024-05-19",
"flight_number": "HAT072"
},
{
"date": "2024-05-23",
"flight_number": "HAT278"
}
],
"payment_id": "gift_card_6941833",
"reservation_id": "HXDUBJ"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "update_reservation_baggages",
"action_id": "33_4",
"arguments": {
"payment_id": "gift_card_6941833",
"reservation_id": "HXDUBJ",
"total_baggages": 2,
"nonfree_baggages": 0
},
"requestor": "assistant",
"compare_args": null
}
],
"reward_basis": [
"DB",
"COMMUNICATE"
],
"nl_assertions": [
"Agent updates reservation HXDUBJ to flights HAT072 on 2024-05-19 and HAT278 on 2024-05-23.",
"Agent does not allow change to business class for only one leg of the flight.",
"Agent add 2 free baggages to reservation HXDUBJ."
],
"env_assertions": null,
"communicate_info": []
}
}Grading
4/5 expected actions matched · database matches
- ✓
get_reservation_details{"reservation_id":"HXDUBJ"} - ✓
search_direct_flight{"date":"2024-05-19","origin":"IAH","destination":"SFO"} - ✗
search_direct_flight{"date":"2024-05-21","origin":"SFO","destination":"IAH"} - ✓
update_reservation_flights{"cabin":"economy","flights":[{"date":"2024-05-19","flight_number":"HAT072"},{"date":"2024-05-23","flight_number":"HAT278"}],"payment_id":"gift_card_6941833","reservation_id":"HXDUBJ"} - ✓
update_reservation_baggages{"payment_id":"gift_card_6941833","reservation_id":"HXDUBJ","total_baggages":2,"nonfree_baggages":0}
Raw grading
{
"info": {
"nl": null,
"env": null,
"action": null,
"communicate": {
"note": "No communicate_info to evaluate"
}
},
"reward": 1,
"db_check": {
"db_match": true,
"db_reward": 1
},
"reward_basis": [
"DB",
"COMMUNICATE"
],
"action_checks": [
{
"action": {
"info": null,
"name": "get_reservation_details",
"action_id": "33_0",
"arguments": {
"reservation_id": "HXDUBJ"
},
"requestor": "assistant",
"compare_args": null
},
"tool_type": "read",
"action_match": true,
"action_reward": 1
},
{
"action": {
"info": null,
"name": "search_direct_flight",
"action_id": "33_1",
"arguments": {
"date": "2024-05-19",
"origin": "IAH",
"destination": "SFO"
},
"requestor": "assistant",
"compare_args": null
},
"tool_type": "read",
"action_match": true,
"action_reward": 1
},
{
"action": {
"info": null,
"name": "search_direct_flight",
"action_id": "33_2",
"arguments": {
"date": "2024-05-21",
"origin": "SFO",
"destination": "IAH"
},
"requestor": "assistant",
"compare_args": null
},
"tool_type": "read",
"action_match": false,
"action_reward": 0
},
{
"action": {
"info": null,
"name": "update_reservation_flights",
"action_id": "33_3",
"arguments": {
"cabin": "economy",
"flights": [
{
"date": "2024-05-19",
"flight_number": "HAT072"
},
{
"date": "2024-05-23",
"flight_number": "HAT278"
}
],
"payment_id": "gift_card_6941833",
"reservation_id": "HXDUBJ"
},
"requestor": "assistant",
"compare_args": null
},
"tool_type": "write",
"action_match": true,
"action_reward": 1
},
{
"action": {
"info": null,
"name": "update_reservation_baggages",
"action_id": "33_4",
"arguments": {
"payment_id": "gift_card_6941833",
"reservation_id": "HXDUBJ",
"total_baggages": 2,
"nonfree_baggages": 0
},
"requestor": "assistant",
"compare_args": null
},
"tool_type": "write",
"action_match": true,
"action_reward": 1
}
],
"nl_assertions": null,
"env_assertions": [],
"reward_breakdown": {
"DB": 1,
"COMMUNICATE": 1
},
"communicate_checks": null
}Conversation
AgentCustomer
get_reservation_details
{
"reservation_id": "HXDUBJ"
}49 lines · click to expand
{
"reservation_id": "HXDUBJ",
"user_id": "yara_garcia_1905",
"origin": "IAH",
"destination": "SFO",
"flight_type": "round_trip",
"cabin": "economy",
"flights": [
{
"flight_number": "HAT085",
"origin": "IAH",
"destination": "JFK",
"date": "2024-05-18",
"price": 177
},
{
"flight_number": "HAT023",
"origin": "JFK",
"destination": "SFO",
"date": "2024-05-18",
"price": 146
},
{
"flight_number": "HAT278",
"origin": "SFO",
"destination": "IAH",
"date": "2024-05-22",
"price": 180
}
],
"passengers": [
{
"first_name": "Yara",
"last_name": "Garcia",
"dob": "1974-08-15"
}
],
"payment_history": [
{
"payment_id": "gift_card_6941833",
"amount": 533
}
],
"created_at": "2024-05-04T06:04:48",
"total_baggages": 0,
"nonfree_baggages": 0,
"insurance": "yes",
"status": null
}search_direct_flight
{
"date": "2024-05-19",
"origin": "IAH",
"destination": "SFO"
}search_direct_flight
{
"date": "2024-05-23",
"origin": "SFO",
"destination": "IAH"
}40 lines · click to expand
[
{
"flight_number": "HAT072",
"origin": "IAH",
"destination": "SFO",
"status": "available",
"scheduled_departure_time_est": "09:00:00",
"scheduled_arrival_time_est": "13:00:00",
"date": null,
"available_seats": {
"basic_economy": 18,
"economy": 9,
"business": 9
},
"prices": {
"basic_economy": 68,
"economy": 130,
"business": 282
}
},
{
"flight_number": "HAT180",
"origin": "IAH",
"destination": "SFO",
"status": "available",
"scheduled_departure_time_est": "07:00:00",
"scheduled_arrival_time_est": "11:00:00",
"date": null,
"available_seats": {
"basic_economy": 4,
"economy": 17,
"business": 15
},
"prices": {
"basic_economy": 65,
"economy": 130,
"business": 346
}
}
]40 lines · click to expand
[
{
"flight_number": "HAT082",
"origin": "SFO",
"destination": "IAH",
"status": "available",
"scheduled_departure_time_est": "23:00:00",
"scheduled_arrival_time_est": "03:00:00+1",
"date": null,
"available_seats": {
"basic_economy": 19,
"economy": 9,
"business": 2
},
"prices": {
"basic_economy": 92,
"economy": 164,
"business": 323
}
},
{
"flight_number": "HAT278",
"origin": "SFO",
"destination": "IAH",
"status": "available",
"scheduled_departure_time_est": "16:00:00",
"scheduled_arrival_time_est": "20:00:00",
"date": null,
"available_seats": {
"basic_economy": 13,
"economy": 5,
"business": 3
},
"prices": {
"basic_economy": 67,
"economy": 187,
"business": 443
}
}
]update_reservation_flights15 lines · click to expand
{
"cabin": "economy",
"flights": [
{
"date": "2024-05-19",
"flight_number": "HAT072"
},
{
"date": "2024-05-23",
"flight_number": "HAT278"
}
],
"payment_id": "gift_card_6941833",
"reservation_id": "HXDUBJ"
}46 lines · click to expand
{
"reservation_id": "HXDUBJ",
"user_id": "yara_garcia_1905",
"origin": "IAH",
"destination": "SFO",
"flight_type": "round_trip",
"cabin": "economy",
"flights": [
{
"flight_number": "HAT072",
"origin": "IAH",
"destination": "SFO",
"date": "2024-05-19",
"price": 130
},
{
"flight_number": "HAT278",
"origin": "SFO",
"destination": "IAH",
"date": "2024-05-23",
"price": 187
}
],
"passengers": [
{
"first_name": "Yara",
"last_name": "Garcia",
"dob": "1974-08-15"
}
],
"payment_history": [
{
"payment_id": "gift_card_6941833",
"amount": 533
},
{
"payment_id": "gift_card_6941833",
"amount": -186
}
],
"created_at": "2024-05-04T06:04:48",
"total_baggages": 0,
"nonfree_baggages": 0,
"insurance": "yes",
"status": null
}get_user_details
{
"user_id": "yara_garcia_1905"
}48 lines · click to expand
{
"user_id": "yara_garcia_1905",
"name": {
"first_name": "Yara",
"last_name": "Garcia"
},
"address": {
"address1": "377 Pine Lane",
"address2": "Suite 694",
"city": "Denver",
"country": "USA",
"state": "CO",
"zip": "80212"
},
"email": "yara.garcia6882@example.com",
"dob": "1974-08-15",
"payment_methods": {
"certificate_2345996": {
"source": "certificate",
"id": "certificate_2345996",
"amount": 150
},
"gift_card_6941833": {
"source": "gift_card",
"id": "gift_card_6941833",
"amount": 338
},
"gift_card_1646646": {
"source": "gift_card",
"id": "gift_card_1646646",
"amount": 200
}
},
"saved_passengers": [
{
"first_name": "James",
"last_name": "Khan",
"dob": "1959-08-18"
}
],
"membership": "gold",
"reservations": [
"HXDUBJ",
"DXG8T6",
"46BBSE",
"P9VXUO"
]
}get_flight_status
{
"date": "2024-05-19",
"flight_number": "HAT072"
}get_flight_status
{
"date": "2024-05-23",
"flight_number": "HAT278"
}result
available
result
available
update_reservation_flights15 lines · click to expand
{
"cabin": "business",
"flights": [
{
"date": "2024-05-19",
"flight_number": "HAT072"
},
{
"date": "2024-05-23",
"flight_number": "HAT278"
}
],
"payment_id": "gift_card_6941833",
"reservation_id": "HXDUBJ"
}result
Error: Gift card balance is not enough
get_flight_status
{
"date": "2024-05-19",
"flight_number": "HAT072"
}get_flight_status
{
"date": "2024-05-23",
"flight_number": "HAT278"
}result
available
result
available
update_reservation_baggages
{
"payment_id": "gift_card_6941833",
"reservation_id": "HXDUBJ",
"total_baggages": 2,
"nonfree_baggages": 0
}46 lines · click to expand
{
"reservation_id": "HXDUBJ",
"user_id": "yara_garcia_1905",
"origin": "IAH",
"destination": "SFO",
"flight_type": "round_trip",
"cabin": "economy",
"flights": [
{
"flight_number": "HAT072",
"origin": "IAH",
"destination": "SFO",
"date": "2024-05-19",
"price": 130
},
{
"flight_number": "HAT278",
"origin": "SFO",
"destination": "IAH",
"date": "2024-05-23",
"price": 187
}
],
"passengers": [
{
"first_name": "Yara",
"last_name": "Garcia",
"dob": "1974-08-15"
}
],
"payment_history": [
{
"payment_id": "gift_card_6941833",
"amount": 533
},
{
"payment_id": "gift_card_6941833",
"amount": -186
}
],
"created_at": "2024-05-04T06:04:48",
"total_baggages": 2,
"nonfree_baggages": 0,
"insurance": "yes",
"status": null
}