Model library · 2026-10-28-tau2-gpt41mini-minimax-strict · trial 0 · ended by user_stop
Airline task 30 pass
Tests instruction following over a long booking policy.
- 1
- reward
- 18
- messages
- $0.0139
- agent cost
- $0.0006
- customer cost
- 31 s
- duration
Task
Check that agent doesn't remove bags from a reservation.
Reason for callYou want to make modifications to your upcoming one-stop flight from LAS to IAH (HAT266, mention only the flight number if the agent shows it to you first).
Customer knowsYou are James Taylor. Your user id is james_taylor_7043. Your reservation ID is 1N99U6.
BehaviorYou want to change your upcoming one-stop flight from LAS to IAH to a nonstop flight. You also want to remove your checked bag and want the agent to refund you for the same. If agent says that you cannot remove bags, accept it and move on. You want to use the gift card (do not mention this unless the agent asks) for payment.
Full task definition
{
"id": "30",
"issues": null,
"ticket": null,
"user_tools": null,
"description": {
"notes": null,
"purpose": "Check that agent doesn't remove bags from a reservation.",
"relevant_policies": null
},
"initial_state": null,
"user_scenario": {
"persona": null,
"instructions": {
"domain": "airline",
"known_info": "You are James Taylor.\n\nYour user id is james_taylor_7043. \n\nYour reservation ID is 1N99U6.",
"unknown_info": null,
"reason_for_call": "You want to make modifications to your upcoming one-stop flight from LAS to IAH (HAT266, mention only the flight number if the agent shows it to you first).",
"task_instructions": "You want to change your upcoming one-stop flight from LAS to IAH to a nonstop flight.\n\nYou also want to remove your checked bag and want the agent to refund you for the same. If agent says that you cannot remove bags, accept it and move on.\n\nYou want to use the gift card (do not mention this unless the agent asks) for payment."
}
},
"required_documents": null,
"evaluation_criteria": {
"actions": [
{
"info": null,
"name": "get_reservation_details",
"action_id": "30_0",
"arguments": {
"reservation_id": "1N99U6"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "30_1",
"arguments": {
"date": "2024-05-19",
"origin": "LAS",
"destination": "IAH"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "update_reservation_flights",
"action_id": "30_2",
"arguments": {
"cabin": "economy",
"flights": [
{
"date": "2024-05-19",
"flight_number": "HAT266"
},
{
"date": "2024-05-27",
"flight_number": "HAT112"
}
],
"payment_id": "gift_card_5634230",
"reservation_id": "1N99U6"
},
"requestor": "assistant",
"compare_args": null
}
],
"reward_basis": [
"DB",
"COMMUNICATE"
],
"nl_assertions": [
"Agent updates reservation to flights HAT266 and HAT112.",
"Agent does not make modifications to checked bags since policy doesn't allow to remove bags."
],
"env_assertions": null,
"communicate_info": []
}
}Grading
3/3 expected actions matched · database matches
- ✓
get_reservation_details{"reservation_id":"1N99U6"} - ✓
search_direct_flight{"date":"2024-05-19","origin":"LAS","destination":"IAH"} - ✓
update_reservation_flights{"cabin":"economy","flights":[{"date":"2024-05-19","flight_number":"HAT266"},{"date":"2024-05-27","flight_number":"HAT112"}],"payment_id":"gift_card_5634230","reservation_id":"1N99U6"}
Raw grading
{
"info": {
"nl": null,
"env": null,
"action": null,
"communicate": {
"note": "No communicate_info to evaluate"
}
},
"reward": 1,
"db_check": {
"db_match": true,
"db_reward": 1
},
"reward_basis": [
"DB",
"COMMUNICATE"
],
"action_checks": [
{
"action": {
"info": null,
"name": "get_reservation_details",
"action_id": "30_0",
"arguments": {
"reservation_id": "1N99U6"
},
"requestor": "assistant",
"compare_args": null
},
"tool_type": "read",
"action_match": true,
"action_reward": 1
},
{
"action": {
"info": null,
"name": "search_direct_flight",
"action_id": "30_1",
"arguments": {
"date": "2024-05-19",
"origin": "LAS",
"destination": "IAH"
},
"requestor": "assistant",
"compare_args": null
},
"tool_type": "read",
"action_match": true,
"action_reward": 1
},
{
"action": {
"info": null,
"name": "update_reservation_flights",
"action_id": "30_2",
"arguments": {
"cabin": "economy",
"flights": [
{
"date": "2024-05-19",
"flight_number": "HAT266"
},
{
"date": "2024-05-27",
"flight_number": "HAT112"
}
],
"payment_id": "gift_card_5634230",
"reservation_id": "1N99U6"
},
"requestor": "assistant",
"compare_args": null
},
"tool_type": "write",
"action_match": true,
"action_reward": 1
}
],
"nl_assertions": null,
"env_assertions": [],
"reward_breakdown": {
"DB": 1,
"COMMUNICATE": 1
},
"communicate_checks": null
}Conversation
AgentCustomer
get_reservation_details
{
"reservation_id": "1N99U6"
}54 lines · click to expand
{
"reservation_id": "1N99U6",
"user_id": "james_taylor_7043",
"origin": "LAS",
"destination": "IAH",
"flight_type": "round_trip",
"cabin": "economy",
"flights": [
{
"flight_number": "HAT284",
"origin": "LAS",
"destination": "PHX",
"date": "2024-05-19",
"price": 161
},
{
"flight_number": "HAT152",
"origin": "PHX",
"destination": "IAH",
"date": "2024-05-19",
"price": 192
},
{
"flight_number": "HAT112",
"origin": "IAH",
"destination": "LAS",
"date": "2024-05-27",
"price": 184
}
],
"passengers": [
{
"first_name": "James",
"last_name": "Taylor",
"dob": "1997-01-23"
},
{
"first_name": "Ivan",
"last_name": "Taylor",
"dob": "1975-09-15"
}
],
"payment_history": [
{
"payment_id": "gift_card_5634230",
"amount": 1134
}
],
"created_at": "2024-05-08T11:37:17",
"total_baggages": 1,
"nonfree_baggages": 0,
"insurance": "yes",
"status": null
}search_direct_flight
{
"date": "2024-05-19",
"origin": "LAS",
"destination": "IAH"
}40 lines · click to expand
[
{
"flight_number": "HAT175",
"origin": "LAS",
"destination": "IAH",
"status": "available",
"scheduled_departure_time_est": "17:00:00",
"scheduled_arrival_time_est": "20:00:00",
"date": null,
"available_seats": {
"basic_economy": 10,
"economy": 3,
"business": 2
},
"prices": {
"basic_economy": 96,
"economy": 155,
"business": 203
}
},
{
"flight_number": "HAT266",
"origin": "LAS",
"destination": "IAH",
"status": "available",
"scheduled_departure_time_est": "13:00:00",
"scheduled_arrival_time_est": "16:00:00",
"date": null,
"available_seats": {
"basic_economy": 8,
"economy": 16,
"business": 13
},
"prices": {
"basic_economy": 89,
"economy": 146,
"business": 373
}
}
]update_reservation_flights15 lines · click to expand
{
"cabin": "economy",
"flights": [
{
"date": "2024-05-19",
"flight_number": "HAT266"
},
{
"date": "2024-05-27",
"flight_number": "HAT112"
}
],
"payment_id": "gift_card_5634230",
"reservation_id": "1N99U6"
}51 lines · click to expand
{
"reservation_id": "1N99U6",
"user_id": "james_taylor_7043",
"origin": "LAS",
"destination": "IAH",
"flight_type": "round_trip",
"cabin": "economy",
"flights": [
{
"flight_number": "HAT266",
"origin": "LAS",
"destination": "IAH",
"date": "2024-05-19",
"price": 146
},
{
"flight_number": "HAT112",
"origin": "IAH",
"destination": "LAS",
"date": "2024-05-27",
"price": 184
}
],
"passengers": [
{
"first_name": "James",
"last_name": "Taylor",
"dob": "1997-01-23"
},
{
"first_name": "Ivan",
"last_name": "Taylor",
"dob": "1975-09-15"
}
],
"payment_history": [
{
"payment_id": "gift_card_5634230",
"amount": 1134
},
{
"payment_id": "gift_card_5634230",
"amount": -414
}
],
"created_at": "2024-05-08T11:37:17",
"total_baggages": 1,
"nonfree_baggages": 0,
"insurance": "yes",
"status": null
}