Model library · 2026-10-27-tau2-grok420-minimax-strict · trial 1 · ended by infrastructure_error
Airline task 44 error
Tests instruction following over a long booking policy.
- —
- reward
- 0
- messages
- $0.0000
- agent cost
- —
- customer cost
- 0 s
- duration
Task
Test that agent can collect information about reservation, reason about durations as well as cancellation and upgrades options.
Reason for callYou want to cancel all your future reservations that contain any flights that are longer than 4 hours. You need the agent to tell you the flight durations to make the right decision. For the flights that are under or equal to 3 hours (including layovers), ask the agent to upgrade you to business wherever possible.
Customer knowsYou are Sophia Silva. Your user id is sophia_silva_7557.
BehaviorYou are busy so for both the cancellation and upgrade you want to let the agent figure out which flights meet the duration conditions you have set. Before they do the upgrade to business, ask the agent to tell you how much it will cost you in total. You DO NOT want to upgrade to business and then cancel it. You do NOT have a health related issue, you are healthy now.
Full task definition
{
"id": "44",
"issues": null,
"ticket": null,
"user_tools": null,
"description": {
"notes": null,
"purpose": "Test that agent can collect information about reservation, reason about durations as well as cancellation and upgrades options.",
"relevant_policies": null
},
"initial_state": null,
"user_scenario": {
"persona": null,
"instructions": {
"domain": "airline",
"known_info": "You are Sophia Silva.\nYour user id is sophia_silva_7557.",
"unknown_info": null,
"reason_for_call": "You want to cancel all your future reservations that contain any flights that are longer than 4 hours. \n\nYou need the agent to tell you the flight durations to make the right decision. For the flights that are under or equal to 3 hours (including layovers), ask the agent to upgrade you to business wherever possible.",
"task_instructions": "You are busy so for both the cancellation and upgrade you want to let the agent figure out which flights meet the duration conditions you have set.\n\nBefore they do the upgrade to business, ask the agent to tell you how much it will cost you in total.\n\nYou DO NOT want to upgrade to business and then cancel it. You do NOT have a health related issue, you are healthy now."
}
},
"required_documents": null,
"evaluation_criteria": {
"actions": [
{
"info": null,
"name": "get_user_details",
"action_id": "44_0",
"arguments": {
"user_id": "sophia_silva_7557"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "get_reservation_details",
"action_id": "44_1",
"arguments": {
"reservation_id": "NM1VX1"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "get_reservation_details",
"action_id": "44_2",
"arguments": {
"reservation_id": "KC18K6"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "get_reservation_details",
"action_id": "44_3",
"arguments": {
"reservation_id": "S61CZX"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "get_reservation_details",
"action_id": "44_4",
"arguments": {
"reservation_id": "H8Q05L"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "get_reservation_details",
"action_id": "44_5",
"arguments": {
"reservation_id": "WUNA5K"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "44_6",
"arguments": {
"date": "2024-05-25",
"origin": "MSP",
"destination": "EWR"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "44_7",
"arguments": {
"date": "2024-05-27",
"origin": "EWR",
"destination": "MSP"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "44_8",
"arguments": {
"date": "2024-05-21",
"origin": "MSP",
"destination": "EWR"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "44_9",
"arguments": {
"date": "2024-05-21",
"origin": "EWR",
"destination": "CLT"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "44_10",
"arguments": {
"date": "2024-05-23",
"origin": "LAX",
"destination": "EWR"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "44_11",
"arguments": {
"date": "2024-05-24",
"origin": "EWR",
"destination": "CLT"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "44_12",
"arguments": {
"date": "2024-05-24",
"origin": "CLT",
"destination": "EWR"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "44_13",
"arguments": {
"date": "2024-05-25",
"origin": "EWR",
"destination": "LAX"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "44_14",
"arguments": {
"date": "2024-05-24",
"origin": "JFK",
"destination": "ATL"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "search_direct_flight",
"action_id": "44_15",
"arguments": {
"date": "2024-05-10",
"origin": "ORD",
"destination": "PHL"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "update_reservation_flights",
"action_id": "44_17",
"arguments": {
"cabin": "business",
"flights": [
{
"date": "2024-05-25",
"flight_number": "HAT300"
},
{
"date": "2024-05-27",
"flight_number": "HAT208"
}
],
"payment_id": "credit_card_4196779",
"reservation_id": "NM1VX1"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "update_reservation_flights",
"action_id": "44_18",
"arguments": {
"cabin": "business",
"flights": [
{
"date": "2024-05-24",
"flight_number": "HAT268"
}
],
"payment_id": "credit_card_4196779",
"reservation_id": "H8Q05L"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "update_reservation_flights",
"action_id": "44_19",
"arguments": {
"cabin": "business",
"flights": [
{
"date": "2024-05-21",
"flight_number": "HAT300"
},
{
"date": "2024-05-21",
"flight_number": "HAT215"
}
],
"payment_id": "credit_card_4196779",
"reservation_id": "KC18K6"
},
"requestor": "assistant",
"compare_args": null
}
],
"reward_basis": [
"DB",
"COMMUNICATE"
],
"nl_assertions": [
"Agent does not cancel reservation S61CZX as the user is healthy.",
"The total cost that the agent mentions is between $1380 and $1390.",
"Agent upgrades NM1VX1 to business.",
"Agent upgrades H8Q05L to business.",
"Agent updates KC18K6 to business."
],
"env_assertions": null,
"communicate_info": []
}
}Grading
0/0 expected actions matched
Raw grading
nullConversation
AgentCustomer