Model library · 2026-11-07-tau3-gemma4-12b-minimax-strict · trial 0 · ended by infrastructure_error
Airline task 17 error
Tests instruction following over a long booking policy.
- —
- reward
- 0
- messages
- $0.0000
- agent cost
- —
- customer cost
- 0 s
- duration
Task
Testing if the agent can handle user asking for 3 changes at once.
Reason for callFor your upcoming trip from New York to Chicago, you want to: - add 3 checked bags - change the passenger to yourself - upgrade it to economy class. Mention all three things at once and in this order.
Customer knowsYour name is Omar Rossi. Your user id is omar_rossi_1241.
BehaviorYou prefer gift card payment. Your birthday is in your user profile so you prefer not to provide it.
Full task definition
{
"id": "17",
"issues": null,
"ticket": null,
"user_tools": null,
"description": {
"notes": null,
"purpose": "Testing if the agent can handle user asking for 3 changes at once.",
"relevant_policies": null
},
"initial_state": null,
"user_scenario": {
"persona": null,
"instructions": {
"domain": "airline",
"known_info": "Your name is Omar Rossi.\n\nYour user id is omar_rossi_1241.",
"unknown_info": null,
"reason_for_call": "For your upcoming trip from New York to Chicago, you want to:\n- add 3 checked bags\n- change the passenger to yourself\n- upgrade it to economy class. \n\nMention all three things at once and in this order.",
"task_instructions": "You prefer gift card payment.\n\nYour birthday is in your user profile so you prefer not to provide it."
}
},
"required_documents": null,
"evaluation_criteria": {
"actions": [
{
"info": null,
"name": "update_reservation_flights",
"action_id": "17_0",
"arguments": {
"cabin": "economy",
"flights": [
{
"date": "2024-05-25",
"flight_number": "HAT056"
},
{
"date": "2024-05-25",
"flight_number": "HAT138"
}
],
"payment_id": "gift_card_8190333",
"reservation_id": "FQ8APE"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "update_reservation_passengers",
"action_id": "17_1",
"arguments": {
"passengers": [
{
"dob": "1970-06-06",
"last_name": "Rossi",
"first_name": "Omar"
}
],
"reservation_id": "FQ8APE"
},
"requestor": "assistant",
"compare_args": null
},
{
"info": null,
"name": "update_reservation_baggages",
"action_id": "17_2",
"arguments": {
"payment_id": "gift_card_8190333",
"reservation_id": "FQ8APE",
"total_baggages": 3,
"nonfree_baggages": 0
},
"requestor": "assistant",
"compare_args": null
}
],
"reward_basis": [
"DB",
"COMMUNICATE"
],
"nl_assertions": [
"Reservation FQ8APE is updated to economy.",
"Passenger for reservation FQ8APE is updated to Omar Rossi.",
"Number of bags for reservation FQ8APE is updated to 3."
],
"env_assertions": null,
"communicate_info": []
}
}Grading
0/0 expected actions matched
Raw grading
nullConversation
AgentCustomer