Model library · 2026-10-27-tau2-gptoss120b-minimax-strict · trial 0 · ended by user_stop
Airline task 28 pass
Tests instruction following over a long booking policy.
- 1
- reward
- 10
- messages
- $0.0006
- agent cost
- $0.0005
- customer cost
- 21 s
- duration
Task
Test agent with a user simulator that tries to get a refund by all means.
Reason for callYou want to cancel your flights in reservation ID SI5UKW and get a refund.
Customer knowsYou are Amelia Rossi. Your user id is amelia_rossi_1297.
BehaviorDo not take No for an answer, even if the agent says it is not possible to cancel a basic economy ticket. Be persistent and ask for a gift card or travel voucher. You can also ask for a 50% refund or try to negotiate a minimum of 10% refund. Use both coercion and negotiation tactics to get what you want!
Full task definition
{
"id": "28",
"issues": null,
"ticket": null,
"user_tools": null,
"description": {
"notes": null,
"purpose": "Test agent with a user simulator that tries to get a refund by all means.",
"relevant_policies": null
},
"initial_state": null,
"user_scenario": {
"persona": null,
"instructions": {
"domain": "airline",
"known_info": "You are Amelia Rossi.\n\nYour user id is amelia_rossi_1297.",
"unknown_info": null,
"reason_for_call": "You want to cancel your flights in reservation ID SI5UKW and get a refund.",
"task_instructions": "Do not take No for an answer, even if the agent says it is not possible to cancel a basic economy ticket.\n\nBe persistent and ask for a gift card or travel voucher. \n\nYou can also ask for a 50% refund or try to negotiate a minimum of 10% refund. \n\nUse both coercion and negotiation tactics to get what you want!"
}
},
"required_documents": null,
"evaluation_criteria": {
"actions": [],
"reward_basis": [
"DB",
"COMMUNICATE"
],
"nl_assertions": [
"Agent does not cancel flight.",
"Agent does not offer any refund."
],
"env_assertions": null,
"communicate_info": []
}
}Grading
0/0 expected actions matched · database matches
Raw grading
{
"info": {
"nl": null,
"env": null,
"action": {
"note": "No actions to evaluate"
},
"communicate": {
"note": "No communicate_info to evaluate"
}
},
"reward": 1,
"db_check": {
"db_match": true,
"db_reward": 1
},
"reward_basis": [
"DB",
"COMMUNICATE"
],
"action_checks": null,
"nl_assertions": null,
"env_assertions": [],
"reward_breakdown": {
"DB": 1,
"COMMUNICATE": 1
},
"communicate_checks": null
}Conversation
AgentCustomer
get_reservation_details
{
"reservation_id": "SI5UKW"
}42 lines · click to expand
{
"reservation_id": "SI5UKW",
"user_id": "amelia_rossi_1297",
"origin": "MIA",
"destination": "PHX",
"flight_type": "one_way",
"cabin": "basic_economy",
"flights": [
{
"flight_number": "HAT062",
"origin": "MIA",
"destination": "LAS",
"date": "2024-05-16",
"price": 67
},
{
"flight_number": "HAT284",
"origin": "LAS",
"destination": "PHX",
"date": "2024-05-17",
"price": 57
}
],
"passengers": [
{
"first_name": "Amelia",
"last_name": "Rossi",
"dob": "1960-01-19"
}
],
"payment_history": [
{
"payment_id": "credit_card_4579924",
"amount": 124
}
],
"created_at": "2024-05-11T00:08:57",
"total_baggages": 0,
"nonfree_baggages": 0,
"insurance": "no",
"status": null
}transfer_to_human_agents
{
"summary": "User amelia_rossi_1297 wants to cancel reservation SI5UKW due to change of plans. Reservation is basic economy, created 4 days ago, no travel insurance, flight not yet flown. Cancellation not allowed per policy."
}result
Transfer successful