{"caveats":["Costs are not comparable across harnesses: some cells carry the vendor's own figure and others are derived from list prices, and token counts partly measure the verbosity of our own instructions.","Each cell is backed by one to three rollouts per case; a one-case gap is noise.","Cells marked stale were measured under an older case manifest or an unpinned tool surface, and are not comparable with cells on another one.","A harness that reports no token usage carries no dollar figure rather than a zero."],"description":"How each agent harness and model scores on real CFO work against a live ledger — score, cost, and the cases behind every cell. Generated from Economico's own eval dataset.","last_run":"2026-08-18T07:04:36Z","manifests":[{"cells":77,"current":true,"last_run":"2026-08-18T07:04:36Z","sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db"}],"pairings":[{"cost_source":"harness","cost_usd":6.610692899999999,"harness":"claude","high":1,"key":"claude__claude-haiku-4-5","last_run":"2026-08-17T19:56:57Z","low":0.3333333333333333,"max_samples":1,"mean":0.8606060606060606,"min_samples":1,"model":"claude-haiku-4-5","priced":11,"skills":11,"stale":false},{"cost_source":"harness","cost_usd":35.203998500000004,"effort":"medium","harness":"claude","high":1,"key":"claude__claude-opus-5__medium","last_run":"2026-08-18T07:04:36Z","low":1,"max_samples":3,"mean":1,"min_samples":1,"model":"claude-opus-5","priced":11,"skills":11,"stale":false},{"cost_source":"harness","cost_usd":14.9407403,"effort":"medium","harness":"claude","high":1,"key":"claude__claude-sonnet-5__medium","last_run":"2026-08-17T12:56:50Z","low":1,"max_samples":1,"mean":1,"min_samples":1,"model":"claude-sonnet-5","priced":11,"skills":11,"stale":false},{"cost_source":"estimated","cost_usd":0.88717532,"harness":"codex","high":1,"key":"codex__gpt-5.6-luna","last_run":"2026-08-18T06:37:11Z","low":0.2222222222222222,"max_samples":3,"mean":0.898989898989899,"min_samples":1,"model":"gpt-5.6-luna","priced":11,"skills":11,"stale":false},{"cost_source":"estimated","cost_usd":14.635252999999999,"harness":"codex","high":1,"key":"codex__gpt-5.6-sol","last_run":"2026-08-17T12:40:04Z","low":1,"max_samples":1,"mean":1,"min_samples":1,"model":"gpt-5.6-sol","priced":11,"skills":11,"stale":false},{"cost_source":"estimated","cost_usd":8.4487796,"harness":"codex","high":1,"key":"codex__gpt-5.6-terra","last_run":"2026-08-18T05:15:40Z","low":0.7777777777777777,"max_samples":3,"mean":0.9797979797979799,"min_samples":1,"model":"gpt-5.6-terra","priced":11,"skills":11,"stale":false},{"cost_source":"harness","cost_usd":4.33148848,"harness":"grok","high":1,"key":"grok__grok-4.6","last_run":"2026-08-17T13:26:29Z","low":1,"max_samples":1,"mean":1,"min_samples":1,"model":"grok-4.6","priced":11,"skills":11,"stale":false}],"prices_as_of":"2026-08-16","reports":77,"schema_version":"benchmarks/v1","skills":[{"cases":[{"id":"company-accounts-one-pool","of":7,"passed":6},{"id":"company-captable-safe","of":7,"passed":7},{"id":"company-profile-wrong-entity","of":7,"passed":7}],"cells":[{"cache_read_tokens":777708,"cache_write_tokens":88937,"cases":[{"id":"company-accounts-one-pool","pass":false,"samples":1,"score":0},{"id":"company-captable-safe","pass":true,"samples":1,"score":1},{"id":"company-profile-wrong-entity","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:05:40Z","cost_source":"harness","cost_usd":0.2972068,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":2552,"output_tokens":14939,"pairing":"claude__claude-haiku-4-5","passed":2,"samples":1,"score":0.6666666666666666,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":12},{"cache_read_tokens":926203,"cache_write_tokens":55939,"cases":[{"id":"company-accounts-one-pool","pass":true,"samples":1,"score":1},{"id":"company-captable-safe","pass":true,"samples":1,"score":1},{"id":"company-profile-wrong-entity","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:14:25Z","cost_source":"harness","cost_usd":1.1985105,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":2171,"output_tokens":6972,"pairing":"claude__claude-opus-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":36},{"cache_read_tokens":641062,"cache_write_tokens":50778,"cases":[{"id":"company-accounts-one-pool","pass":true,"samples":1,"score":1},{"id":"company-captable-safe","pass":true,"samples":1,"score":1},{"id":"company-profile-wrong-entity","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:12:22Z","cost_source":"harness","cost_usd":0.6009226,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":2118,"output_tokens":6808,"pairing":"claude__claude-sonnet-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":32},{"cache_read_tokens":769280,"cases":[{"id":"company-accounts-one-pool","pass":true,"samples":1,"score":1},{"id":"company-captable-safe","pass":true,"samples":1,"score":1},{"id":"company-profile-wrong-entity","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:05:47Z","cost_source":"estimated","cost_usd":0.0404708,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":99578,"output_tokens":4308,"pairing":"codex__gpt-5.6-luna","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":722432,"cases":[{"id":"company-accounts-one-pool","pass":true,"samples":1,"score":1},{"id":"company-captable-safe","pass":true,"samples":1,"score":1},{"id":"company-profile-wrong-entity","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:07:04Z","cost_source":"estimated","cost_usd":1.184011,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":139215,"output_tokens":4224,"pairing":"codex__gpt-5.6-sol","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":731136,"cases":[{"id":"company-accounts-one-pool","pass":true,"samples":1,"score":1},{"id":"company-captable-safe","pass":true,"samples":1,"score":1},{"id":"company-profile-wrong-entity","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:46:23Z","cost_source":"estimated","cost_usd":0.40447520000000003,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":106618,"output_tokens":3751,"pairing":"codex__gpt-5.6-terra","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":1675648,"cases":[{"id":"company-accounts-one-pool","pass":true,"samples":1,"score":1},{"id":"company-captable-safe","pass":true,"samples":1,"score":1},{"id":"company-profile-wrong-entity","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:16:15Z","cost_source":"harness","cost_usd":0.24464428,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":245760,"output_tokens":18290,"pairing":"grok__grok-4.6","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":25}],"skill":"company-setup","what":"First-run setup of the business itself — the legal-entity profile, named bank and wallet accounts as one pool of value each, and a cap table with founder shares and an early SAFE."},{"cases":[{"id":"cpa-cogs-opex-classification","of":7,"passed":7},{"id":"cpa-period-end-accruals-recommend","of":7,"passed":7}],"cells":[{"cache_read_tokens":608170,"cache_write_tokens":90043,"cases":[{"id":"cpa-cogs-opex-classification","pass":true,"samples":1,"score":1},{"id":"cpa-period-end-accruals-recommend","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T14:02:43Z","cost_source":"harness","cost_usd":0.286037,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":1663,"output_tokens":15224,"pairing":"claude__claude-haiku-4-5","passed":2,"samples":1,"score":1,"scored":2,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":33},{"cache_read_tokens":1725475,"cache_write_tokens":245284,"cases":[{"id":"cpa-cogs-opex-classification","pass":true,"samples":3,"score":1},{"id":"cpa-period-end-accruals-recommend","pass":true,"samples":3,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-18T07:04:36Z","cost_source":"harness","cost_usd":4.320062500000001,"git_sha":"8402e53da8cd930a9bcae3fa747ff851f94f691f","input_tokens":6394,"output_tokens":39979,"pairing":"claude__claude-opus-5__medium","passed":2,"samples":3,"score":1,"scored":2,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":106},{"cache_read_tokens":472644,"cache_write_tokens":76844,"cases":[{"id":"cpa-cogs-opex-classification","pass":true,"samples":1,"score":1},{"id":"cpa-period-end-accruals-recommend","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:43:39Z","cost_source":"harness","cost_usd":0.8007941999999999,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":1376,"output_tokens":13121,"pairing":"claude__claude-sonnet-5__medium","passed":2,"samples":1,"score":1,"scored":2,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":32},{"cache_read_tokens":634112,"cases":[{"id":"cpa-cogs-opex-classification","pass":true,"samples":1,"score":1},{"id":"cpa-period-end-accruals-recommend","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:30:38Z","cost_source":"estimated","cost_usd":0.039702440000000006,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":94547,"output_tokens":6759,"pairing":"codex__gpt-5.6-luna","passed":2,"samples":1,"score":1,"scored":2,"stale":false,"std_err":0,"turns":2},{"cache_read_tokens":625152,"cases":[{"id":"cpa-cogs-opex-classification","pass":true,"samples":1,"score":1},{"id":"cpa-period-end-accruals-recommend","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:32:11Z","cost_source":"estimated","cost_usd":0.9892409999999999,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":98181,"output_tokens":6192,"pairing":"codex__gpt-5.6-sol","passed":2,"samples":1,"score":1,"scored":2,"stale":false,"std_err":0,"turns":2},{"cache_read_tokens":496128,"cases":[{"id":"cpa-cogs-opex-classification","pass":true,"samples":1,"score":1},{"id":"cpa-period-end-accruals-recommend","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:11:29Z","cost_source":"estimated","cost_usd":0.3646396,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":99641,"output_tokens":5511,"pairing":"codex__gpt-5.6-terra","passed":2,"samples":1,"score":1,"scored":2,"stale":false,"std_err":0,"turns":2},{"cache_read_tokens":1369856,"cases":[{"id":"cpa-cogs-opex-classification","pass":true,"samples":1,"score":1},{"id":"cpa-period-end-accruals-recommend","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:03:47Z","cost_source":"harness","cost_usd":0.22485832,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":226001,"output_tokens":30961,"pairing":"grok__grok-4.6","passed":2,"samples":1,"score":1,"scored":2,"stale":false,"std_err":0,"turns":18}],"skill":"cpa","what":"An auditor's read: period-end accruals to recommend, and COGS-versus-OpEx classification calls — findings only, never posting."},{"cases":[{"id":"contracts-annual-saas","of":7,"passed":7},{"id":"contracts-upsell-amendment","of":7,"passed":7},{"id":"contracts-vendor-llm-usage","of":7,"passed":7}],"cells":[{"cache_read_tokens":2707849,"cache_write_tokens":237316,"cases":[{"id":"contracts-annual-saas","pass":true,"samples":2,"score":1},{"id":"contracts-upsell-amendment","pass":true,"samples":1,"score":1},{"id":"contracts-vendor-llm-usage","pass":true,"samples":2,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T17:39:17Z","cost_source":"harness","cost_usd":1.0661806999999999,"git_sha":"df04c61366d398d4352db485e1b44d81f602b7e9","input_tokens":4442,"output_tokens":41025,"pairing":"claude__claude-haiku-4-5","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":5},{"cache_read_tokens":1230639,"cache_write_tokens":91882,"cases":[{"id":"contracts-annual-saas","pass":true,"samples":2,"score":1},{"id":"contracts-upsell-amendment","pass":true,"samples":1,"score":1},{"id":"contracts-vendor-llm-usage","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T17:33:02Z","cost_source":"harness","cost_usd":1.7731175000000001,"git_sha":"df04c61366d398d4352db485e1b44d81f602b7e9","input_tokens":2727,"output_tokens":9495,"pairing":"claude__claude-opus-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":45},{"cache_read_tokens":805705,"cache_write_tokens":65795,"cases":[{"id":"contracts-annual-saas","pass":true,"samples":1,"score":1},{"id":"contracts-upsell-amendment","pass":true,"samples":1,"score":1},{"id":"contracts-vendor-llm-usage","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:20:02Z","cost_source":"harness","cost_usd":0.7364515,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":2063,"output_tokens":6557,"pairing":"claude__claude-sonnet-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":30},{"cache_read_tokens":1324800,"cases":[{"id":"contracts-annual-saas","pass":true,"samples":2,"score":1},{"id":"contracts-upsell-amendment","pass":true,"samples":1,"score":1},{"id":"contracts-vendor-llm-usage","pass":true,"samples":2,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T17:35:13Z","cost_source":"estimated","cost_usd":0.0680962,"git_sha":"df04c61366d398d4352db485e1b44d81f602b7e9","input_tokens":159533,"output_tokens":8078,"pairing":"codex__gpt-5.6-luna","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":5},{"cache_read_tokens":1129216,"cases":[{"id":"contracts-annual-saas","pass":true,"samples":1,"score":1},{"id":"contracts-upsell-amendment","pass":true,"samples":1,"score":1},{"id":"contracts-vendor-llm-usage","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:13:43Z","cost_source":"estimated","cost_usd":1.374288,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":133454,"output_tokens":4747,"pairing":"codex__gpt-5.6-sol","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":1387008,"cases":[{"id":"contracts-annual-saas","pass":true,"samples":2,"score":1},{"id":"contracts-upsell-amendment","pass":true,"samples":1,"score":1},{"id":"contracts-vendor-llm-usage","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T17:25:13Z","cost_source":"estimated","cost_usd":0.6567896,"git_sha":"df04c61366d398d4352db485e1b44d81f602b7e9","input_tokens":148384,"output_tokens":6885,"pairing":"codex__gpt-5.6-terra","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":4},{"cache_read_tokens":2068224,"cases":[{"id":"contracts-annual-saas","pass":true,"samples":1,"score":1},{"id":"contracts-upsell-amendment","pass":true,"samples":1,"score":1},{"id":"contracts-vendor-llm-usage","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:28:15Z","cost_source":"harness","cost_usd":0.28594646,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":279827,"output_tokens":14712,"pairing":"grok__grok-4.6","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":28}],"skill":"creating-contracts","what":"Build the contract spine a bill or invoice hangs off — an annual SaaS order form, a usage-metered vendor contract, and an upsell that must amend the existing contract instead of replacing it."},{"cases":[{"id":"expense-aws-already-booked","of":7,"passed":7},{"id":"expense-openai-usage-spine","of":7,"passed":7},{"id":"expense-render-dev-rnd","of":7,"passed":7}],"cells":[{"cache_read_tokens":2000836,"cache_write_tokens":208301,"cases":[{"id":"expense-aws-already-booked","pass":true,"samples":1,"score":1},{"id":"expense-openai-usage-spine","pass":true,"samples":1,"score":1},{"id":"expense-render-dev-rnd","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:41:49Z","cost_source":"harness","cost_usd":0.6958936000000002,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":3030,"output_tokens":33166,"pairing":"claude__claude-haiku-4-5","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":34},{"cache_read_tokens":1762710,"cache_write_tokens":115576,"cases":[{"id":"expense-aws-already-booked","pass":true,"samples":1,"score":1},{"id":"expense-openai-usage-spine","pass":true,"samples":1,"score":1},{"id":"expense-render-dev-rnd","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:34:56Z","cost_source":"harness","cost_usd":2.390684,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":2552,"output_tokens":13989,"pairing":"claude__claude-opus-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":62},{"cache_read_tokens":2001524,"cache_write_tokens":110312,"cases":[{"id":"expense-aws-already-booked","pass":true,"samples":1,"score":1},{"id":"expense-openai-usage-spine","pass":true,"samples":1,"score":1},{"id":"expense-render-dev-rnd","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:30:57Z","cost_source":"harness","cost_usd":1.4936392,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":2066,"output_tokens":15302,"pairing":"claude__claude-sonnet-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":70},{"cache_read_tokens":2010112,"cases":[{"id":"expense-aws-already-booked","pass":true,"samples":1,"score":1},{"id":"expense-openai-usage-spine","pass":true,"samples":1,"score":1},{"id":"expense-render-dev-rnd","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:21:38Z","cost_source":"estimated","cost_usd":0.09274184,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":202458,"output_tokens":10040,"pairing":"codex__gpt-5.6-luna","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":1759232,"cases":[{"id":"expense-aws-already-booked","pass":true,"samples":1,"score":1},{"id":"expense-openai-usage-spine","pass":true,"samples":1,"score":1},{"id":"expense-render-dev-rnd","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:23:14Z","cost_source":"estimated","cost_usd":1.903911,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":159127,"output_tokens":7622,"pairing":"codex__gpt-5.6-sol","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":2149376,"cases":[{"id":"expense-aws-already-booked","pass":true,"samples":1,"score":1},{"id":"expense-openai-usage-spine","pass":true,"samples":1,"score":1},{"id":"expense-render-dev-rnd","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:02:55Z","cost_source":"estimated","cost_usd":0.8567292000000001,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":163267,"output_tokens":8360,"pairing":"codex__gpt-5.6-terra","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":4297472,"cases":[{"id":"expense-aws-already-booked","pass":true,"samples":1,"score":1},{"id":"expense-openai-usage-spine","pass":true,"samples":1,"score":1},{"id":"expense-render-dev-rnd","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:43:56Z","cost_source":"harness","cost_usd":0.5081385,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":353218,"output_tokens":22313,"pairing":"grok__grok-4.6","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":45}],"skill":"expense-tracking","what":"Accounts payable from a vendor receipt: split infrastructure spend between production COGS and R\u0026D, build the vendor spine for metered LLM usage, and notice when a bill is already booked."},{"cases":[{"id":"analyst-committed-runrate","of":7,"passed":6},{"id":"analyst-dead-annual-writeoff","of":7,"passed":7},{"id":"analyst-quarter-trend-refusal","of":7,"passed":6}],"cells":[{"cache_read_tokens":602827,"cache_write_tokens":115297,"cases":[{"id":"analyst-committed-runrate","pass":false,"samples":1,"score":0},{"id":"analyst-dead-annual-writeoff","pass":true,"samples":1,"score":1},{"id":"analyst-quarter-trend-refusal","pass":false,"samples":1,"score":0}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:58:51Z","cost_source":"harness","cost_usd":0.39174045,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":2319,"output_tokens":29964,"pairing":"claude__claude-haiku-4-5","passed":1,"samples":1,"score":0.3333333333333333,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":16},{"cache_read_tokens":384749,"cache_write_tokens":63718,"cases":[{"id":"analyst-committed-runrate","pass":true,"samples":1,"score":1},{"id":"analyst-dead-annual-writeoff","pass":true,"samples":1,"score":1},{"id":"analyst-quarter-trend-refusal","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:42:54Z","cost_source":"harness","cost_usd":1.0124695,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":1961,"output_tokens":7274,"pairing":"claude__claude-opus-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":29},{"cache_read_tokens":604005,"cache_write_tokens":96122,"cases":[{"id":"analyst-committed-runrate","pass":true,"samples":1,"score":1},{"id":"analyst-dead-annual-writeoff","pass":true,"samples":1,"score":1},{"id":"analyst-quarter-trend-refusal","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:39:47Z","cost_source":"harness","cost_usd":0.9885324999999999,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":1973,"output_tokens":15270,"pairing":"claude__claude-sonnet-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":39},{"cache_read_tokens":553984,"cases":[{"id":"analyst-committed-runrate","pass":true,"samples":1,"score":1},{"id":"analyst-dead-annual-writeoff","pass":true,"samples":1,"score":1},{"id":"analyst-quarter-trend-refusal","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:27:45Z","cost_source":"estimated","cost_usd":0.03911328,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":107906,"output_tokens":5377,"pairing":"codex__gpt-5.6-luna","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":552192,"cases":[{"id":"analyst-committed-runrate","pass":true,"samples":1,"score":1},{"id":"analyst-dead-annual-writeoff","pass":true,"samples":1,"score":1},{"id":"analyst-quarter-trend-refusal","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:29:00Z","cost_source":"estimated","cost_usd":0.961776,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":115398,"output_tokens":3623,"pairing":"codex__gpt-5.6-sol","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":605440,"cases":[{"id":"analyst-committed-runrate","pass":true,"samples":1,"score":1},{"id":"analyst-dead-annual-writeoff","pass":true,"samples":1,"score":1},{"id":"analyst-quarter-trend-refusal","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:08:53Z","cost_source":"estimated","cost_usd":0.423326,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":123933,"output_tokens":4531,"pairing":"codex__gpt-5.6-terra","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":948224,"cases":[{"id":"analyst-committed-runrate","pass":true,"samples":1,"score":1},{"id":"analyst-dead-annual-writeoff","pass":true,"samples":1,"score":1},{"id":"analyst-quarter-trend-refusal","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:54:03Z","cost_source":"harness","cost_usd":0.20557794,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":328135,"output_tokens":13150,"pairing":"grok__grok-4.6","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":14}],"skill":"financial-analyst","what":"Read-only analysis that has to stay grounded in the ledger: committed run-rate, a dead annual contract that needs writing off, and a quarter-over-quarter trend the books cannot actually support — where the right answer is to refuse."},{"cases":[{"id":"forecasting-hiring-plan-overlay","of":7,"passed":6},{"id":"forecasting-price-hike-overlay","of":7,"passed":6},{"id":"forecasting-signed-vendor-bill-real-books","of":7,"passed":6}],"cells":[{"cache_read_tokens":991519,"cache_write_tokens":261707,"cases":[{"id":"forecasting-hiring-plan-overlay","pass":true,"samples":1,"score":1},{"id":"forecasting-price-hike-overlay","pass":true,"samples":1,"score":1},{"id":"forecasting-signed-vendor-bill-real-books","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T14:25:30Z","cost_source":"harness","cost_usd":0.8023326500000001,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":2361,"output_tokens":38541,"pairing":"claude__claude-haiku-4-5","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":92},{"cache_read_tokens":6497985,"cache_write_tokens":1032246,"cases":[{"id":"forecasting-hiring-plan-overlay","pass":true,"samples":3,"score":1},{"id":"forecasting-price-hike-overlay","pass":true,"samples":3,"score":1},{"id":"forecasting-signed-vendor-bill-real-books","pass":true,"samples":3,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-18T06:21:41Z","cost_source":"harness","cost_usd":17.000282499999997,"git_sha":"8402e53da8cd930a9bcae3fa747ff851f94f691f","input_tokens":9599,"output_tokens":136835,"pairing":"claude__claude-opus-5__medium","passed":3,"samples":3,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":401},{"cache_read_tokens":1689461,"cache_write_tokens":747556,"cases":[{"id":"forecasting-hiring-plan-overlay","pass":true,"samples":1,"score":1},{"id":"forecasting-price-hike-overlay","pass":true,"samples":1,"score":1},{"id":"forecasting-signed-vendor-bill-real-books","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:56:50Z","cost_source":"harness","cost_usd":5.955617299999999,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":2069,"output_tokens":64114,"pairing":"claude__claude-sonnet-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":162},{"cache_read_tokens":4054528,"cases":[{"id":"forecasting-hiring-plan-overlay","pass":false,"samples":3,"score":0},{"id":"forecasting-price-hike-overlay","pass":false,"samples":3,"score":0.3333333333333333},{"id":"forecasting-signed-vendor-bill-real-books","pass":false,"samples":3,"score":0.3333333333333333}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-18T06:37:11Z","cost_source":"estimated","cost_usd":0.25093556,"git_sha":"8402e53da8cd930a9bcae3fa747ff851f94f691f","input_tokens":634209,"output_tokens":35836,"pairing":"codex__gpt-5.6-luna","passed":0,"samples":3,"score":0.2222222222222222,"scored":3,"stale":false,"std_err":0.15713484026367724,"turns":9},{"cache_read_tokens":1758720,"cases":[{"id":"forecasting-hiring-plan-overlay","pass":true,"samples":1,"score":1},{"id":"forecasting-price-hike-overlay","pass":true,"samples":1,"score":1},{"id":"forecasting-signed-vendor-bill-real-books","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:40:04Z","cost_source":"estimated","cost_usd":2.215185,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":199641,"output_tokens":11254,"pairing":"codex__gpt-5.6-sol","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":4799488,"cases":[{"id":"forecasting-hiring-plan-overlay","pass":true,"samples":3,"score":0.6666666666666666},{"id":"forecasting-price-hike-overlay","pass":true,"samples":3,"score":0.6666666666666666},{"id":"forecasting-signed-vendor-bill-real-books","pass":true,"samples":3,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-18T05:15:40Z","cost_source":"estimated","cost_usd":2.5810516,"git_sha":"8402e53da8cd930a9bcae3fa747ff851f94f691f","input_tokens":584719,"output_tokens":37643,"pairing":"codex__gpt-5.6-terra","passed":3,"samples":3,"score":0.7777777777777777,"scored":3,"stale":false,"std_err":0.15713484026367724,"turns":9},{"cache_read_tokens":7758336,"cases":[{"id":"forecasting-hiring-plan-overlay","pass":true,"samples":1,"score":1},{"id":"forecasting-price-hike-overlay","pass":true,"samples":1,"score":1},{"id":"forecasting-signed-vendor-bill-real-books","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:26:29Z","cost_source":"harness","cost_usd":1.3785888199999998,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":929463,"output_tokens":73468,"pairing":"grok__grok-4.6","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":62}],"skill":"forecasting","what":"What-ifs that must stay off the real ledger: a price-hike overlay, a hiring-plan overlay, and a signed vendor bill that belongs on the real books rather than in a scenario."},{"cases":[{"id":"investor-reporting-board-share-link","of":7,"passed":7},{"id":"investor-reporting-consulting-retainer-base","of":7,"passed":6},{"id":"investor-reporting-saas-cost-base","of":7,"passed":7}],"cells":[{"cache_read_tokens":428653,"cache_write_tokens":60106,"cases":[{"id":"investor-reporting-board-share-link","pass":true,"samples":1,"score":1},{"id":"investor-reporting-consulting-retainer-base","pass":true,"samples":1,"score":1},{"id":"investor-reporting-saas-cost-base","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T14:06:40Z","cost_source":"harness","cost_usd":0.1956113,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":2168,"output_tokens":9855,"pairing":"claude__claude-haiku-4-5","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":21},{"cache_read_tokens":367086,"cache_write_tokens":45370,"cases":[{"id":"investor-reporting-board-share-link","pass":true,"samples":1,"score":1},{"id":"investor-reporting-consulting-retainer-base","pass":true,"samples":1,"score":1},{"id":"investor-reporting-saas-cost-base","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:50:12Z","cost_source":"harness","cost_usd":0.720936,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":1910,"output_tokens":3307,"pairing":"claude__claude-opus-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":18},{"cache_read_tokens":263328,"cache_write_tokens":40635,"cases":[{"id":"investor-reporting-board-share-link","pass":true,"samples":1,"score":1},{"id":"investor-reporting-consulting-retainer-base","pass":true,"samples":1,"score":1},{"id":"investor-reporting-saas-cost-base","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:46:15Z","cost_source":"harness","cost_usd":0.37095939999999994,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":2639,"output_tokens":2966,"pairing":"claude__claude-sonnet-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":15},{"cache_read_tokens":339200,"cases":[{"id":"investor-reporting-board-share-link","pass":true,"samples":1,"score":1},{"id":"investor-reporting-consulting-retainer-base","pass":false,"samples":1,"score":0},{"id":"investor-reporting-saas-cost-base","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:32:23Z","cost_source":"estimated","cost_usd":0.0249906,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":76213,"output_tokens":2470,"pairing":"codex__gpt-5.6-luna","passed":2,"samples":1,"score":0.6666666666666666,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":343296,"cases":[{"id":"investor-reporting-board-share-link","pass":true,"samples":1,"score":1},{"id":"investor-reporting-consulting-retainer-base","pass":true,"samples":1,"score":1},{"id":"investor-reporting-saas-cost-base","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:33:51Z","cost_source":"estimated","cost_usd":0.625843,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":80657,"output_tokens":1697,"pairing":"codex__gpt-5.6-sol","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":426752,"cases":[{"id":"investor-reporting-board-share-link","pass":true,"samples":1,"score":1},{"id":"investor-reporting-consulting-retainer-base","pass":true,"samples":1,"score":1},{"id":"investor-reporting-saas-cost-base","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:13:07Z","cost_source":"estimated","cost_usd":0.2780364,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":85507,"output_tokens":1806,"pairing":"codex__gpt-5.6-terra","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":566784,"cases":[{"id":"investor-reporting-board-share-link","pass":true,"samples":1,"score":1},{"id":"investor-reporting-consulting-retainer-base","pass":true,"samples":1,"score":1},{"id":"investor-reporting-saas-cost-base","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:06:09Z","cost_source":"harness","cost_usd":0.12281718,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":203649,"output_tokens":5294,"pairing":"grok__grok-4.6","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":12}],"skill":"investor-reporting","what":"Investor-facing metrics from the same books — a consulting retainer base, a SaaS cost base, and a board-ready report share that must be scoped and revocable."},{"cases":[{"id":"invoicing-ad-hoc-obligation","of":7,"passed":7},{"id":"invoicing-already-billed-milestone","of":7,"passed":6},{"id":"invoicing-grant-funded","of":7,"passed":7},{"id":"invoicing-kinoko-paid","of":7,"passed":7},{"id":"invoicing-kinoko-paid-no-account","of":7,"passed":7}],"cells":[{"cache_read_tokens":2453519,"cache_write_tokens":277164,"cases":[{"id":"invoicing-ad-hoc-obligation","pass":true,"samples":1,"score":1},{"id":"invoicing-already-billed-milestone","pass":false,"samples":1,"score":0},{"id":"invoicing-grant-funded","pass":true,"samples":1,"score":1},{"id":"invoicing-kinoko-paid","pass":true,"samples":1,"score":1},{"id":"invoicing-kinoko-paid-no-account","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:34:51Z","cost_source":"harness","cost_usd":0.87418315,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":4413,"output_tokens":44251,"pairing":"claude__claude-haiku-4-5","passed":4,"samples":1,"score":0.8,"scored":5,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":13},{"cache_read_tokens":956067,"cache_write_tokens":98229,"cases":[{"id":"invoicing-ad-hoc-obligation","pass":true,"samples":1,"score":1},{"id":"invoicing-already-billed-milestone","pass":true,"samples":1,"score":1},{"id":"invoicing-grant-funded","pass":true,"samples":1,"score":1},{"id":"invoicing-kinoko-paid","pass":true,"samples":1,"score":1},{"id":"invoicing-kinoko-paid-no-account","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:29:43Z","cost_source":"harness","cost_usd":1.7425385,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":3562,"output_tokens":11137,"pairing":"claude__claude-opus-5__medium","passed":5,"samples":1,"score":1,"scored":5,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":44},{"cache_read_tokens":1445148,"cache_write_tokens":111338,"cases":[{"id":"invoicing-ad-hoc-obligation","pass":true,"samples":1,"score":1},{"id":"invoicing-already-billed-milestone","pass":true,"samples":1,"score":1},{"id":"invoicing-grant-funded","pass":true,"samples":1,"score":1},{"id":"invoicing-kinoko-paid","pass":true,"samples":1,"score":1},{"id":"invoicing-kinoko-paid-no-account","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:25:42Z","cost_source":"harness","cost_usd":1.2741444,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":3839,"output_tokens":11209,"pairing":"claude__claude-sonnet-5__medium","passed":5,"samples":1,"score":1,"scored":5,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":62},{"cache_read_tokens":2585088,"cases":[{"id":"invoicing-ad-hoc-obligation","pass":true,"samples":3,"score":1},{"id":"invoicing-already-billed-milestone","pass":true,"samples":3,"score":1},{"id":"invoicing-grant-funded","pass":true,"samples":1,"score":1},{"id":"invoicing-kinoko-paid","pass":true,"samples":1,"score":1},{"id":"invoicing-kinoko-paid-no-account","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T19:56:57Z","cost_source":"estimated","cost_usd":0.13385596,"git_sha":"fecbb549dc72083dd78c176dd6195189cfcbe247","input_tokens":330659,"output_tokens":13352,"pairing":"codex__gpt-5.6-luna","passed":5,"samples":1,"score":1,"scored":5,"stale":false,"std_err":0,"turns":9},{"cache_read_tokens":1331968,"cases":[{"id":"invoicing-ad-hoc-obligation","pass":true,"samples":1,"score":1},{"id":"invoicing-already-billed-milestone","pass":true,"samples":1,"score":1},{"id":"invoicing-grant-funded","pass":true,"samples":1,"score":1},{"id":"invoicing-kinoko-paid","pass":true,"samples":1,"score":1},{"id":"invoicing-kinoko-paid-no-account","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:18:22Z","cost_source":"estimated","cost_usd":1.709719,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":170467,"output_tokens":6380,"pairing":"codex__gpt-5.6-sol","passed":5,"samples":1,"score":1,"scored":5,"stale":false,"std_err":0,"turns":5},{"cache_read_tokens":1133824,"cases":[{"id":"invoicing-ad-hoc-obligation","pass":true,"samples":1,"score":1},{"id":"invoicing-already-billed-milestone","pass":true,"samples":1,"score":1},{"id":"invoicing-grant-funded","pass":true,"samples":1,"score":1},{"id":"invoicing-kinoko-paid","pass":true,"samples":1,"score":1},{"id":"invoicing-kinoko-paid-no-account","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:58:08Z","cost_source":"estimated","cost_usd":0.6332488000000001,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":169180,"output_tokens":5677,"pairing":"codex__gpt-5.6-terra","passed":5,"samples":1,"score":1,"scored":5,"stale":false,"std_err":0,"turns":5},{"cache_read_tokens":3081216,"cases":[{"id":"invoicing-ad-hoc-obligation","pass":true,"samples":1,"score":1},{"id":"invoicing-already-billed-milestone","pass":true,"samples":1,"score":1},{"id":"invoicing-grant-funded","pass":true,"samples":1,"score":1},{"id":"invoicing-kinoko-paid","pass":true,"samples":1,"score":1},{"id":"invoicing-kinoko-paid-no-account","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:36:28Z","cost_source":"harness","cost_usd":0.4445092,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":478504,"output_tokens":19524,"pairing":"grok__grok-4.6","passed":5,"samples":1,"score":1,"scored":5,"stale":false,"std_err":0,"turns":44}],"skill":"invoicing","what":"Bill a customer correctly: grant tranches, an ad-hoc charge that still needs an obligation, a milestone that was already billed and must not be billed again, and a payment recorded against the right funding account."},{"cases":[{"id":"rails-one-address-two-chains","of":7,"passed":7},{"id":"rails-quote-before-paying","of":7,"passed":7},{"id":"rails-wire-fee-account","of":7,"passed":6}],"cells":[{"cache_read_tokens":2057560,"cache_write_tokens":198970,"cases":[{"id":"rails-one-address-two-chains","pass":true,"samples":1,"score":1},{"id":"rails-quote-before-paying","pass":true,"samples":1,"score":1},{"id":"rails-wire-fee-account","pass":false,"samples":1,"score":0}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:52:41Z","cost_source":"harness","cost_usd":0.7162767499999999,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":3415,"output_tokens":44459,"pairing":"claude__claude-haiku-4-5","passed":2,"samples":1,"score":0.6666666666666666,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":35},{"cache_read_tokens":881884,"cache_write_tokens":74818,"cases":[{"id":"rails-one-address-two-chains","pass":true,"samples":1,"score":1},{"id":"rails-quote-before-paying","pass":true,"samples":1,"score":1},{"id":"rails-wire-fee-account","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:39:24Z","cost_source":"harness","cost_usd":1.421468,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":3254,"output_tokens":9012,"pairing":"claude__claude-opus-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":33},{"cache_read_tokens":637913,"cache_write_tokens":69898,"cases":[{"id":"rails-one-address-two-chains","pass":true,"samples":1,"score":1},{"id":"rails-quote-before-paying","pass":true,"samples":1,"score":1},{"id":"rails-wire-fee-account","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:34:56Z","cost_source":"harness","cost_usd":0.7593879,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":3431,"output_tokens":9529,"pairing":"claude__claude-sonnet-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":28},{"cache_read_tokens":890880,"cases":[{"id":"rails-one-address-two-chains","pass":true,"samples":1,"score":1},{"id":"rails-quote-before-paying","pass":true,"samples":1,"score":1},{"id":"rails-wire-fee-account","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:25:05Z","cost_source":"estimated","cost_usd":0.046776,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":106548,"output_tokens":6374,"pairing":"codex__gpt-5.6-luna","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":754944,"cases":[{"id":"rails-one-address-two-chains","pass":true,"samples":1,"score":1},{"id":"rails-quote-before-paying","pass":true,"samples":1,"score":1},{"id":"rails-wire-fee-account","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:26:03Z","cost_source":"estimated","cost_usd":1.120412,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":122452,"output_tokens":4356,"pairing":"codex__gpt-5.6-sol","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":1031936,"cases":[{"id":"rails-one-address-two-chains","pass":true,"samples":1,"score":1},{"id":"rails-quote-before-paying","pass":true,"samples":1,"score":1},{"id":"rails-wire-fee-account","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:06:12Z","cost_source":"estimated","cost_usd":0.5127852,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":116629,"output_tokens":6095,"pairing":"codex__gpt-5.6-terra","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":1773056,"cases":[{"id":"rails-one-address-two-chains","pass":true,"samples":1,"score":1},{"id":"rails-quote-before-paying","pass":true,"samples":1,"score":1},{"id":"rails-wire-fee-account","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:49:34Z","cost_source":"harness","cost_usd":0.26865202,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":301319,"output_tokens":15190,"pairing":"grok__grok-4.6","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":25}],"skill":"payment-rails","what":"Model how money moves — one address reachable on two chains, quoting routes before paying, and posting a wire fee as its own journal leg rather than netting it into the payment."},{"cases":[{"id":"pricing-agent-per-call","of":7,"passed":7},{"id":"pricing-bespoke-enterprise","of":7,"passed":7},{"id":"pricing-prepaid-packs","of":7,"passed":7}],"cells":[{"cache_read_tokens":3212228,"cache_write_tokens":275836,"cases":[{"id":"pricing-agent-per-call","pass":true,"samples":1,"score":1},{"id":"pricing-bespoke-enterprise","pass":true,"samples":1,"score":1},{"id":"pricing-prepaid-packs","pass":true,"samples":3,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T19:56:57Z","cost_source":"harness","cost_usd":1.05349805,"git_sha":"fecbb549dc72083dd78c176dd6195189cfcbe247","input_tokens":4631,"output_tokens":56479,"pairing":"claude__claude-haiku-4-5","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":43},{"cache_read_tokens":1139276,"cache_write_tokens":90192,"cases":[{"id":"pricing-agent-per-call","pass":true,"samples":1,"score":1},{"id":"pricing-bespoke-enterprise","pass":true,"samples":1,"score":1},{"id":"pricing-prepaid-packs","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:19:13Z","cost_source":"harness","cost_usd":1.774917,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":2077,"output_tokens":12086,"pairing":"claude__claude-opus-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":48},{"cache_read_tokens":1133618,"cache_write_tokens":79131,"cases":[{"id":"pricing-agent-per-call","pass":true,"samples":1,"score":1},{"id":"pricing-bespoke-enterprise","pass":true,"samples":1,"score":1},{"id":"pricing-prepaid-packs","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:16:41Z","cost_source":"harness","cost_usd":1.0028084000000002,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":2395,"output_tokens":12356,"pairing":"claude__claude-sonnet-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":52},{"cache_read_tokens":1132544,"cases":[{"id":"pricing-agent-per-call","pass":true,"samples":1,"score":1},{"id":"pricing-bespoke-enterprise","pass":true,"samples":1,"score":1},{"id":"pricing-prepaid-packs","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:09:18Z","cost_source":"estimated","cost_usd":0.054323880000000005,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":113089,"output_tokens":7546,"pairing":"codex__gpt-5.6-luna","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":970240,"cases":[{"id":"pricing-agent-per-call","pass":true,"samples":1,"score":1},{"id":"pricing-bespoke-enterprise","pass":true,"samples":1,"score":1},{"id":"pricing-prepaid-packs","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:10:30Z","cost_source":"estimated","cost_usd":1.261285,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":121111,"output_tokens":5687,"pairing":"codex__gpt-5.6-sol","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":256e4,"cases":[{"id":"pricing-agent-per-call","pass":true,"samples":1,"score":1},{"id":"pricing-bespoke-enterprise","pass":true,"samples":1,"score":1},{"id":"pricing-prepaid-packs","pass":true,"samples":3,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T19:56:57Z","cost_source":"estimated","cost_usd":1.222292,"git_sha":"fecbb549dc72083dd78c176dd6195189cfcbe247","input_tokens":247764,"output_tokens":17897,"pairing":"codex__gpt-5.6-terra","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":5},{"cache_read_tokens":2472704,"cases":[{"id":"pricing-agent-per-call","pass":true,"samples":1,"score":1},{"id":"pricing-bespoke-enterprise","pass":true,"samples":1,"score":1},{"id":"pricing-prepaid-packs","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:22:42Z","cost_source":"harness","cost_usd":0.32699296,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":293810,"output_tokens":16586,"pairing":"grok__grok-4.6","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":32}],"skill":"pricing","what":"Turn a business model into priced catalog items and obligations: a bespoke enterprise deal, per-call agent pricing, and prepaid packs that have to draw down rather than bill twice."},{"cases":[{"id":"setup-go-live-real-customer","of":7,"passed":7},{"id":"setup-headless-agent","of":7,"passed":7},{"id":"setup-sandbox-dry-run","of":7,"passed":7}],"cells":[{"cache_read_tokens":555997,"cache_write_tokens":74879,"cases":[{"id":"setup-go-live-real-customer","pass":true,"samples":1,"score":1},{"id":"setup-headless-agent","pass":true,"samples":1,"score":1},{"id":"setup-sandbox-dry-run","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T13:01:15Z","cost_source":"harness","cost_usd":0.23173244999999998,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":2105,"output_tokens":11170,"pairing":"claude__claude-haiku-4-5","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":10},{"cache_read_tokens":1223653,"cache_write_tokens":78589,"cases":[{"id":"setup-go-live-real-customer","pass":true,"samples":1,"score":1},{"id":"setup-headless-agent","pass":true,"samples":1,"score":1},{"id":"setup-sandbox-dry-run","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:10:20Z","cost_source":"harness","cost_usd":1.8490125000000002,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":1849,"output_tokens":18003,"pairing":"claude__claude-opus-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":45},{"cache_read_tokens":1160063,"cache_write_tokens":65390,"cases":[{"id":"setup-go-live-real-customer","pass":true,"samples":1,"score":1},{"id":"setup-headless-agent","pass":true,"samples":1,"score":1},{"id":"setup-sandbox-dry-run","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:09:04Z","cost_source":"harness","cost_usd":0.9574829,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":1857,"output_tokens":14369,"pairing":"claude__claude-sonnet-5__medium","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"tool_surface":"claude-builtin-2026-08-17","turns":45},{"cache_read_tokens":1782528,"cases":[{"id":"setup-go-live-real-customer","pass":true,"samples":1,"score":1},{"id":"setup-headless-agent","pass":true,"samples":1,"score":1},{"id":"setup-sandbox-dry-run","pass":true,"samples":3,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T19:56:57Z","cost_source":"estimated","cost_usd":0.09616875999999999,"git_sha":"fecbb549dc72083dd78c176dd6195189cfcbe247","input_tokens":228125,"output_tokens":12411,"pairing":"codex__gpt-5.6-luna","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":5},{"cache_read_tokens":1054464,"cases":[{"id":"setup-go-live-real-customer","pass":true,"samples":1,"score":1},{"id":"setup-headless-agent","pass":true,"samples":1,"score":1},{"id":"setup-sandbox-dry-run","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:01:55Z","cost_source":"estimated","cost_usd":1.289582,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":113332,"output_tokens":6523,"pairing":"codex__gpt-5.6-sol","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":994560,"cases":[{"id":"setup-go-live-real-customer","pass":true,"samples":1,"score":1},{"id":"setup-headless-agent","pass":true,"samples":1,"score":1},{"id":"setup-sandbox-dry-run","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:43:47Z","cost_source":"estimated","cost_usd":0.515406,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":115419,"output_tokens":7138,"pairing":"codex__gpt-5.6-terra","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":3},{"cache_read_tokens":2113408,"cases":[{"id":"setup-go-live-real-customer","pass":true,"samples":1,"score":1},{"id":"setup-headless-agent","pass":true,"samples":1,"score":1},{"id":"setup-sandbox-dry-run","pass":true,"samples":1,"score":1}],"cases_sha256":"383a5380ba3b7c19acc21cf2faadffcb4f81aa5e5e3cf31c24f3de36e14642db","completed_at":"2026-08-17T12:09:47Z","cost_source":"harness","cost_usd":0.3207628,"git_sha":"f613c5f8911f57f6ddbf0202a9b29099a3546ccd","input_tokens":361641,"output_tokens":17809,"pairing":"grok__grok-4.6","passed":3,"samples":1,"score":1,"scored":3,"stale":false,"std_err":0,"turns":30}],"skill":"setup-economico","what":"Connect the ledger and prove the connection: rehearse in a sandbox scenario without touching the real books, take a first real customer live, and register a headless agent without handing it write access it does not need."}],"title":"Agent benchmarks — which model and harness actually does the bookkeeping","totals":{"cells":77,"measured":77,"priced":77},"url":"https://economi.co/benchmarks/"}