{"updated":"2026-10-17","scope":"Synthetic development/regression repositories; local hot memory; medium effort; usage-based API price estimates, not invoices or hosted-service costs.","models":{"openai":{"model":"gpt-6.1-sol","input":2,"cacheRead":0.1,"cacheWrite":2.5,"output":10},"anthropic":{"model":"claude-opus-5-5","input":4,"cacheRead":0.2,"cacheWrite":5,"cacheWriteLong":8,"output":20}},"main":{"trials":384,"conditions":[{"provider":"anthropic","condition":"cold","trials":48,"successes":38,"cost_usd":3.80910587,"total_tokens":2749605},{"provider":"anthropic","condition":"handoff_md","trials":48,"successes":48,"cost_usd":5.49194228,"total_tokens":3406591},{"provider":"anthropic","condition":"native","trials":48,"successes":48,"cost_usd":3.314996880000001,"total_tokens":2494588},{"provider":"anthropic","condition":"onlybots","trials":48,"successes":48,"cost_usd":3.891651289999999,"total_tokens":2675191},{"provider":"openai","condition":"cold","trials":48,"successes":38,"cost_usd":4.234450630000001,"total_tokens":5577704},{"provider":"openai","condition":"handoff_md","trials":48,"successes":48,"cost_usd":4.808558309999999,"total_tokens":6011964},{"provider":"openai","condition":"native","trials":48,"successes":48,"cost_usd":2.6362595399999993,"total_tokens":4962301},{"provider":"openai","condition":"onlybots","trials":48,"successes":48,"cost_usd":4.16879562,"total_tokens":5108872}],"comparisons":[{"provider":"anthropic","baseline":"cold","cost_savings_fraction":-0.021670550207101957,"cost_savings_95":[-0.06062086872873862,0.010255714529408544],"token_savings_fraction":0.02706352366976339},{"provider":"anthropic","baseline":"handoff_md","cost_savings_fraction":0.2913888945679155,"cost_savings_95":[0.22004814453582366,0.35169820320844614],"token_savings_fraction":0.2147014419987605},{"provider":"anthropic","baseline":"native","cost_savings_fraction":-0.17395322857739814,"cost_savings_95":[-0.23310762248012407,-0.11305319607460951],"token_savings_fraction":-0.07239792703243975},{"provider":"openai","baseline":"cold","cost_savings_fraction":0.015504965280466965,"cost_savings_95":[-0.007158119577040267,0.03666675815456355],"token_savings_fraction":0.08405465761539166},{"provider":"openai","baseline":"handoff_md","cost_savings_fraction":0.13304667402483872,"cost_savings_95":[0.10959492055031006,0.15325212390565224],"token_savings_fraction":0.15021580302210724},{"provider":"openai","baseline":"native","cost_savings_fraction":-0.5813297426701771,"cost_savings_95":[-0.7617941314500769,-0.4200985728535209],"token_savings_fraction":-0.02953690233623485}]},"cross":{"trials":72,"conditions":[{"provider":"openai->anthropic","condition":"cold","trials":12,"successes":8,"cost_usd":1.06998394,"total_tokens":1041337},{"provider":"openai->anthropic","condition":"handoff_md","trials":12,"successes":12,"cost_usd":1.3402889599999996,"total_tokens":1250118},{"provider":"openai->anthropic","condition":"onlybots","trials":12,"successes":12,"cost_usd":1.0517503899999998,"total_tokens":1034465},{"provider":"anthropic->openai","condition":"cold","trials":12,"successes":8,"cost_usd":1.12471322,"total_tokens":1155509},{"provider":"anthropic->openai","condition":"handoff_md","trials":12,"successes":12,"cost_usd":1.45158371,"total_tokens":1127693},{"provider":"anthropic->openai","condition":"onlybots","trials":12,"successes":12,"cost_usd":1.1329556800000002,"total_tokens":971141}],"comparisons":[{"provider":"openai->anthropic","baseline":"cold","cost_savings_fraction":0.01704095670819128,"cost_savings_95":[-0.06287741261697621,0.09101776577840137],"token_savings_fraction":0.006599208517511501},{"provider":"openai->anthropic","baseline":"handoff_md","cost_savings_fraction":0.21528086749293218,"cost_savings_95":[0.169336576147057,0.2651528080870602],"token_savings_fraction":0.172506115422704},{"provider":"anthropic->openai","baseline":"cold","cost_savings_fraction":-0.00732849925957102,"cost_savings_95":[-0.03897892798784652,0.028691020222323393],"token_savings_fraction":0.15955565902126245},{"provider":"anthropic->openai","baseline":"handoff_md","cost_savings_fraction":0.21950372397055906,"cost_savings_95":[0.17268915837510368,0.2683429791612617],"token_savings_fraction":0.1388250170924179}]},"compaction":{"scope":"Two synthetic regression tasks, two manual native compactions; controlled input sizes, not full-window/default-threshold or adaptive-timing evidence. Agents may not recover facts by searching transcript files outside the workspace.","groups":[{"provider":"openai","condition":"native_compact","planned":2,"finished":2,"successes":2,"cost_usd":0.93432232,"input_tokens":1520290,"output_tokens":12531,"total_tokens":1532821,"cache_read_tokens":1246417},{"provider":"openai","condition":"onlybots_compact","planned":2,"finished":2,"successes":2,"cost_usd":0.8757174199999999,"input_tokens":1374587,"output_tokens":12133,"total_tokens":1386720,"cache_read_tokens":1117043},{"provider":"openai","condition":"handoff_md","planned":2,"finished":2,"successes":2,"cost_usd":1.0374210799999999,"input_tokens":1669384,"output_tokens":12535,"total_tokens":1681919,"cache_read_tokens":1358874},{"provider":"openai","condition":"onlybots","planned":2,"finished":2,"successes":2,"cost_usd":0.9766898199999999,"input_tokens":1455288,"output_tokens":7459,"total_tokens":1462747,"cache_read_tokens":1139677},{"provider":"anthropic","condition":"native_compact","planned":2,"finished":2,"successes":1,"cost_usd":2.94234985,"input_tokens":913555,"output_tokens":17639,"total_tokens":931194,"cache_read_tokens":369519},{"provider":"anthropic","condition":"onlybots_compact","planned":2,"finished":2,"successes":2,"cost_usd":3.2656174800000004,"input_tokens":1119263,"output_tokens":19405,"total_tokens":1138668,"cache_read_tokens":548352},{"provider":"anthropic","condition":"handoff_md","planned":2,"finished":2,"successes":2,"cost_usd":1.84299669,"input_tokens":1112877,"output_tokens":14949,"total_tokens":1127826,"cache_read_tokens":837558},{"provider":"anthropic","condition":"onlybots","planned":2,"finished":2,"successes":2,"cost_usd":1.6540348599999999,"input_tokens":647652,"output_tokens":11050,"total_tokens":658702,"cache_read_tokens":376074}],"rows":[{"run_id":"pilot5-compaction-retry-after-openai-onlybots","task":"compaction-retry-after","provider":"openai","condition":"onlybots","success":true,"cost_usd":0.50121414,"total_tokens":771987,"compactions":[],"compaction_hooks":{"before":0,"after":0}},{"run_id":"pilot6-compaction-retry-after-openai-onlybots_compact","task":"compaction-retry-after","provider":"openai","condition":"onlybots_compact","success":true,"cost_usd":0.4199245499999999,"total_tokens":614250,"compactions":[{"after_episode":"pressure-a","started_at":"2026-10-17T04:12:54.294Z","ended_at":"2026-10-17T04:13:18.280Z","exit_code":0,"confirmed":true,"pre_tokens":58759,"post_tokens":8781,"cost_usd":0.01920991},{"after_episode":"pressure-b","started_at":"2026-10-17T04:13:47.039Z","ended_at":"2026-10-17T04:14:12.252Z","exit_code":0,"confirmed":true,"pre_tokens":57883,"post_tokens":9424,"cost_usd":0.0210612}],"compaction_hooks":{"before":2,"after":2}},{"run_id":"pilot5-compaction-retry-after-openai-handoff_md","task":"compaction-retry-after","provider":"openai","condition":"handoff_md","success":true,"cost_usd":0.49856945999999996,"total_tokens":789627,"compactions":[],"compaction_hooks":{"before":0,"after":0}},{"run_id":"pilot5-compaction-retry-after-openai-native_compact","task":"compaction-retry-after","provider":"openai","condition":"native_compact","success":true,"cost_usd":0.47576645,"total_tokens":748150,"compactions":[{"after_episode":"pressure-a","started_at":"2026-10-17T04:03:12.838Z","ended_at":"2026-10-17T04:03:38.597Z","exit_code":0,"confirmed":true,"pre_tokens":74004,"post_tokens":9029,"cost_usd":0.0218956},{"after_episode":"pressure-b","started_at":"2026-10-17T04:04:07.875Z","ended_at":"2026-10-17T04:04:36.110Z","exit_code":0,"confirmed":true,"pre_tokens":58254,"post_tokens":9644,"cost_usd":0.0224768}],"compaction_hooks":{"before":0,"after":0}},{"run_id":"pilot5-compaction-retry-after-anthropic-onlybots_compact","task":"compaction-retry-after","provider":"anthropic","condition":"onlybots_compact","success":true,"cost_usd":1.6576586400000002,"total_tokens":529329,"compactions":[{"after_episode":"pressure-a","started_at":"2026-10-17T03:59:57.826Z","ended_at":"2026-10-17T04:00:21.179Z","exit_code":0,"confirmed":true,"pre_tokens":72331,"post_tokens":2263,"cost_usd":0.431833},{"after_episode":"pressure-b","started_at":"2026-10-17T04:00:44.025Z","ended_at":"2026-10-17T04:01:03.657Z","exit_code":0,"confirmed":true,"pre_tokens":70562,"post_tokens":2271,"cost_usd":0.3271188}],"compaction_hooks":{"before":2,"after":2}},{"run_id":"pilot5-compaction-retry-after-anthropic-onlybots","task":"compaction-retry-after","provider":"anthropic","condition":"onlybots","success":true,"cost_usd":0.83614641,"total_tokens":272876,"compactions":[],"compaction_hooks":{"before":0,"after":0}},{"run_id":"pilot5-compaction-retry-after-anthropic-handoff_md","task":"compaction-retry-after","provider":"anthropic","condition":"handoff_md","success":true,"cost_usd":0.9160426500000001,"total_tokens":621657,"compactions":[],"compaction_hooks":{"before":0,"after":0}},{"run_id":"pilot5-compaction-retry-after-anthropic-native_compact","task":"compaction-retry-after","provider":"anthropic","condition":"native_compact","success":false,"cost_usd":1.4400164199999999,"total_tokens":436345,"compactions":[{"after_episode":"pressure-a","started_at":"2026-10-17T04:02:23.440Z","ended_at":"2026-10-17T04:02:47.543Z","exit_code":0,"confirmed":true,"pre_tokens":71281,"post_tokens":2328,"cost_usd":0.3197328},{"after_episode":"pressure-b","started_at":"2026-10-17T04:03:05.661Z","ended_at":"2026-10-17T04:03:25.840Z","exit_code":0,"confirmed":true,"pre_tokens":69813,"post_tokens":2259,"cost_usd":0.3203408}],"compaction_hooks":{"before":0,"after":0}},{"run_id":"pilot5-compaction-config-precedence-openai-onlybots","task":"compaction-config-precedence","provider":"openai","condition":"onlybots","success":true,"cost_usd":0.47547567999999996,"total_tokens":690760,"compactions":[],"compaction_hooks":{"before":0,"after":0}},{"run_id":"pilot6-compaction-config-precedence-openai-onlybots_compact","task":"compaction-config-precedence","provider":"openai","condition":"onlybots_compact","success":true,"cost_usd":0.45579286999999996,"total_tokens":772470,"compactions":[{"after_episode":"pressure-a","started_at":"2026-10-17T04:12:59.510Z","ended_at":"2026-10-17T04:13:21.192Z","exit_code":0,"confirmed":true,"pre_tokens":68983,"post_tokens":8834,"cost_usd":0.0203635},{"after_episode":"pressure-b","started_at":"2026-10-17T04:14:10.290Z","ended_at":"2026-10-17T04:14:40.662Z","exit_code":0,"confirmed":true,"pre_tokens":59186,"post_tokens":10351,"cost_usd":0.0243191}],"compaction_hooks":{"before":2,"after":2}},{"run_id":"pilot5-compaction-config-precedence-openai-handoff_md","task":"compaction-config-precedence","provider":"openai","condition":"handoff_md","success":true,"cost_usd":0.53885162,"total_tokens":892292,"compactions":[],"compaction_hooks":{"before":0,"after":0}},{"run_id":"pilot5-compaction-config-precedence-openai-native_compact","task":"compaction-config-precedence","provider":"openai","condition":"native_compact","success":true,"cost_usd":0.45855587000000003,"total_tokens":784671,"compactions":[{"after_episode":"pressure-a","started_at":"2026-10-17T04:09:45.369Z","ended_at":"2026-10-17T04:10:10.286Z","exit_code":0,"confirmed":true,"pre_tokens":72793,"post_tokens":8981,"cost_usd":0.0216483},{"after_episode":"pressure-b","started_at":"2026-10-17T04:10:56.746Z","ended_at":"2026-10-17T04:11:23.498Z","exit_code":0,"confirmed":true,"pre_tokens":58574,"post_tokens":10183,"cost_usd":0.0236831}],"compaction_hooks":{"before":0,"after":0}},{"run_id":"pilot5-compaction-config-precedence-anthropic-handoff_md","task":"compaction-config-precedence","provider":"anthropic","condition":"handoff_md","success":true,"cost_usd":0.92695404,"total_tokens":506169,"compactions":[],"compaction_hooks":{"before":0,"after":0}},{"run_id":"pilot5-compaction-config-precedence-anthropic-onlybots","task":"compaction-config-precedence","provider":"anthropic","condition":"onlybots","success":true,"cost_usd":0.81788845,"total_tokens":385826,"compactions":[],"compaction_hooks":{"before":0,"after":0}},{"run_id":"pilot5-compaction-config-precedence-anthropic-onlybots_compact","task":"compaction-config-precedence","provider":"anthropic","condition":"onlybots_compact","success":true,"cost_usd":1.6079588400000002,"total_tokens":609339,"compactions":[{"after_episode":"pressure-a","started_at":"2026-10-17T04:05:18.428Z","ended_at":"2026-10-17T04:05:40.474Z","exit_code":0,"confirmed":true,"pre_tokens":70619,"post_tokens":2369,"cost_usd":0.3742178},{"after_episode":"pressure-b","started_at":"2026-10-17T04:06:12.421Z","ended_at":"2026-10-17T04:06:32.768Z","exit_code":0,"confirmed":true,"pre_tokens":72951,"post_tokens":2640,"cost_usd":0.3857478}],"compaction_hooks":{"before":2,"after":2}},{"run_id":"pilot5-compaction-config-precedence-anthropic-native_compact","task":"compaction-config-precedence","provider":"anthropic","condition":"native_compact","success":true,"cost_usd":1.50233343,"total_tokens":494849,"compactions":[{"after_episode":"pressure-a","started_at":"2026-10-17T04:06:01.242Z","ended_at":"2026-10-17T04:06:23.509Z","exit_code":0,"confirmed":true,"pre_tokens":75936,"post_tokens":2428,"cost_usd":0.3445988},{"after_episode":"pressure-b","started_at":"2026-10-17T04:06:49.302Z","ended_at":"2026-10-17T04:07:12.084Z","exit_code":0,"confirmed":true,"pre_tokens":70703,"post_tokens":2492,"cost_usd":0.3381288}],"compaction_hooks":{"before":0,"after":0}}],"recovery_diagnostic":{"run_id":"pilot7-compaction-retry-after-anthropic-native_compact","selection":"One diagnostic selected after a restricted-access failure; excluded from comparative success rates.","transcript_recovery":"same_session_path_only","status":"completed","success":true,"valid":true,"successful_transcript_tool_calls":2,"compactions":[{"after_episode":"pressure-a","started_at":"2026-10-17T04:15:48.442Z","ended_at":"2026-10-17T04:16:08.865Z","exit_code":0,"confirmed":true,"pre_tokens":69087,"post_tokens":2269},{"after_episode":"pressure-b","started_at":"2026-10-17T04:16:24.786Z","ended_at":"2026-10-17T04:16:47.095Z","exit_code":0,"confirmed":true,"pre_tokens":69528,"post_tokens":2335}],"cost_usd":1.5657274099999998},"spending":{"openai":{"cap":9,"settled":5.762352589999997,"accounted":5.762352589999997,"uncertain":0,"pending":0},"anthropic":{"cap":16,"settled":14.375115359999997,"accounted":14.375115359999997,"uncertain":0,"pending":0}}},"sources":[{"file":"benchmark/continuity/results/frontier-fixes-2026-10/quota-recovery/summary.json","sha256":"e9530a83b5bf36e03d63c821afb9b9f969831623a7821904a6543fad3bcbc3d0"},{"file":"benchmark/continuity/results/frontier-fixes-2026-10/summary.json","sha256":"e0cb0c228e865876596a19966b3343185d0a74021df6fd67478112fd51a9b1de"},{"file":"benchmark/continuity/results/compaction-2026-10/summary.json","sha256":"3ebcc7b477b804c5fe4288ba47e4292e1c08c2321304251d3f4c322ef5ddb985"}]}