{ "schema": "agent-rerun-receipt.v1", "record": { "id": "20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": "byok/openai/gpt-4.1-mini", "wallSeconds": 11.536, "inputTokens": 915, "outputTokens": 478, "totalTokens": 1393, "costUsd": 0.001131, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Diagnostic only: request-local initial tool narrowing skipped the caller-authorized behavioral review. Generic harness regression repaired in 1.4.564; independent scorer outcomes are retained but these timings do not establish a successful latency repair." ], "evidenceFile": "20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40.json", "archived": true }, "sourceRecord": { "id": "20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40", "originalCampaign": "agent_comparison_20261015", "rerun": true, "correctionOf": "20261016T123350Z_verdict-code_byok_openai_gpt-4.1-mini_9b3037", "cohort": "original-model", "family": "same-model-harness-rerun", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": "byok/openai/gpt-4.1-mini", "status": "FAIL", "wallSeconds": 11.536, "inputTokens": 915, "outputTokens": 478, "totalTokens": 1393, "costUsd": 0.001131, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedCodePass": false, "exitCode": 0, "evidencePaths": [ "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40" ], "arm": "E", "admission": "NOT_VERIFIED", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Diagnostic only: request-local initial tool narrowing skipped the caller-authorized behavioral review. Generic harness regression repaired in 1.4.564; independent scorer outcomes are retained but these timings do not establish a successful latency repair." ], "freeze": { "task_sha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "oracle_sha256": "3675736119622cac168d711045426c6678bd08212b9bedf1aab71126a7ef1a76", "starter_sha256": "595a2a41e7bfd09ad28045cb3be95f0126d702eb25e6f53c723c0b77c46b44e2", "frozen_before_execution_at": 1792154239.814358 }, "mutableRuntimeFiles": [ "budget_ledger.json", "58bad20ef7ea498d83d95ad71d9a4356.timing.json", "412ef4f4bc024aa1b572d9d47c453a99.request.json", "412ef4f4bc024aa1b572d9d47c453a99.response.txt", "3904f4853fb347d28de8c1135a93f086.http.json", "09d41c4b0fc84825ac81794419ad8d88.timing.json", "58bad20ef7ea498d83d95ad71d9a4356.request.json", "2bd16bc06a4a405aaa060784f6df0da1.timing.json", "ba082b63a966418f96578d6a878b4e6d.request.json", "ba082b63a966418f96578d6a878b4e6d.timing.json", "58bad20ef7ea498d83d95ad71d9a4356.response.txt", "ba082b63a966418f96578d6a878b4e6d.response.txt", "412ef4f4bc024aa1b572d9d47c453a99.timing.json", "2bd16bc06a4a405aaa060784f6df0da1.http.json", "3904f4853fb347d28de8c1135a93f086.timing.json", "3904f4853fb347d28de8c1135a93f086.request.json", "3904f4853fb347d28de8c1135a93f086.response.txt", "ba082b63a966418f96578d6a878b4e6d.http.json", "2bd16bc06a4a405aaa060784f6df0da1.response.txt", "412ef4f4bc024aa1b572d9d47c453a99.http.json", "2bd16bc06a4a405aaa060784f6df0da1.request.json", "58bad20ef7ea498d83d95ad71d9a4356.http.json", "09d41c4b0fc84825ac81794419ad8d88.http.json", "09d41c4b0fc84825ac81794419ad8d88.response.txt", "09d41c4b0fc84825ac81794419ad8d88.request.json" ], "terminalBudgetSnapshot": "budget_ledger_at_terminal.json", "terminalTransportSnapshot": "terminal_evidence/", "rawStatus": "FAIL", "metadataTransportDiagnosticCount": 0, "diagnosticOnly": true, "tokenSource": "provider", "accountingComplete": true, "identityVerified": true, "providerCallCount": 1, "partialInputTokens": 915, "partialOutputTokens": 478, "partialTotalTokens": 1393, "partialEstimatedCostUsd": 0.001131, "reportedAdditiveReasoningTokens": null, "reportedReasoningTokens": null, "allCallsReportReasoning": false, "tokensPerSecond": 120.752, "knownTokensIncludingAdditiveReasoning": 1393, "knownTokensIncludingAdditiveReasoningPerSecond": 120.752, "generationCensored": false, "budgetControls": { "schema": "agent_comparison_rerun_policy.v1", "temperature": 0, "timeout_seconds": 240, "per_request_output_tokens": 8192, "proxy_shared_max_input_tokens": 160000, "proxy_shared_max_output_tokens": 32000, "proxy_reservation": "entire request UTF8 bytes +32/message+128; providerusage settled after eachcall; failclosed ifmissing", "maximum_input_tokens": 160000, "total_token_budget": 400000, "max_turns": 24, "proxy_read_timeout_seconds": 270, "verdict_stream_idle_timeout_seconds": 240, "aider_request_timeout_seconds": 240, "reasoning": { "mode": "default" }, "models": "exact pins, no failover", "task_file": "solution.py", "hidden_oracle_fed_to_agent": false, "producer_location": "outside candidate workspace; candidate read denied", "aider_context": "live exact catalog context metadata; history ceiling=min(inputbudget,context/2)", "appendix_model_changed": false }, "reasoningEffort": "default", "sharedProxyCapsEnforced": true, "actualBelowCommonCaps": true, "timeoutSeconds": 240, "terminalError": null, "terminalCompleted": true, "agentFailureReason": null, "sameGatewayInstance": true, "comparisonEligible": false, "publicEvidencePath": "artifacts/agent_comparison_completion_repair_20261016/matrix/public_evidence/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40.json", "publicEvidenceSha256": "3d64aef2635a83729959cedab5c47d94045d8b3740d136e0d125481bf9a7c70c", "apparatusInvalidation": "initial_tool_narrowing_skipped_behavioral_review" }, "sourceFile": "artifacts/agent_comparison_completion_repair_20261016/matrix/reviewed-summary.json", "sourceFileSha256": "cfdb4bb21c1038588f0951f2cbe4774e077c01691d7e0e53434881d3b9a16419", "files": [ { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/09d41c4b0fc84825ac81794419ad8d88.http.json", "sha256": "7b7d2ba7da38df349730b844f5a7ee404601546b1255c0860ee52f313dde4f73", "bytes": 138 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/09d41c4b0fc84825ac81794419ad8d88.request.json", "sha256": "4a1bfd643069ca290c79d1510f86f36e4cbb06adcb11c761a6e6c8869178402a", "bytes": 10819 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/09d41c4b0fc84825ac81794419ad8d88.response.txt", "sha256": "c47e8ef04543cd904446c64e2b1a60e062733ad48e671649fdf2120b7c52e4bb", "bytes": 13047 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/09d41c4b0fc84825ac81794419ad8d88.timing.json", "sha256": "92203001b0f2d73f1c3eccbbb22c70774659abc8d786beda7bcab76e5afd199b", "bytes": 307 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/2bd16bc06a4a405aaa060784f6df0da1.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/2bd16bc06a4a405aaa060784f6df0da1.request.json", "sha256": "1ce9fd30eb6307a789013531fafa6783691de6024645b61fa7535304f3e49dd4", "bytes": 58 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/2bd16bc06a4a405aaa060784f6df0da1.response.txt", "sha256": "95a7678e96278f7641daaf7f8e36d9b35742fe3aabbce5a046b1f79124e3da5b", "bytes": 992 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/2bd16bc06a4a405aaa060784f6df0da1.timing.json", "sha256": "26943b5e97506d4fcd6ac2a89ce5257316a151a132153c54b02d9f7083a8eb2b", "bytes": 307 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/3904f4853fb347d28de8c1135a93f086.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/3904f4853fb347d28de8c1135a93f086.request.json", "sha256": "c25f6aa1a57fedf1a8b70de26031235b08750a9132570d894678200161e77c12", "bytes": 61 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/3904f4853fb347d28de8c1135a93f086.response.txt", "sha256": "0398162f3d22f78fe804778d9e6f3bf1c189970aeef88838d1085d4b7197f55b", "bytes": 59657 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/3904f4853fb347d28de8c1135a93f086.timing.json", "sha256": "26c0e1942c8704bae97a63f262f4e99217ed43534750a663580d9aa196cc1c4e", "bytes": 306 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/412ef4f4bc024aa1b572d9d47c453a99.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/412ef4f4bc024aa1b572d9d47c453a99.request.json", "sha256": "1ce9fd30eb6307a789013531fafa6783691de6024645b61fa7535304f3e49dd4", "bytes": 58 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/412ef4f4bc024aa1b572d9d47c453a99.response.txt", "sha256": "d5f0928538dd5176a4b8dc1555f217970eaccb5665dd58a29412bebec130806e", "bytes": 993 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/412ef4f4bc024aa1b572d9d47c453a99.timing.json", "sha256": "c86dabc0a1cac90aa2d303f3d657008f7e65418e7a0588c6e45c74bffe44d27d", "bytes": 304 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/58bad20ef7ea498d83d95ad71d9a4356.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/58bad20ef7ea498d83d95ad71d9a4356.request.json", "sha256": "00d4db88f698768b96dbb446b10cb99c60359031af92a6ca1bd72189b88e34c4", "bytes": 75 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/58bad20ef7ea498d83d95ad71d9a4356.response.txt", "sha256": "ac8e2971e37be1dbd285e413df30ce34a72d1fb1a6c5669586ce252145f3a18b", "bytes": 1075602 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/58bad20ef7ea498d83d95ad71d9a4356.timing.json", "sha256": "2a55ca11c0739156114701f8225b855b74025b6fbd7cd64b74a0a5f80533396a", "bytes": 305 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/ba082b63a966418f96578d6a878b4e6d.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/ba082b63a966418f96578d6a878b4e6d.request.json", "sha256": "63dc7d3bf0ea328c7500cf3bae2e7fd27c9e892aefb048095452176b8b824609", "bytes": 101 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/ba082b63a966418f96578d6a878b4e6d.response.txt", "sha256": "ed1b023f479e19556fa6e1706214774e58507e5b5ba80bed839d6ef7ac83a272", "bytes": 153 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/ba082b63a966418f96578d6a878b4e6d.timing.json", "sha256": "abb06b099fbb652bc4b962b1e76b653c4fc7902a2283792d1530d4ba8672205e", "bytes": 305 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/budget_ledger.json", "sha256": "b766571c595524addaa12a5947db9e62f56042314a6f8fabf1ded77e1edb1485", "bytes": 116 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/budget_ledger_at_terminal.json", "sha256": "b766571c595524addaa12a5947db9e62f56042314a6f8fabf1ded77e1edb1485", "bytes": 116 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/freeze.json", "sha256": "5a4646acc410e09135dadbd612c97594199bb05cd2adee51b220b49f8eca1bf0", "bytes": 313, "content": "{\n \"task_sha256\": \"6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe\",\n \"oracle_sha256\": \"3675736119622cac168d711045426c6678bd08212b9bedf1aab71126a7ef1a76\",\n \"starter_sha256\": \"595a2a41e7bfd09ad28045cb3be95f0126d702eb25e6f53c723c0b77c46b44e2\",\n \"frozen_before_execution_at\": 1792154239.814358\n}", "contentRedacted": false, "publishedContentSha256": "5a4646acc410e09135dadbd612c97594199bb05cd2adee51b220b49f8eca1bf0" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/health_after.json", "sha256": "dafb792d17be7fe7ae6a068efd849e920de0c4e637b71bf1b30f538c31c6dee4", "bytes": 1327 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/health_before.json", "sha256": "ee38b10bae773a638f9713fce7ca9d3eca5bba0f59906d2d6428e4024639cef5", "bytes": 1327 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/invocation.json", "sha256": "ee0520da830e943c1225ca021a97b745b8e32e2cab4303a7bfaf7715361fa7b6", "bytes": 2182 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/launch.json", "sha256": "7d29433d77a5fa111d2b286e6d033b962656f43df102ee7247bff74e454d91f9", "bytes": 236 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/oracle.stderr.txt", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes": 0, "content": "", "contentRedacted": false, "publishedContentSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/oracle.stdout.txt", "sha256": "3e7b0982f217a41f229865a072cdee2a778735cbabe56238a3f0d4e91cab0746", "bytes": 1580, "content": "{\"passed\": 18, \"total\": 20, \"cases\": [{\"input\": \"0\", \"expected\": 0, \"actual\": 0, \"pass\": true}, {\"input\": \" 120\\t\", \"expected\": 120, \"actual\": 120, \"pass\": true}, {\"input\": \"0007\", \"expected\": 7, \"actual\": 7, \"pass\": true}, {\"input\": \"-1\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"+1\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"1.5\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"\\uff11\\uff12\", \"expected\": null, \"actual\": 12, \"pass\": false}, {\"input\": \"\\u0661\\u0662\", \"expected\": null, \"actual\": 12, \"pass\": false}, {\"input\": \"\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": null, \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": 4, \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 GMT\", \"expected\": 1, \"actual\": 1, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:01 GMT\", \"expected\": 2, \"actual\": 2, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:27:00 GMT\", \"expected\": 0, \"actual\": 0, \"pass\": true}, {\"input\": \"Tue, 21 Oct 2015 07:28:00 GMT\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 UTC\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"wed, 21 Oct 2015 07:28:00 GMT\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 31 Feb 2015 07:28:00 GMT\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 GMT trailing\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 +0000\", \"expected\": null, \"actual\": null, \"pass\": true}]}\n", "contentRedacted": false, "publishedContentSha256": "3e7b0982f217a41f229865a072cdee2a778735cbabe56238a3f0d4e91cab0746" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/policy.json", "sha256": "e1d7a7f7f022f69dcb3e7fc79a820f84a8425324b888b94cfbc0099cb6f4bec7", "bytes": 921, "content": "{\n \"schema\": \"agent_comparison_rerun_policy.v1\",\n \"temperature\": 0,\n \"timeout_seconds\": 240,\n \"per_request_output_tokens\": 8192,\n \"proxy_shared_max_input_tokens\": 160000,\n \"proxy_shared_max_output_tokens\": 32000,\n \"proxy_reservation\": \"entire request UTF8 bytes +32/message+128; providerusage settled after eachcall; failclosed ifmissing\",\n \"maximum_input_tokens\": 160000,\n \"total_token_budget\": 400000,\n \"max_turns\": 24,\n \"proxy_read_timeout_seconds\": 270,\n \"verdict_stream_idle_timeout_seconds\": 240,\n \"aider_request_timeout_seconds\": 240,\n \"reasoning\": {\n \"mode\": \"default\"\n },\n \"models\": \"exact pins, no failover\",\n \"task_file\": \"solution.py\",\n \"hidden_oracle_fed_to_agent\": false,\n \"producer_location\": \"outside candidate workspace; candidate read denied\",\n \"aider_context\": \"live exact catalog context metadata; history ceiling=min(inputbudget,context/2)\",\n \"appendix_model_changed\": false\n}", "contentRedacted": false, "publishedContentSha256": "e1d7a7f7f022f69dcb3e7fc79a820f84a8425324b888b94cfbc0099cb6f4bec7" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/post-terminal-settlement.json", "sha256": "643fd167b0900ea34d63f94874be56342009f996530e64aaddd19f3e3239fae6", "bytes": 381 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/private_producer/gateway-0001.sse", "sha256": "c47e8ef04543cd904446c64e2b1a60e062733ad48e671649fdf2120b7c52e4bb", "bytes": 13047 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/private_producer/runtime/.verdict_code_usage.inv-830178e0d7c146848645d43b2ebf503d.json", "sha256": "cced61791a19214eb2910832fcc1bb35a53737fb4186d8ba5e8563c74ab86890", "bytes": 18468 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/private_producer/runtime/artifacts/runs/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/comms.txt", "sha256": "dab053bd232e9e2c7a5ebabe673f46bbbd58c4f09e9b911b9834a33ea640972d", "bytes": 1227 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/private_producer/runtime/artifacts/runs/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40-t1/comms.txt", "sha256": "c7ecbd2a6afd44070267188890b51e94fdc32280112e6bf4d93d754ea5718108", "bytes": 645 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/private_producer/runtime/compaction_events.jsonl", "sha256": "7acc0441312353ab24cdf872490dbc798793c77bd3cd0348b6b11844db7e5494", "bytes": 473 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/private_producer/runtime/model_cache/availability_cache.json", "sha256": "b90366fc89fd069c335048905b6259edd0f5bcbac6f4b1b3f752276e780e3e43", "bytes": 275 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/private_producer/state.json", "sha256": "8870ca6011c2280870864f2703a73563784745aee34e24eed655f59236ce9c4f", "bytes": 2292 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/product_source.json", "sha256": "886ed65f0e33e1ee8f21ada21c10572aa8251406bf136ae7e79faa5af069a741", "bytes": 846, "content": "{\n \"head\": \"e4e758492ee2749b88db2bf027fa134d93c66428\",\n \"files\": {\n \"scripts/benchmarks/agent_comparison_rerun_20261016.py\": \"c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0\",\n \"tools/verdict-code/verdict_code/agent.py\": \"7c0f6171d9cca143ca04f629570aceeb7d3c728e6c5cd3a30434b886c4007e0e\",\n \"tools/verdict-code/verdict_code/tools/__init__.py\": \"609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57\",\n \"tools/verdict-code/verdict_code/main.py\": \"af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab\",\n \"tools/verdict-code/VERSION\": \"c71dc09ace4b582eadda89d7dfda246c80fa899802686708c2cf11cc33590e06\",\n \"scripts/benchmarks/agent_comparison_completion_repair_20261016.py\": \"827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa\"\n },\n \"retainedSourceBytes\": \"source_snapshot/\"\n}", "contentRedacted": false, "publishedContentSha256": "886ed65f0e33e1ee8f21ada21c10572aa8251406bf136ae7e79faa5af069a741" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/prompt.txt", "sha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "bytes": 837, "content": "Synthetic benchmark task: Retry-After parsing. Implement `retry_after(value, now)` in solution.py using only Python standard library. now is an aware UTC datetime. Return a nonnegative integer number of seconds or None for invalid input. Strings containing only ASCII decimal digits after surrounding whitespace return that integer. Otherwise accept an RFC HTTP-date of form `Wed, 21 Oct 2015 07:28:00 GMT`, case sensitive, validated weekday must agree with date. For a valid HTTP date return max(0, ceil((date-now).total_seconds())). Reject non-string inputs, signs, fractions, non-ASCII digits, extra text, and non-GMT timezone names. Do not change any other source file. Hidden tests are unavailable. Write the implementation to disk and finish. Do not install dependencies, access network, use other models, or edit unrelated paths.\n", "contentRedacted": false, "publishedContentSha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/result.json", "sha256": "98332b609998b84e549ddc03e41eff93a65f3a191aca16a2cf0808a547d826c0", "bytes": 9778 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/sandbox.sb", "sha256": "ea2c0a2b2e4c5476f56645f2c73510a2522b7ab92b8ef48bd426895990543bba", "bytes": 5843 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/sandbox_preflight.json", "sha256": "8415d156075475eba2b82a04636f833e85484020de5e6424f6140cd0ac61390f", "bytes": 93 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/source_snapshot/scripts/benchmarks/agent_comparison_completion_repair_20261016.py", "sha256": "827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa", "bytes": 13274 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/source_snapshot/scripts/benchmarks/agent_comparison_rerun_20261016.py", "sha256": "c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0", "bytes": 38332 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/source_snapshot/tools/verdict-code/VERSION", "sha256": "c71dc09ace4b582eadda89d7dfda246c80fa899802686708c2cf11cc33590e06", "bytes": 8 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/source_snapshot/tools/verdict-code/verdict_code/agent.py", "sha256": "7c0f6171d9cca143ca04f629570aceeb7d3c728e6c5cd3a30434b886c4007e0e", "bytes": 763436 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/source_snapshot/tools/verdict-code/verdict_code/main.py", "sha256": "af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab", "bytes": 625279 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/source_snapshot/tools/verdict-code/verdict_code/tools/__init__.py", "sha256": "609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57", "bytes": 269877 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/stderr.txt", "sha256": "12918d24b35f57fe9ae75bfe6f5ace516ed564f8484a1a307d2186a36df7291a", "bytes": 6039 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/stdout.txt", "sha256": "bcc833bf9a793c329128951e9e93cb72f6496be6a2a624555771b1294a2f0244", "bytes": 6610 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/09d41c4b0fc84825ac81794419ad8d88.http.json", "sha256": "7b7d2ba7da38df349730b844f5a7ee404601546b1255c0860ee52f313dde4f73", "bytes": 138 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/09d41c4b0fc84825ac81794419ad8d88.request.json", "sha256": "4a1bfd643069ca290c79d1510f86f36e4cbb06adcb11c761a6e6c8869178402a", "bytes": 10819 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/09d41c4b0fc84825ac81794419ad8d88.response.txt", "sha256": "c47e8ef04543cd904446c64e2b1a60e062733ad48e671649fdf2120b7c52e4bb", "bytes": 13047 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/09d41c4b0fc84825ac81794419ad8d88.timing.json", "sha256": "92203001b0f2d73f1c3eccbbb22c70774659abc8d786beda7bcab76e5afd199b", "bytes": 307 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/2bd16bc06a4a405aaa060784f6df0da1.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/2bd16bc06a4a405aaa060784f6df0da1.request.json", "sha256": "1ce9fd30eb6307a789013531fafa6783691de6024645b61fa7535304f3e49dd4", "bytes": 58 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/2bd16bc06a4a405aaa060784f6df0da1.response.txt", "sha256": "95a7678e96278f7641daaf7f8e36d9b35742fe3aabbce5a046b1f79124e3da5b", "bytes": 992 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/2bd16bc06a4a405aaa060784f6df0da1.timing.json", "sha256": "26943b5e97506d4fcd6ac2a89ce5257316a151a132153c54b02d9f7083a8eb2b", "bytes": 307 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/3904f4853fb347d28de8c1135a93f086.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/3904f4853fb347d28de8c1135a93f086.request.json", "sha256": "c25f6aa1a57fedf1a8b70de26031235b08750a9132570d894678200161e77c12", "bytes": 61 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/3904f4853fb347d28de8c1135a93f086.response.txt", "sha256": "0398162f3d22f78fe804778d9e6f3bf1c189970aeef88838d1085d4b7197f55b", "bytes": 59657 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/3904f4853fb347d28de8c1135a93f086.timing.json", "sha256": "26c0e1942c8704bae97a63f262f4e99217ed43534750a663580d9aa196cc1c4e", "bytes": 306 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/412ef4f4bc024aa1b572d9d47c453a99.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/412ef4f4bc024aa1b572d9d47c453a99.request.json", "sha256": "1ce9fd30eb6307a789013531fafa6783691de6024645b61fa7535304f3e49dd4", "bytes": 58 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/412ef4f4bc024aa1b572d9d47c453a99.response.txt", "sha256": "d5f0928538dd5176a4b8dc1555f217970eaccb5665dd58a29412bebec130806e", "bytes": 993 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/412ef4f4bc024aa1b572d9d47c453a99.timing.json", "sha256": "c86dabc0a1cac90aa2d303f3d657008f7e65418e7a0588c6e45c74bffe44d27d", "bytes": 304 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/58bad20ef7ea498d83d95ad71d9a4356.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/58bad20ef7ea498d83d95ad71d9a4356.request.json", "sha256": "00d4db88f698768b96dbb446b10cb99c60359031af92a6ca1bd72189b88e34c4", "bytes": 75 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/58bad20ef7ea498d83d95ad71d9a4356.response.txt", "sha256": "ac8e2971e37be1dbd285e413df30ce34a72d1fb1a6c5669586ce252145f3a18b", "bytes": 1075602 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/58bad20ef7ea498d83d95ad71d9a4356.timing.json", "sha256": "2a55ca11c0739156114701f8225b855b74025b6fbd7cd64b74a0a5f80533396a", "bytes": 305 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/ba082b63a966418f96578d6a878b4e6d.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/ba082b63a966418f96578d6a878b4e6d.request.json", "sha256": "63dc7d3bf0ea328c7500cf3bae2e7fd27c9e892aefb048095452176b8b824609", "bytes": 101 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/ba082b63a966418f96578d6a878b4e6d.response.txt", "sha256": "ed1b023f479e19556fa6e1706214774e58507e5b5ba80bed839d6ef7ac83a272", "bytes": 153 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/ba082b63a966418f96578d6a878b4e6d.timing.json", "sha256": "abb06b099fbb652bc4b962b1e76b653c4fc7902a2283792d1530d4ba8672205e", "bytes": 305 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/terminal_evidence/budget_ledger.json", "sha256": "b766571c595524addaa12a5947db9e62f56042314a6f8fabf1ded77e1edb1485", "bytes": 116 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/workspace/solution.py", "sha256": "9ac3f14c158e48c78c52f10abfbadf05a3b66a8253fa4ad2f2b99491c93fea53", "bytes": 1616, "content": "from datetime import datetime, timezone, timedelta\nimport math\n\n\ndef retry_after(value, now):\n \"\"\"Parse an HTTP Retry-After header according to the task contract.\"\"\"\n # Reject non-string inputs\n if not isinstance(value, str):\n return None\n\n s = value.strip()\n\n # Check if s is only ASCII digits\n if s.isdigit():\n # No signs, no fractions, no extra text\n try:\n n = int(s)\n if n < 0:\n return None\n return n\n except Exception:\n return None\n\n # Otherwise, parse as HTTP-date of form 'Wed, 21 Oct 2015 07:28:00 GMT'\n # Validate format strictly\n # Must end with ' GMT'\n if not s.endswith(' GMT'):\n return None\n\n # Remove trailing ' GMT'\n date_str = s[:-4]\n\n # Parse date_str with format '%a, %d %b %Y %H:%M:%S'\n # This format is case sensitive for weekday and month\n # We must validate weekday matches the date\n try:\n dt = datetime.strptime(date_str, '%a, %d %b %Y %H:%M:%S')\n except Exception:\n return None\n\n # dt is naive, interpret as GMT (UTC)\n dt = dt.replace(tzinfo=timezone.utc)\n\n # Validate weekday matches\n # dt.weekday() Monday=0 ... Sunday=6\n # dt.strftime('%a') gives abbreviated weekday name\n # Compare with the weekday in the input string (first 3 chars)\n input_weekday = s[:3]\n actual_weekday = dt.strftime('%a')\n if input_weekday != actual_weekday:\n return None\n\n # Calculate difference in seconds\n diff = (dt - now).total_seconds()\n # Return max(0, ceil(diff)) as int\n return max(0, math.ceil(diff))\n", "contentRedacted": false, "publishedContentSha256": "9ac3f14c158e48c78c52f10abfbadf05a3b66a8253fa4ad2f2b99491c93fea53" } ], "disclosure": "Synthetic inputs and observed execution. Original-byte hashes retained; selected source and oracle outputs reproduced. Private provider transcripts remain local.", "reviewedEvidence": { "provenance": "Synthetic test input; actual installed CLI execution; reviewed public-safe evidence fixture.", "runId": "20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40", "task": "Synthetic benchmark task: Retry-After parsing. Implement `retry_after(value, now)` in solution.py using only Python standard library. now is an aware UTC datetime. Return a nonnegative integer number of seconds or None for invalid input. Strings containing only ASCII decimal digits after surrounding whitespace return that integer. Otherwise accept an RFC HTTP-date of form `Wed, 21 Oct 2015 07:28:00 GMT`, case sensitive, validated weekday must agree with date. For a valid HTTP date return max(0, ceil((date-now).total_seconds())). Reject non-string inputs, signs, fractions, non-ASCII digits, extra text, and non-GMT timezone names. Do not change any other source file. Hidden tests are unavailable. Write the implementation to disk and finish. Do not install dependencies, access network, use other models, or edit unrelated paths.\n", "freeze": { "task_sha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "oracle_sha256": "3675736119622cac168d711045426c6678bd08212b9bedf1aab71126a7ef1a76", "starter_sha256": "595a2a41e7bfd09ad28045cb3be95f0126d702eb25e6f53c723c0b77c46b44e2", "frozen_before_execution_at": 1792154239.814358 }, "candidateCode": "from datetime import datetime, timezone, timedelta\nimport math\n\n\ndef retry_after(value, now):\n \"\"\"Parse an HTTP Retry-After header according to the task contract.\"\"\"\n # Reject non-string inputs\n if not isinstance(value, str):\n return None\n\n s = value.strip()\n\n # Check if s is only ASCII digits\n if s.isdigit():\n # No signs, no fractions, no extra text\n try:\n n = int(s)\n if n < 0:\n return None\n return n\n except Exception:\n return None\n\n # Otherwise, parse as HTTP-date of form 'Wed, 21 Oct 2015 07:28:00 GMT'\n # Validate format strictly\n # Must end with ' GMT'\n if not s.endswith(' GMT'):\n return None\n\n # Remove trailing ' GMT'\n date_str = s[:-4]\n\n # Parse date_str with format '%a, %d %b %Y %H:%M:%S'\n # This format is case sensitive for weekday and month\n # We must validate weekday matches the date\n try:\n dt = datetime.strptime(date_str, '%a, %d %b %Y %H:%M:%S')\n except Exception:\n return None\n\n # dt is naive, interpret as GMT (UTC)\n dt = dt.replace(tzinfo=timezone.utc)\n\n # Validate weekday matches\n # dt.weekday() Monday=0 ... Sunday=6\n # dt.strftime('%a') gives abbreviated weekday name\n # Compare with the weekday in the input string (first 3 chars)\n input_weekday = s[:3]\n actual_weekday = dt.strftime('%a')\n if input_weekday != actual_weekday:\n return None\n\n # Calculate difference in seconds\n diff = (dt - now).total_seconds()\n # Return max(0, ceil(diff)) as int\n return max(0, math.ceil(diff))\n", "oracle": { "passed": 18, "total": 20, "cases": [ { "input": "0", "expected": 0, "actual": 0, "pass": true }, { "input": " 120\t", "expected": 120, "actual": 120, "pass": true }, { "input": "0007", "expected": 7, "actual": 7, "pass": true }, { "input": "-1", "expected": null, "actual": null, "pass": true }, { "input": "+1", "expected": null, "actual": null, "pass": true }, { "input": "1.5", "expected": null, "actual": null, "pass": true }, { "input": "１２", "expected": null, "actual": 12, "pass": false }, { "input": "١٢", "expected": null, "actual": 12, "pass": false }, { "input": "", "expected": null, "actual": null, "pass": true }, { "input": null, "expected": null, "actual": null, "pass": true }, { "input": 4, "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 GMT", "expected": 1, "actual": 1, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:01 GMT", "expected": 2, "actual": 2, "pass": true }, { "input": "Wed, 21 Oct 2015 07:27:00 GMT", "expected": 0, "actual": 0, "pass": true }, { "input": "Tue, 21 Oct 2015 07:28:00 GMT", "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 UTC", "expected": null, "actual": null, "pass": true }, { "input": "wed, 21 Oct 2015 07:28:00 GMT", "expected": null, "actual": null, "pass": true }, { "input": "Wed, 31 Feb 2015 07:28:00 GMT", "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 GMT trailing", "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 +0000", "expected": null, "actual": null, "pass": true } ] }, "providerCalls": [ { "requestEvidence": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/09d41c4b0fc84825ac81794419ad8d88.request.json", "requestSha256": "4a1bfd643069ca290c79d1510f86f36e4cbb06adcb11c761a6e6c8869178402a", "responseSha256": "c47e8ef04543cd904446c64e2b1a60e062733ad48e671649fdf2120b7c52e4bb", "httpStatus": 200, "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": "byok/openai/gpt-4.1-mini", "backendModel": "openai/gpt-4.1-mini", "resolutionSource": "explicit_byok_route", "generationControls": { "max_tokens": 8192, "temperature": 0, "reasoning_choice": { "mode": "default" }, "auto_failover": false, "stream": true }, "inputTokens": 915, "outputTokens": 478, "totalTokens": 1393, "costUsd": 0.001131, "energyWh": null, "finishReasons": [ "tool_calls" ], "additiveReasoningTokens": null, "nestedReasoningTokens": null } ], "gatewayBefore": { "status": "healthy", "service": "local_gateway", "mode": "full", "port": 6120, "timestamp": "2026-10-16T12:37:19.952603+00:00", "uptime_seconds": 249.26363587379456, "server_start_time": 1792153990.688971, "git_sha": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "build_time": "2026-10-16T08:32:43-04:00", "version": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "interop_min": 1, "interop_max": 1, "dependencies": { "event_stream": "http://localhost:6102" }, "lightrag": { "enabled": true, "status": "online", "files_indexed": 8020, "url": "http://127.0.0.1:6150" }, "cloud_relay": { "connected": true, "authenticated": true }, "context_status": { "summary": "ok", "target_context": 65536, "providers": [ { "provider": "ollama", "status": "ok", "actual_context": 131072, "target_context": 65536, "model": "deepseek-r1:7b" }, { "provider": "lmstudio", "status": "ok", "actual_context": 262144, "target_context": 65536, "model": "verdict-4-35b-toolterse-v5-mlx" }, { "provider": "llamacpp", "status": "context_unknown", "actual_context": null, "target_context": 65536, "model": null } ] } }, "gatewayAfter": { "status": "healthy", "service": "local_gateway", "mode": "full", "port": 6120, "timestamp": "2026-10-16T12:37:32.061096+00:00", "uptime_seconds": 261.37212800979614, "server_start_time": 1792153990.688971, "git_sha": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "build_time": "2026-10-16T08:32:43-04:00", "version": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "interop_min": 1, "interop_max": 1, "dependencies": { "event_stream": "http://localhost:6102" }, "lightrag": { "enabled": true, "status": "online", "files_indexed": 8020, "url": "http://127.0.0.1:6150" }, "cloud_relay": { "connected": true, "authenticated": true }, "context_status": { "summary": "ok", "target_context": 65536, "providers": [ { "provider": "ollama", "status": "ok", "actual_context": 131072, "target_context": 65536, "model": "deepseek-r1:7b" }, { "provider": "lmstudio", "status": "ok", "actual_context": 262144, "target_context": 65536, "model": "verdict-4-35b-toolterse-v5-mlx" }, { "provider": "llamacpp", "status": "context_unknown", "actual_context": null, "target_context": 65536, "model": null } ] } }, "rawArtifacts": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40", "rawResultSha256": "98332b609998b84e549ddc03e41eff93a65f3a191aca16a2cf0808a547d826c0", "rawEvidenceHashes": { "health_before.json": "ee38b10bae773a638f9713fce7ca9d3eca5bba0f59906d2d6428e4024639cef5", "stdout.txt": "bcc833bf9a793c329128951e9e93cb72f6496be6a2a624555771b1294a2f0244", "policy.json": "e1d7a7f7f022f69dcb3e7fc79a820f84a8425324b888b94cfbc0099cb6f4bec7", "health_after.json": "dafb792d17be7fe7ae6a068efd849e920de0c4e637b71bf1b30f538c31c6dee4", "sandbox_preflight.json": "8415d156075475eba2b82a04636f833e85484020de5e6424f6140cd0ac61390f", "budget_ledger_at_terminal.json": "b766571c595524addaa12a5947db9e62f56042314a6f8fabf1ded77e1edb1485", "prompt.txt": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "oracle.stdout.txt": "3e7b0982f217a41f229865a072cdee2a778735cbabe56238a3f0d4e91cab0746", "sandbox.sb": "ea2c0a2b2e4c5476f56645f2c73510a2522b7ab92b8ef48bd426895990543bba", "stderr.txt": "12918d24b35f57fe9ae75bfe6f5ace516ed564f8484a1a307d2186a36df7291a", "launch.json": "7d29433d77a5fa111d2b286e6d033b962656f43df102ee7247bff74e454d91f9", "invocation.json": "ee0520da830e943c1225ca021a97b745b8e32e2cab4303a7bfaf7715361fa7b6", "oracle.stderr.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "freeze.json": "5a4646acc410e09135dadbd612c97594199bb05cd2adee51b220b49f8eca1bf0", "product_source.json": "886ed65f0e33e1ee8f21ada21c10572aa8251406bf136ae7e79faa5af069a741", "post-terminal-settlement.json": "643fd167b0900ea34d63f94874be56342009f996530e64aaddd19f3e3239fae6", "workspace/solution.py": "9ac3f14c158e48c78c52f10abfbadf05a3b66a8253fa4ad2f2b99491c93fea53", "private_producer/state.json": "8870ca6011c2280870864f2703a73563784745aee34e24eed655f59236ce9c4f", "private_producer/gateway-0001.sse": "c47e8ef04543cd904446c64e2b1a60e062733ad48e671649fdf2120b7c52e4bb", "private_producer/runtime/compaction_events.jsonl": "7acc0441312353ab24cdf872490dbc798793c77bd3cd0348b6b11844db7e5494", "private_producer/runtime/.verdict_code_usage.inv-830178e0d7c146848645d43b2ebf503d.json": "cced61791a19214eb2910832fcc1bb35a53737fb4186d8ba5e8563c74ab86890", "private_producer/runtime/model_cache/availability_cache.json": "b90366fc89fd069c335048905b6259edd0f5bcbac6f4b1b3f752276e780e3e43", "private_producer/runtime/artifacts/runs/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40-t1/comms.txt": "c7ecbd2a6afd44070267188890b51e94fdc32280112e6bf4d93d754ea5718108", "private_producer/runtime/artifacts/runs/20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40/comms.txt": "dab053bd232e9e2c7a5ebabe673f46bbbd58c4f09e9b911b9834a33ea640972d", "terminal_evidence/58bad20ef7ea498d83d95ad71d9a4356.timing.json": "2a55ca11c0739156114701f8225b855b74025b6fbd7cd64b74a0a5f80533396a", "terminal_evidence/412ef4f4bc024aa1b572d9d47c453a99.request.json": "1ce9fd30eb6307a789013531fafa6783691de6024645b61fa7535304f3e49dd4", "terminal_evidence/412ef4f4bc024aa1b572d9d47c453a99.response.txt": "d5f0928538dd5176a4b8dc1555f217970eaccb5665dd58a29412bebec130806e", "terminal_evidence/3904f4853fb347d28de8c1135a93f086.http.json": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "terminal_evidence/09d41c4b0fc84825ac81794419ad8d88.timing.json": "92203001b0f2d73f1c3eccbbb22c70774659abc8d786beda7bcab76e5afd199b", "terminal_evidence/58bad20ef7ea498d83d95ad71d9a4356.request.json": "00d4db88f698768b96dbb446b10cb99c60359031af92a6ca1bd72189b88e34c4", "terminal_evidence/2bd16bc06a4a405aaa060784f6df0da1.timing.json": "26943b5e97506d4fcd6ac2a89ce5257316a151a132153c54b02d9f7083a8eb2b", "terminal_evidence/ba082b63a966418f96578d6a878b4e6d.request.json": "63dc7d3bf0ea328c7500cf3bae2e7fd27c9e892aefb048095452176b8b824609", "terminal_evidence/ba082b63a966418f96578d6a878b4e6d.timing.json": "abb06b099fbb652bc4b962b1e76b653c4fc7902a2283792d1530d4ba8672205e", "terminal_evidence/58bad20ef7ea498d83d95ad71d9a4356.response.txt": "ac8e2971e37be1dbd285e413df30ce34a72d1fb1a6c5669586ce252145f3a18b", "terminal_evidence/ba082b63a966418f96578d6a878b4e6d.response.txt": "ed1b023f479e19556fa6e1706214774e58507e5b5ba80bed839d6ef7ac83a272", "terminal_evidence/412ef4f4bc024aa1b572d9d47c453a99.timing.json": "c86dabc0a1cac90aa2d303f3d657008f7e65418e7a0588c6e45c74bffe44d27d", "terminal_evidence/2bd16bc06a4a405aaa060784f6df0da1.http.json": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "terminal_evidence/budget_ledger.json": "b766571c595524addaa12a5947db9e62f56042314a6f8fabf1ded77e1edb1485", "terminal_evidence/3904f4853fb347d28de8c1135a93f086.timing.json": "26c0e1942c8704bae97a63f262f4e99217ed43534750a663580d9aa196cc1c4e", "terminal_evidence/3904f4853fb347d28de8c1135a93f086.request.json": "c25f6aa1a57fedf1a8b70de26031235b08750a9132570d894678200161e77c12", "terminal_evidence/3904f4853fb347d28de8c1135a93f086.response.txt": "0398162f3d22f78fe804778d9e6f3bf1c189970aeef88838d1085d4b7197f55b", "terminal_evidence/ba082b63a966418f96578d6a878b4e6d.http.json": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "terminal_evidence/2bd16bc06a4a405aaa060784f6df0da1.response.txt": "95a7678e96278f7641daaf7f8e36d9b35742fe3aabbce5a046b1f79124e3da5b", "terminal_evidence/412ef4f4bc024aa1b572d9d47c453a99.http.json": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "terminal_evidence/2bd16bc06a4a405aaa060784f6df0da1.request.json": "1ce9fd30eb6307a789013531fafa6783691de6024645b61fa7535304f3e49dd4", "terminal_evidence/58bad20ef7ea498d83d95ad71d9a4356.http.json": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "terminal_evidence/09d41c4b0fc84825ac81794419ad8d88.http.json": "7b7d2ba7da38df349730b844f5a7ee404601546b1255c0860ee52f313dde4f73", "terminal_evidence/09d41c4b0fc84825ac81794419ad8d88.response.txt": "c47e8ef04543cd904446c64e2b1a60e062733ad48e671649fdf2120b7c52e4bb", "terminal_evidence/09d41c4b0fc84825ac81794419ad8d88.request.json": "4a1bfd643069ca290c79d1510f86f36e4cbb06adcb11c761a6e6c8869178402a", "source_snapshot/tools/verdict-code/VERSION": "c71dc09ace4b582eadda89d7dfda246c80fa899802686708c2cf11cc33590e06", "source_snapshot/tools/verdict-code/verdict_code/agent.py": "7c0f6171d9cca143ca04f629570aceeb7d3c728e6c5cd3a30434b886c4007e0e", "source_snapshot/tools/verdict-code/verdict_code/main.py": "af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab", "source_snapshot/tools/verdict-code/verdict_code/tools/__init__.py": "609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57", "source_snapshot/scripts/benchmarks/agent_comparison_rerun_20261016.py": "c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0", "source_snapshot/scripts/benchmarks/agent_comparison_completion_repair_20261016.py": "827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa" }, "productSource": { "head": "e4e758492ee2749b88db2bf027fa134d93c66428", "files": { "scripts/benchmarks/agent_comparison_rerun_20261016.py": "c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0", "tools/verdict-code/verdict_code/agent.py": "7c0f6171d9cca143ca04f629570aceeb7d3c728e6c5cd3a30434b886c4007e0e", "tools/verdict-code/verdict_code/tools/__init__.py": "609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57", "tools/verdict-code/verdict_code/main.py": "af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab", "tools/verdict-code/VERSION": "c71dc09ace4b582eadda89d7dfda246c80fa899802686708c2cf11cc33590e06", "scripts/benchmarks/agent_comparison_completion_repair_20261016.py": "827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa" }, "retainedSourceBytes": "source_snapshot/" }, "protocol": { "schema": "agent_comparison_rerun_policy.v1", "temperature": 0, "timeout_seconds": 240, "per_request_output_tokens": 8192, "proxy_shared_max_input_tokens": 160000, "proxy_shared_max_output_tokens": 32000, "proxy_reservation": "entire request UTF8 bytes +32/message+128; providerusage settled after eachcall; failclosed ifmissing", "maximum_input_tokens": 160000, "total_token_budget": 400000, "max_turns": 24, "proxy_read_timeout_seconds": 270, "verdict_stream_idle_timeout_seconds": 240, "aider_request_timeout_seconds": 240, "reasoning": { "mode": "default" }, "models": "exact pins, no failover", "task_file": "solution.py", "hidden_oracle_fed_to_agent": false, "producer_location": "outside candidate workspace; candidate read denied", "aider_context": "live exact catalog context metadata; history ceiling=min(inputbudget,context/2)", "appendix_model_changed": false }, "apparatusInvalidation": "Diagnostic only: request-local initial tool narrowing skipped the caller-authorized behavioral review. Generic harness regression repaired in 1.4.564; independent scorer outcomes are retained but these timings do not establish a successful latency repair." } }