{ "schema": "agent-rerun-receipt.v1", "record": { "id": "20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/google/models/gemini-2.5-flash", "resolvedModel": "byok/google/models/gemini-2.5-flash", "wallSeconds": 19.476, "inputTokens": 992, "outputTokens": 2706, "totalTokens": 3698, "costUsd": 0.0070626000000000005, "energyWh": null, "assertionsPassed": 16, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Diagnostic only: request-local initial tool narrowing skipped the caller-authorized behavioral review. Generic harness regression repaired in 1.4.564; independent scorer outcomes are retained but these timings do not establish a successful latency repair." ], "evidenceFile": "20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660.json", "archived": true }, "sourceRecord": { "id": "20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660", "originalCampaign": "agent_comparison_20261015", "rerun": true, "correctionOf": null, "cohort": "original-model", "family": "same-model-harness-rerun", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/google/models/gemini-2.5-flash", "resolvedModel": "byok/google/models/gemini-2.5-flash", "status": "FAIL", "wallSeconds": 19.476, "inputTokens": 992, "outputTokens": 2706, "totalTokens": 3698, "costUsd": 0.0070626000000000005, "energyWh": null, "assertionsPassed": 16, "assertionsTotal": 20, "observedCodePass": false, "exitCode": 0, "evidencePaths": [ "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660" ], "arm": "E", "admission": "NOT_VERIFIED", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Diagnostic only: request-local initial tool narrowing skipped the caller-authorized behavioral review. Generic harness regression repaired in 1.4.564; independent scorer outcomes are retained but these timings do not establish a successful latency repair." ], "freeze": { "task_sha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "oracle_sha256": "3675736119622cac168d711045426c6678bd08212b9bedf1aab71126a7ef1a76", "starter_sha256": "595a2a41e7bfd09ad28045cb3be95f0126d702eb25e6f53c723c0b77c46b44e2", "frozen_before_execution_at": 1792154259.764983 }, "mutableRuntimeFiles": [ "budget_ledger.json", "8b72190571c1432880cad84644ac3cf8.timing.json", "90a8998147c244e0896747419dc231d7.response.txt", "90a8998147c244e0896747419dc231d7.request.json", "1cdda834a6b34330a4ed20721729a98c.request.json", "1cdda834a6b34330a4ed20721729a98c.response.txt", "16288a36d0ce4664b58effcdcd80d021.http.json", "16288a36d0ce4664b58effcdcd80d021.request.json", "16288a36d0ce4664b58effcdcd80d021.response.txt", "b84dc63fa6c14c038b8abfefb095a3c0.http.json", "fc3c8f3abe0f46a486610bb511ab714c.response.txt", "fc3c8f3abe0f46a486610bb511ab714c.request.json", "b84dc63fa6c14c038b8abfefb095a3c0.response.txt", "8b72190571c1432880cad84644ac3cf8.http.json", "b84dc63fa6c14c038b8abfefb095a3c0.request.json", "fc3c8f3abe0f46a486610bb511ab714c.timing.json", "fc3c8f3abe0f46a486610bb511ab714c.http.json", "1cdda834a6b34330a4ed20721729a98c.timing.json", "16288a36d0ce4664b58effcdcd80d021.timing.json", "90a8998147c244e0896747419dc231d7.http.json", "1cdda834a6b34330a4ed20721729a98c.http.json", "90a8998147c244e0896747419dc231d7.timing.json", "8b72190571c1432880cad84644ac3cf8.response.txt", "b84dc63fa6c14c038b8abfefb095a3c0.timing.json", "8b72190571c1432880cad84644ac3cf8.request.json" ], "terminalBudgetSnapshot": "budget_ledger_at_terminal.json", "terminalTransportSnapshot": "terminal_evidence/", "rawStatus": "FAIL", "metadataTransportDiagnosticCount": 0, "diagnosticOnly": true, "tokenSource": "provider", "accountingComplete": true, "identityVerified": true, "providerCallCount": 1, "partialInputTokens": 992, "partialOutputTokens": 2706, "partialTotalTokens": 3698, "partialEstimatedCostUsd": 0.0070626000000000005, "reportedAdditiveReasoningTokens": null, "reportedReasoningTokens": 2231, "allCallsReportReasoning": true, "tokensPerSecond": 189.875, "knownTokensIncludingAdditiveReasoning": 3698, "knownTokensIncludingAdditiveReasoningPerSecond": 189.875, "generationCensored": false, "budgetControls": { "schema": "agent_comparison_rerun_policy.v1", "temperature": 0, "timeout_seconds": 240, "per_request_output_tokens": 8192, "proxy_shared_max_input_tokens": 160000, "proxy_shared_max_output_tokens": 32000, "proxy_reservation": "entire request UTF8 bytes +32/message+128; providerusage settled after eachcall; failclosed ifmissing", "maximum_input_tokens": 160000, "total_token_budget": 400000, "max_turns": 24, "proxy_read_timeout_seconds": 270, "verdict_stream_idle_timeout_seconds": 240, "aider_request_timeout_seconds": 240, "reasoning": { "mode": "default" }, "models": "exact pins, no failover", "task_file": "solution.py", "hidden_oracle_fed_to_agent": false, "producer_location": "outside candidate workspace; candidate read denied", "aider_context": "live exact catalog context metadata; history ceiling=min(inputbudget,context/2)", "appendix_model_changed": false }, "reasoningEffort": "default", "sharedProxyCapsEnforced": true, "actualBelowCommonCaps": true, "timeoutSeconds": 240, "terminalError": null, "terminalCompleted": true, "agentFailureReason": null, "sameGatewayInstance": true, "comparisonEligible": false, "publicEvidencePath": "artifacts/agent_comparison_completion_repair_20261016/matrix/public_evidence/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660.json", "publicEvidenceSha256": "cf0de1171022089a2fc87079234766f8e9e664bb3e7652885e814093a26cdcde", "apparatusInvalidation": "initial_tool_narrowing_skipped_behavioral_review" }, "sourceFile": "artifacts/agent_comparison_completion_repair_20261016/matrix/reviewed-summary.json", "sourceFileSha256": "cfdb4bb21c1038588f0951f2cbe4774e077c01691d7e0e53434881d3b9a16419", "files": [ { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/16288a36d0ce4664b58effcdcd80d021.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/16288a36d0ce4664b58effcdcd80d021.request.json", "sha256": "00d4db88f698768b96dbb446b10cb99c60359031af92a6ca1bd72189b88e34c4", "bytes": 75 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/16288a36d0ce4664b58effcdcd80d021.response.txt", "sha256": "ac8e2971e37be1dbd285e413df30ce34a72d1fb1a6c5669586ce252145f3a18b", "bytes": 1075602 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/16288a36d0ce4664b58effcdcd80d021.timing.json", "sha256": "ac9ec6cb49b36a06f060748eec590fe10a0c56416d93f65e8bc107997dc945ef", "bytes": 305 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/1cdda834a6b34330a4ed20721729a98c.http.json", "sha256": "7b7d2ba7da38df349730b844f5a7ee404601546b1255c0860ee52f313dde4f73", "bytes": 138 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/1cdda834a6b34330a4ed20721729a98c.request.json", "sha256": "6dc9f00cb3d42059e4e01748bec0a2e0995090476d2545e0303c9a10db0325e8", "bytes": 10907 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/1cdda834a6b34330a4ed20721729a98c.response.txt", "sha256": "86e102c940dc895f57fe4871f33a7671d7e8e59844bc7ca52477c49f358dd483", "bytes": 24592 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/1cdda834a6b34330a4ed20721729a98c.timing.json", "sha256": "65a189357151ce44f4337f679f1de0cfc90ea103d04f7dcfe717f13a6ff282f4", "bytes": 306 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/8b72190571c1432880cad84644ac3cf8.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/8b72190571c1432880cad84644ac3cf8.request.json", "sha256": "713d1ac64e7236f2a5ce79ea8370cc1d0ed9b675fe33e289d3ced047f8c60673", "bytes": 112 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/8b72190571c1432880cad84644ac3cf8.response.txt", "sha256": "9e7dc5ae9f2877ee585e9b87a34238fedad45709951d5634360e1f3bc94bc638", "bytes": 181 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/8b72190571c1432880cad84644ac3cf8.timing.json", "sha256": "0563f5c1ae4623a308e6edbd8c0c2816e7b2807dab0da2f1e75c9e02be107b05", "bytes": 305 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/90a8998147c244e0896747419dc231d7.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/90a8998147c244e0896747419dc231d7.request.json", "sha256": "1ce9fd30eb6307a789013531fafa6783691de6024645b61fa7535304f3e49dd4", "bytes": 58 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/90a8998147c244e0896747419dc231d7.response.txt", "sha256": "8a1c8b365af2439eb7a8cbbf618eee651baf580f41d190cd03a638c82b14f569", "bytes": 992 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/90a8998147c244e0896747419dc231d7.timing.json", "sha256": "b3a2fab406283018448310359ed0fb1a864f8f94e8cf3e71eb651590ba23cbfd", "bytes": 306 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/b84dc63fa6c14c038b8abfefb095a3c0.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/b84dc63fa6c14c038b8abfefb095a3c0.request.json", "sha256": "c25f6aa1a57fedf1a8b70de26031235b08750a9132570d894678200161e77c12", "bytes": 61 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/b84dc63fa6c14c038b8abfefb095a3c0.response.txt", "sha256": "1d0f898d89c5ddcd96e9de61b3b916ea494993646fd2ecf1924cc51fb94d5ea5", "bytes": 59657 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/b84dc63fa6c14c038b8abfefb095a3c0.timing.json", "sha256": "9b65bb5f3033bb26e72834771e7ea6c4d2976089c20f133772070625d6f0972c", "bytes": 305 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/budget_ledger.json", "sha256": "544a107eaa84eb48e0bf79345290fd5c6725d85a1ca975961fe0c8a8f00463bc", "bytes": 117 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/budget_ledger_at_terminal.json", "sha256": "544a107eaa84eb48e0bf79345290fd5c6725d85a1ca975961fe0c8a8f00463bc", "bytes": 117 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/fc3c8f3abe0f46a486610bb511ab714c.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/fc3c8f3abe0f46a486610bb511ab714c.request.json", "sha256": "1ce9fd30eb6307a789013531fafa6783691de6024645b61fa7535304f3e49dd4", "bytes": 58 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/fc3c8f3abe0f46a486610bb511ab714c.response.txt", "sha256": "a8f20b2223507b0b51fb4dfd0045534419ec3dfdb6f629768093099299ef2d7f", "bytes": 993 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/fc3c8f3abe0f46a486610bb511ab714c.timing.json", "sha256": "8e9fd2c7b594fec5678f32fd5fcb10c310a9e0df6d87b7886cfc9ef432fcd2e9", "bytes": 307 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/freeze.json", "sha256": "fb939eae69850828a5e854b55a2c6a2e70182aad1e2c4aa49294aa5c4e547011", "bytes": 313, "content": "{\n \"task_sha256\": \"6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe\",\n \"oracle_sha256\": \"3675736119622cac168d711045426c6678bd08212b9bedf1aab71126a7ef1a76\",\n \"starter_sha256\": \"595a2a41e7bfd09ad28045cb3be95f0126d702eb25e6f53c723c0b77c46b44e2\",\n \"frozen_before_execution_at\": 1792154259.764983\n}", "contentRedacted": false, "publishedContentSha256": "fb939eae69850828a5e854b55a2c6a2e70182aad1e2c4aa49294aa5c4e547011" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/health_after.json", "sha256": "93abf23398da1d14c4781a069f04b47086e81c56afd403d192ff6f73f6b9bfd2", "bytes": 1327 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/health_before.json", "sha256": "ecf2ea6815c6c77b174457faf2aae9867f7a03551a9e5ddb013d8857d4c88c24", "bytes": 1325 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/invocation.json", "sha256": "492a0b1256be6ee9097b261492b0c5c47dbf021dedd4a039e8550237d3d2c7f6", "bytes": 2248 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/launch.json", "sha256": "bebc35a7d9b27985cf410827410596709b3f52cfaff8c06752dc8179ad9c192b", "bytes": 188 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/oracle.stderr.txt", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes": 0, "content": "", "contentRedacted": false, "publishedContentSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/oracle.stdout.txt", "sha256": "a74c160266f3022da7be962edd43939c083261346fca7a18ca12386897dc1d04", "bytes": 1576, "content": "{\"passed\": 16, \"total\": 20, \"cases\": [{\"input\": \"0\", \"expected\": 0, \"actual\": 0, \"pass\": true}, {\"input\": \" 120\\t\", \"expected\": 120, \"actual\": 120, \"pass\": true}, {\"input\": \"0007\", \"expected\": 7, \"actual\": 7, \"pass\": true}, {\"input\": \"-1\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"+1\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"1.5\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"\\uff11\\uff12\", \"expected\": null, \"actual\": 12, \"pass\": false}, {\"input\": \"\\u0661\\u0662\", \"expected\": null, \"actual\": 12, \"pass\": false}, {\"input\": \"\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": null, \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": 4, \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 GMT\", \"expected\": 1, \"actual\": 1, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:01 GMT\", \"expected\": 2, \"actual\": 2, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:27:00 GMT\", \"expected\": 0, \"actual\": 0, \"pass\": true}, {\"input\": \"Tue, 21 Oct 2015 07:28:00 GMT\", \"expected\": null, \"actual\": 1, \"pass\": false}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 UTC\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"wed, 21 Oct 2015 07:28:00 GMT\", \"expected\": null, \"actual\": 1, \"pass\": false}, {\"input\": \"Wed, 31 Feb 2015 07:28:00 GMT\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 GMT trailing\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 +0000\", \"expected\": null, \"actual\": null, \"pass\": true}]}\n", "contentRedacted": false, "publishedContentSha256": "a74c160266f3022da7be962edd43939c083261346fca7a18ca12386897dc1d04" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/policy.json", "sha256": "e1d7a7f7f022f69dcb3e7fc79a820f84a8425324b888b94cfbc0099cb6f4bec7", "bytes": 921, "content": "{\n \"schema\": \"agent_comparison_rerun_policy.v1\",\n \"temperature\": 0,\n \"timeout_seconds\": 240,\n \"per_request_output_tokens\": 8192,\n \"proxy_shared_max_input_tokens\": 160000,\n \"proxy_shared_max_output_tokens\": 32000,\n \"proxy_reservation\": \"entire request UTF8 bytes +32/message+128; providerusage settled after eachcall; failclosed ifmissing\",\n \"maximum_input_tokens\": 160000,\n \"total_token_budget\": 400000,\n \"max_turns\": 24,\n \"proxy_read_timeout_seconds\": 270,\n \"verdict_stream_idle_timeout_seconds\": 240,\n \"aider_request_timeout_seconds\": 240,\n \"reasoning\": {\n \"mode\": \"default\"\n },\n \"models\": \"exact pins, no failover\",\n \"task_file\": \"solution.py\",\n \"hidden_oracle_fed_to_agent\": false,\n \"producer_location\": \"outside candidate workspace; candidate read denied\",\n \"aider_context\": \"live exact catalog context metadata; history ceiling=min(inputbudget,context/2)\",\n \"appendix_model_changed\": false\n}", "contentRedacted": false, "publishedContentSha256": "e1d7a7f7f022f69dcb3e7fc79a820f84a8425324b888b94cfbc0099cb6f4bec7" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/post-terminal-settlement.json", "sha256": "2292a385cf64c7f1736cb399ff59d67f7761744212a69efbc8d70b1aab15029c", "bytes": 380 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/private_producer/gateway-0001.sse", "sha256": "86e102c940dc895f57fe4871f33a7671d7e8e59844bc7ca52477c49f358dd483", "bytes": 24592 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/private_producer/runtime/.verdict_code_usage.inv-3a932375d67f445c865cc4dc4e9bd3e0.json", "sha256": "504c64ce3194b703798aea10ff30b1b0e0446431705230712f59469eb70a21c7", "bytes": 18665 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/private_producer/runtime/artifacts/runs/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/comms.txt", "sha256": "fd6187cab1074d50e959300ae89d712281bef6895d097d23195041e4ad810513", "bytes": 1282 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/private_producer/runtime/artifacts/runs/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660-t1/comms.txt", "sha256": "f572e467007899ee4b590810054db6f223e58e5d8f8db3560a8ab2f0116dbf3d", "bytes": 667 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/private_producer/runtime/compaction_events.jsonl", "sha256": "72a9994e10429fdd89925fb33fd1fe4b902b23926f107a12969bcf98d4d6b138", "bytes": 495 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/private_producer/runtime/model_cache/availability_cache.json", "sha256": "5dfc96953205766342eb65d8e12af163d89217b12ddc3f828358a346bef0613a", "bytes": 276 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/private_producer/state.json", "sha256": "bf4e5e9e56963cebb2d216fcba485b2b89aa41fdb8ec76587418df2c7dc442c1", "bytes": 2325 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/product_source.json", "sha256": "614bfff42c938a8b92446cfb0df7bd522709646f2518179e9728d8ce45ddfc5d", "bytes": 846, "content": "{\n \"head\": \"95a38c2d8322608f7a97004f3fd88236f4377f0d\",\n \"files\": {\n \"scripts/benchmarks/agent_comparison_rerun_20261016.py\": \"c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0\",\n \"tools/verdict-code/verdict_code/agent.py\": \"7c0f6171d9cca143ca04f629570aceeb7d3c728e6c5cd3a30434b886c4007e0e\",\n \"tools/verdict-code/verdict_code/tools/__init__.py\": \"609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57\",\n \"tools/verdict-code/verdict_code/main.py\": \"af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab\",\n \"tools/verdict-code/VERSION\": \"c71dc09ace4b582eadda89d7dfda246c80fa899802686708c2cf11cc33590e06\",\n \"scripts/benchmarks/agent_comparison_completion_repair_20261016.py\": \"827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa\"\n },\n \"retainedSourceBytes\": \"source_snapshot/\"\n}", "contentRedacted": false, "publishedContentSha256": "614bfff42c938a8b92446cfb0df7bd522709646f2518179e9728d8ce45ddfc5d" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/prompt.txt", "sha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "bytes": 837, "content": "Synthetic benchmark task: Retry-After parsing. Implement `retry_after(value, now)` in solution.py using only Python standard library. now is an aware UTC datetime. Return a nonnegative integer number of seconds or None for invalid input. Strings containing only ASCII decimal digits after surrounding whitespace return that integer. Otherwise accept an RFC HTTP-date of form `Wed, 21 Oct 2015 07:28:00 GMT`, case sensitive, validated weekday must agree with date. For a valid HTTP date return max(0, ceil((date-now).total_seconds())). Reject non-string inputs, signs, fractions, non-ASCII digits, extra text, and non-GMT timezone names. Do not change any other source file. Hidden tests are unavailable. Write the implementation to disk and finish. Do not install dependencies, access network, use other models, or edit unrelated paths.\n", "contentRedacted": false, "publishedContentSha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/result.json", "sha256": "bf695d5faae1d072206bfc0aa905a43cfb501ef0206c6b38be20d2c64b7d5d2e", "bytes": 9774 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/sandbox.sb", "sha256": "f9ba74f69f9d0ce0b4d8ea66b599c9d6510684d668a2c1aa336697252f8ff0bc", "bytes": 6052 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/sandbox_preflight.json", "sha256": "8415d156075475eba2b82a04636f833e85484020de5e6424f6140cd0ac61390f", "bytes": 93 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/source_snapshot/scripts/benchmarks/agent_comparison_completion_repair_20261016.py", "sha256": "827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa", "bytes": 13274 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/source_snapshot/scripts/benchmarks/agent_comparison_rerun_20261016.py", "sha256": "c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0", "bytes": 38332 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/source_snapshot/tools/verdict-code/VERSION", "sha256": "c71dc09ace4b582eadda89d7dfda246c80fa899802686708c2cf11cc33590e06", "bytes": 8 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/source_snapshot/tools/verdict-code/verdict_code/agent.py", "sha256": "7c0f6171d9cca143ca04f629570aceeb7d3c728e6c5cd3a30434b886c4007e0e", "bytes": 763436 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/source_snapshot/tools/verdict-code/verdict_code/main.py", "sha256": "af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab", "bytes": 625279 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/source_snapshot/tools/verdict-code/verdict_code/tools/__init__.py", "sha256": "609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57", "bytes": 269877 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/stderr.txt", "sha256": "4a86cc8c8b38e960a72784d9d58abf827fc0116508f39be1155ef47c1ef5d29c", "bytes": 6281 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/stdout.txt", "sha256": "6445a4761b0b8716b2e97be1cddfa7080f118ad45a0964bd025da8c7e41208e0", "bytes": 6025 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/16288a36d0ce4664b58effcdcd80d021.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/16288a36d0ce4664b58effcdcd80d021.request.json", "sha256": "00d4db88f698768b96dbb446b10cb99c60359031af92a6ca1bd72189b88e34c4", "bytes": 75 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/16288a36d0ce4664b58effcdcd80d021.response.txt", "sha256": "ac8e2971e37be1dbd285e413df30ce34a72d1fb1a6c5669586ce252145f3a18b", "bytes": 1075602 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/16288a36d0ce4664b58effcdcd80d021.timing.json", "sha256": "ac9ec6cb49b36a06f060748eec590fe10a0c56416d93f65e8bc107997dc945ef", "bytes": 305 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/1cdda834a6b34330a4ed20721729a98c.http.json", "sha256": "7b7d2ba7da38df349730b844f5a7ee404601546b1255c0860ee52f313dde4f73", "bytes": 138 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/1cdda834a6b34330a4ed20721729a98c.request.json", "sha256": "6dc9f00cb3d42059e4e01748bec0a2e0995090476d2545e0303c9a10db0325e8", "bytes": 10907 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/1cdda834a6b34330a4ed20721729a98c.response.txt", "sha256": "86e102c940dc895f57fe4871f33a7671d7e8e59844bc7ca52477c49f358dd483", "bytes": 24592 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/1cdda834a6b34330a4ed20721729a98c.timing.json", "sha256": "65a189357151ce44f4337f679f1de0cfc90ea103d04f7dcfe717f13a6ff282f4", "bytes": 306 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/8b72190571c1432880cad84644ac3cf8.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/8b72190571c1432880cad84644ac3cf8.request.json", "sha256": "713d1ac64e7236f2a5ce79ea8370cc1d0ed9b675fe33e289d3ced047f8c60673", "bytes": 112 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/8b72190571c1432880cad84644ac3cf8.response.txt", "sha256": "9e7dc5ae9f2877ee585e9b87a34238fedad45709951d5634360e1f3bc94bc638", "bytes": 181 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/8b72190571c1432880cad84644ac3cf8.timing.json", "sha256": "0563f5c1ae4623a308e6edbd8c0c2816e7b2807dab0da2f1e75c9e02be107b05", "bytes": 305 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/90a8998147c244e0896747419dc231d7.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/90a8998147c244e0896747419dc231d7.request.json", "sha256": "1ce9fd30eb6307a789013531fafa6783691de6024645b61fa7535304f3e49dd4", "bytes": 58 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/90a8998147c244e0896747419dc231d7.response.txt", "sha256": "8a1c8b365af2439eb7a8cbbf618eee651baf580f41d190cd03a638c82b14f569", "bytes": 992 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/90a8998147c244e0896747419dc231d7.timing.json", "sha256": "b3a2fab406283018448310359ed0fb1a864f8f94e8cf3e71eb651590ba23cbfd", "bytes": 306 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/b84dc63fa6c14c038b8abfefb095a3c0.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/b84dc63fa6c14c038b8abfefb095a3c0.request.json", "sha256": "c25f6aa1a57fedf1a8b70de26031235b08750a9132570d894678200161e77c12", "bytes": 61 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/b84dc63fa6c14c038b8abfefb095a3c0.response.txt", "sha256": "1d0f898d89c5ddcd96e9de61b3b916ea494993646fd2ecf1924cc51fb94d5ea5", "bytes": 59657 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/b84dc63fa6c14c038b8abfefb095a3c0.timing.json", "sha256": "9b65bb5f3033bb26e72834771e7ea6c4d2976089c20f133772070625d6f0972c", "bytes": 305 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/budget_ledger.json", "sha256": "544a107eaa84eb48e0bf79345290fd5c6725d85a1ca975961fe0c8a8f00463bc", "bytes": 117 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/fc3c8f3abe0f46a486610bb511ab714c.http.json", "sha256": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "bytes": 122 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/fc3c8f3abe0f46a486610bb511ab714c.request.json", "sha256": "1ce9fd30eb6307a789013531fafa6783691de6024645b61fa7535304f3e49dd4", "bytes": 58 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/fc3c8f3abe0f46a486610bb511ab714c.response.txt", "sha256": "a8f20b2223507b0b51fb4dfd0045534419ec3dfdb6f629768093099299ef2d7f", "bytes": 993 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/terminal_evidence/fc3c8f3abe0f46a486610bb511ab714c.timing.json", "sha256": "8e9fd2c7b594fec5678f32fd5fcb10c310a9e0df6d87b7886cfc9ef432fcd2e9", "bytes": 307 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/workspace/solution.py", "sha256": "fdd7d0e5dbc6a6f3008e375a58ce4c639dd8908b4133efd967b53c9dc936ca78", "bytes": 1524, "content": "import datetime\nimport math\n\ndef retry_after(value, now):\n \"\"\"Parse an HTTP Retry-After header according to the task contract.\"\"\"\n if not isinstance(value, str):\n return None\n\n # Try to parse as an integer\n stripped_value = value.strip()\n if stripped_value.isdigit():\n return int(stripped_value)\n\n # Try to parse as an HTTP-date\n try:\n # Example: Wed, 21 Oct 2015 07:28:00 GMT\n # %a: Weekday as locale’s abbreviated name.\n # %d: Day of the month as a zero-padded decimal number.\n # %b: Month as locale’s abbreviated name.\n # %Y: Year with century as a decimal number.\n # %H: Hour (24-hour clock) as a zero-padded decimal number.\n # %M: Minute as a zero-padded decimal number.\n # %S: Second as a zero-padded decimal number.\n # GMT: Literal string \"GMT\"\n parsed_date_naive = datetime.datetime.strptime(value, \"%a, %d %b %Y %H:%M:%S GMT\")\n \n # The problem states 'now' is an aware UTC datetime.\n # The 'GMT' in the string implies UTC.\n # We need to make 'parsed_date' aware and UTC for correct comparison.\n parsed_date_aware = parsed_date_naive.replace(tzinfo=datetime.timezone.utc)\n\n diff_seconds = (parsed_date_aware - now).total_seconds()\n return max(0, math.ceil(diff_seconds))\n except ValueError:\n # If parsing as HTTP-date fails (e.g., invalid format, weekday mismatch, non-GMT timezone),\n # strptime will raise a ValueError.\n return None\n", "contentRedacted": false, "publishedContentSha256": "fdd7d0e5dbc6a6f3008e375a58ce4c639dd8908b4133efd967b53c9dc936ca78" } ], "disclosure": "Synthetic inputs and observed execution. Original-byte hashes retained; selected source and oracle outputs reproduced. Private provider transcripts remain local.", "reviewedEvidence": { "provenance": "Synthetic test input; actual installed CLI execution; reviewed public-safe evidence fixture.", "runId": "20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660", "task": "Synthetic benchmark task: Retry-After parsing. Implement `retry_after(value, now)` in solution.py using only Python standard library. now is an aware UTC datetime. Return a nonnegative integer number of seconds or None for invalid input. Strings containing only ASCII decimal digits after surrounding whitespace return that integer. Otherwise accept an RFC HTTP-date of form `Wed, 21 Oct 2015 07:28:00 GMT`, case sensitive, validated weekday must agree with date. For a valid HTTP date return max(0, ceil((date-now).total_seconds())). Reject non-string inputs, signs, fractions, non-ASCII digits, extra text, and non-GMT timezone names. Do not change any other source file. Hidden tests are unavailable. Write the implementation to disk and finish. Do not install dependencies, access network, use other models, or edit unrelated paths.\n", "freeze": { "task_sha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "oracle_sha256": "3675736119622cac168d711045426c6678bd08212b9bedf1aab71126a7ef1a76", "starter_sha256": "595a2a41e7bfd09ad28045cb3be95f0126d702eb25e6f53c723c0b77c46b44e2", "frozen_before_execution_at": 1792154259.764983 }, "candidateCode": "import datetime\nimport math\n\ndef retry_after(value, now):\n \"\"\"Parse an HTTP Retry-After header according to the task contract.\"\"\"\n if not isinstance(value, str):\n return None\n\n # Try to parse as an integer\n stripped_value = value.strip()\n if stripped_value.isdigit():\n return int(stripped_value)\n\n # Try to parse as an HTTP-date\n try:\n # Example: Wed, 21 Oct 2015 07:28:00 GMT\n # %a: Weekday as locale’s abbreviated name.\n # %d: Day of the month as a zero-padded decimal number.\n # %b: Month as locale’s abbreviated name.\n # %Y: Year with century as a decimal number.\n # %H: Hour (24-hour clock) as a zero-padded decimal number.\n # %M: Minute as a zero-padded decimal number.\n # %S: Second as a zero-padded decimal number.\n # GMT: Literal string \"GMT\"\n parsed_date_naive = datetime.datetime.strptime(value, \"%a, %d %b %Y %H:%M:%S GMT\")\n \n # The problem states 'now' is an aware UTC datetime.\n # The 'GMT' in the string implies UTC.\n # We need to make 'parsed_date' aware and UTC for correct comparison.\n parsed_date_aware = parsed_date_naive.replace(tzinfo=datetime.timezone.utc)\n\n diff_seconds = (parsed_date_aware - now).total_seconds()\n return max(0, math.ceil(diff_seconds))\n except ValueError:\n # If parsing as HTTP-date fails (e.g., invalid format, weekday mismatch, non-GMT timezone),\n # strptime will raise a ValueError.\n return None\n", "oracle": { "passed": 16, "total": 20, "cases": [ { "input": "0", "expected": 0, "actual": 0, "pass": true }, { "input": " 120\t", "expected": 120, "actual": 120, "pass": true }, { "input": "0007", "expected": 7, "actual": 7, "pass": true }, { "input": "-1", "expected": null, "actual": null, "pass": true }, { "input": "+1", "expected": null, "actual": null, "pass": true }, { "input": "1.5", "expected": null, "actual": null, "pass": true }, { "input": "１２", "expected": null, "actual": 12, "pass": false }, { "input": "١٢", "expected": null, "actual": 12, "pass": false }, { "input": "", "expected": null, "actual": null, "pass": true }, { "input": null, "expected": null, "actual": null, "pass": true }, { "input": 4, "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 GMT", "expected": 1, "actual": 1, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:01 GMT", "expected": 2, "actual": 2, "pass": true }, { "input": "Wed, 21 Oct 2015 07:27:00 GMT", "expected": 0, "actual": 0, "pass": true }, { "input": "Tue, 21 Oct 2015 07:28:00 GMT", "expected": null, "actual": 1, "pass": false }, { "input": "Wed, 21 Oct 2015 07:28:00 UTC", "expected": null, "actual": null, "pass": true }, { "input": "wed, 21 Oct 2015 07:28:00 GMT", "expected": null, "actual": 1, "pass": false }, { "input": "Wed, 31 Feb 2015 07:28:00 GMT", "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 GMT trailing", "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 +0000", "expected": null, "actual": null, "pass": true } ] }, "providerCalls": [ { "requestEvidence": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/1cdda834a6b34330a4ed20721729a98c.request.json", "requestSha256": "6dc9f00cb3d42059e4e01748bec0a2e0995090476d2545e0303c9a10db0325e8", "responseSha256": "86e102c940dc895f57fe4871f33a7671d7e8e59844bc7ca52477c49f358dd483", "httpStatus": 200, "requestedModel": "byok/google/models/gemini-2.5-flash", "resolvedModel": "byok/google/models/gemini-2.5-flash", "backendModel": "google/models/gemini-2.5-flash", "resolutionSource": "explicit_byok_route", "generationControls": { "max_tokens": 8192, "temperature": 0, "reasoning_choice": { "mode": "default" }, "auto_failover": false, "stream": true }, "inputTokens": 992, "outputTokens": 2706, "totalTokens": 3698, "costUsd": 0.0070626000000000005, "energyWh": null, "finishReasons": [ "tool_calls" ], "additiveReasoningTokens": null, "nestedReasoningTokens": 2231 } ], "gatewayBefore": { "status": "healthy", "service": "local_gateway", "mode": "full", "port": 6120, "timestamp": "2026-10-16T12:37:39.821881+00:00", "uptime_seconds": 269.132915019989, "server_start_time": 1792153990.688971, "git_sha": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "build_time": "2026-10-16T08:32:43-04:00", "version": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "interop_min": 1, "interop_max": 1, "dependencies": { "event_stream": "http://localhost:6102" }, "lightrag": { "enabled": true, "status": "online", "files_indexed": 8020, "url": "http://127.0.0.1:6150" }, "cloud_relay": { "connected": true, "authenticated": true }, "context_status": { "summary": "ok", "target_context": 65536, "providers": [ { "provider": "ollama", "status": "ok", "actual_context": 131072, "target_context": 65536, "model": "deepseek-r1:7b" }, { "provider": "lmstudio", "status": "ok", "actual_context": 262144, "target_context": 65536, "model": "verdict-4-35b-toolterse-v5-mlx" }, { "provider": "llamacpp", "status": "context_unknown", "actual_context": null, "target_context": 65536, "model": null } ] } }, "gatewayAfter": { "status": "healthy", "service": "local_gateway", "mode": "full", "port": 6120, "timestamp": "2026-10-16T12:37:59.431414+00:00", "uptime_seconds": 288.74244594573975, "server_start_time": 1792153990.688971, "git_sha": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "build_time": "2026-10-16T08:32:43-04:00", "version": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "interop_min": 1, "interop_max": 1, "dependencies": { "event_stream": "http://localhost:6102" }, "lightrag": { "enabled": true, "status": "online", "files_indexed": 8020, "url": "http://127.0.0.1:6150" }, "cloud_relay": { "connected": true, "authenticated": true }, "context_status": { "summary": "ok", "target_context": 65536, "providers": [ { "provider": "ollama", "status": "ok", "actual_context": 131072, "target_context": 65536, "model": "deepseek-r1:7b" }, { "provider": "lmstudio", "status": "ok", "actual_context": 262144, "target_context": 65536, "model": "verdict-4-35b-toolterse-v5-mlx" }, { "provider": "llamacpp", "status": "context_unknown", "actual_context": null, "target_context": 65536, "model": null } ] } }, "rawArtifacts": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660", "rawResultSha256": "bf695d5faae1d072206bfc0aa905a43cfb501ef0206c6b38be20d2c64b7d5d2e", "rawEvidenceHashes": { "health_before.json": "ecf2ea6815c6c77b174457faf2aae9867f7a03551a9e5ddb013d8857d4c88c24", "stdout.txt": "6445a4761b0b8716b2e97be1cddfa7080f118ad45a0964bd025da8c7e41208e0", "policy.json": "e1d7a7f7f022f69dcb3e7fc79a820f84a8425324b888b94cfbc0099cb6f4bec7", "health_after.json": "93abf23398da1d14c4781a069f04b47086e81c56afd403d192ff6f73f6b9bfd2", "sandbox_preflight.json": "8415d156075475eba2b82a04636f833e85484020de5e6424f6140cd0ac61390f", "budget_ledger_at_terminal.json": "544a107eaa84eb48e0bf79345290fd5c6725d85a1ca975961fe0c8a8f00463bc", "prompt.txt": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "oracle.stdout.txt": "a74c160266f3022da7be962edd43939c083261346fca7a18ca12386897dc1d04", "sandbox.sb": "f9ba74f69f9d0ce0b4d8ea66b599c9d6510684d668a2c1aa336697252f8ff0bc", "stderr.txt": "4a86cc8c8b38e960a72784d9d58abf827fc0116508f39be1155ef47c1ef5d29c", "launch.json": "bebc35a7d9b27985cf410827410596709b3f52cfaff8c06752dc8179ad9c192b", "invocation.json": "492a0b1256be6ee9097b261492b0c5c47dbf021dedd4a039e8550237d3d2c7f6", "oracle.stderr.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "freeze.json": "fb939eae69850828a5e854b55a2c6a2e70182aad1e2c4aa49294aa5c4e547011", "product_source.json": "614bfff42c938a8b92446cfb0df7bd522709646f2518179e9728d8ce45ddfc5d", "post-terminal-settlement.json": "2292a385cf64c7f1736cb399ff59d67f7761744212a69efbc8d70b1aab15029c", "workspace/solution.py": "fdd7d0e5dbc6a6f3008e375a58ce4c639dd8908b4133efd967b53c9dc936ca78", "private_producer/state.json": "bf4e5e9e56963cebb2d216fcba485b2b89aa41fdb8ec76587418df2c7dc442c1", "private_producer/gateway-0001.sse": "86e102c940dc895f57fe4871f33a7671d7e8e59844bc7ca52477c49f358dd483", "private_producer/runtime/compaction_events.jsonl": "72a9994e10429fdd89925fb33fd1fe4b902b23926f107a12969bcf98d4d6b138", "private_producer/runtime/.verdict_code_usage.inv-3a932375d67f445c865cc4dc4e9bd3e0.json": "504c64ce3194b703798aea10ff30b1b0e0446431705230712f59469eb70a21c7", "private_producer/runtime/model_cache/availability_cache.json": "5dfc96953205766342eb65d8e12af163d89217b12ddc3f828358a346bef0613a", "private_producer/runtime/artifacts/runs/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660/comms.txt": "fd6187cab1074d50e959300ae89d712281bef6895d097d23195041e4ad810513", "private_producer/runtime/artifacts/runs/20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660-t1/comms.txt": "f572e467007899ee4b590810054db6f223e58e5d8f8db3560a8ab2f0116dbf3d", "terminal_evidence/8b72190571c1432880cad84644ac3cf8.timing.json": "0563f5c1ae4623a308e6edbd8c0c2816e7b2807dab0da2f1e75c9e02be107b05", "terminal_evidence/90a8998147c244e0896747419dc231d7.response.txt": "8a1c8b365af2439eb7a8cbbf618eee651baf580f41d190cd03a638c82b14f569", "terminal_evidence/90a8998147c244e0896747419dc231d7.request.json": "1ce9fd30eb6307a789013531fafa6783691de6024645b61fa7535304f3e49dd4", "terminal_evidence/1cdda834a6b34330a4ed20721729a98c.request.json": "6dc9f00cb3d42059e4e01748bec0a2e0995090476d2545e0303c9a10db0325e8", "terminal_evidence/1cdda834a6b34330a4ed20721729a98c.response.txt": "86e102c940dc895f57fe4871f33a7671d7e8e59844bc7ca52477c49f358dd483", "terminal_evidence/16288a36d0ce4664b58effcdcd80d021.http.json": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "terminal_evidence/16288a36d0ce4664b58effcdcd80d021.request.json": "00d4db88f698768b96dbb446b10cb99c60359031af92a6ca1bd72189b88e34c4", "terminal_evidence/16288a36d0ce4664b58effcdcd80d021.response.txt": "ac8e2971e37be1dbd285e413df30ce34a72d1fb1a6c5669586ce252145f3a18b", "terminal_evidence/b84dc63fa6c14c038b8abfefb095a3c0.http.json": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "terminal_evidence/fc3c8f3abe0f46a486610bb511ab714c.response.txt": "a8f20b2223507b0b51fb4dfd0045534419ec3dfdb6f629768093099299ef2d7f", "terminal_evidence/fc3c8f3abe0f46a486610bb511ab714c.request.json": "1ce9fd30eb6307a789013531fafa6783691de6024645b61fa7535304f3e49dd4", "terminal_evidence/b84dc63fa6c14c038b8abfefb095a3c0.response.txt": "1d0f898d89c5ddcd96e9de61b3b916ea494993646fd2ecf1924cc51fb94d5ea5", "terminal_evidence/8b72190571c1432880cad84644ac3cf8.http.json": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "terminal_evidence/b84dc63fa6c14c038b8abfefb095a3c0.request.json": "c25f6aa1a57fedf1a8b70de26031235b08750a9132570d894678200161e77c12", "terminal_evidence/budget_ledger.json": "544a107eaa84eb48e0bf79345290fd5c6725d85a1ca975961fe0c8a8f00463bc", "terminal_evidence/fc3c8f3abe0f46a486610bb511ab714c.timing.json": "8e9fd2c7b594fec5678f32fd5fcb10c310a9e0df6d87b7886cfc9ef432fcd2e9", "terminal_evidence/fc3c8f3abe0f46a486610bb511ab714c.http.json": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "terminal_evidence/1cdda834a6b34330a4ed20721729a98c.timing.json": "65a189357151ce44f4337f679f1de0cfc90ea103d04f7dcfe717f13a6ff282f4", "terminal_evidence/16288a36d0ce4664b58effcdcd80d021.timing.json": "ac9ec6cb49b36a06f060748eec590fe10a0c56416d93f65e8bc107997dc945ef", "terminal_evidence/90a8998147c244e0896747419dc231d7.http.json": "835c69d94140a3e5dd8b63f42d24456f2d8db61d822cf173e22e80d06155a4b8", "terminal_evidence/1cdda834a6b34330a4ed20721729a98c.http.json": "7b7d2ba7da38df349730b844f5a7ee404601546b1255c0860ee52f313dde4f73", "terminal_evidence/90a8998147c244e0896747419dc231d7.timing.json": "b3a2fab406283018448310359ed0fb1a864f8f94e8cf3e71eb651590ba23cbfd", "terminal_evidence/8b72190571c1432880cad84644ac3cf8.response.txt": "9e7dc5ae9f2877ee585e9b87a34238fedad45709951d5634360e1f3bc94bc638", "terminal_evidence/b84dc63fa6c14c038b8abfefb095a3c0.timing.json": "9b65bb5f3033bb26e72834771e7ea6c4d2976089c20f133772070625d6f0972c", "terminal_evidence/8b72190571c1432880cad84644ac3cf8.request.json": "713d1ac64e7236f2a5ce79ea8370cc1d0ed9b675fe33e289d3ced047f8c60673", "source_snapshot/tools/verdict-code/VERSION": "c71dc09ace4b582eadda89d7dfda246c80fa899802686708c2cf11cc33590e06", "source_snapshot/tools/verdict-code/verdict_code/agent.py": "7c0f6171d9cca143ca04f629570aceeb7d3c728e6c5cd3a30434b886c4007e0e", "source_snapshot/tools/verdict-code/verdict_code/main.py": "af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab", "source_snapshot/tools/verdict-code/verdict_code/tools/__init__.py": "609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57", "source_snapshot/scripts/benchmarks/agent_comparison_rerun_20261016.py": "c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0", "source_snapshot/scripts/benchmarks/agent_comparison_completion_repair_20261016.py": "827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa" }, "productSource": { "head": "95a38c2d8322608f7a97004f3fd88236f4377f0d", "files": { "scripts/benchmarks/agent_comparison_rerun_20261016.py": "c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0", "tools/verdict-code/verdict_code/agent.py": "7c0f6171d9cca143ca04f629570aceeb7d3c728e6c5cd3a30434b886c4007e0e", "tools/verdict-code/verdict_code/tools/__init__.py": "609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57", "tools/verdict-code/verdict_code/main.py": "af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab", "tools/verdict-code/VERSION": "c71dc09ace4b582eadda89d7dfda246c80fa899802686708c2cf11cc33590e06", "scripts/benchmarks/agent_comparison_completion_repair_20261016.py": "827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa" }, "retainedSourceBytes": "source_snapshot/" }, "protocol": { "schema": "agent_comparison_rerun_policy.v1", "temperature": 0, "timeout_seconds": 240, "per_request_output_tokens": 8192, "proxy_shared_max_input_tokens": 160000, "proxy_shared_max_output_tokens": 32000, "proxy_reservation": "entire request UTF8 bytes +32/message+128; providerusage settled after eachcall; failclosed ifmissing", "maximum_input_tokens": 160000, "total_token_budget": 400000, "max_turns": 24, "proxy_read_timeout_seconds": 270, "verdict_stream_idle_timeout_seconds": 240, "aider_request_timeout_seconds": 240, "reasoning": { "mode": "default" }, "models": "exact pins, no failover", "task_file": "solution.py", "hidden_oracle_fed_to_agent": false, "producer_location": "outside candidate workspace; candidate read denied", "aider_context": "live exact catalog context metadata; history ceiling=min(inputbudget,context/2)", "appendix_model_changed": false }, "apparatusInvalidation": "Diagnostic only: request-local initial tool narrowing skipped the caller-authorized behavioral review. Generic harness regression repaired in 1.4.564; independent scorer outcomes are retained but these timings do not establish a successful latency repair." } }