{ "schema": "agent-rerun-receipt.v1", "record": { "id": "20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 9.204, "inputTokens": 822, "outputTokens": 449, "totalTokens": 1271, "costUsd": 0.0003478, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "low", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14.json", "archived": false }, "sourceRecord": { "id": "20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14", "originalCampaign": "agent_comparison_20261015", "rerun": true, "correctionOf": null, "cohort": "current-model", "family": "same-model-harness-rerun", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "status": "PASS", "wallSeconds": 9.204, "inputTokens": 822, "outputTokens": 449, "totalTokens": 1271, "costUsd": 0.0003478, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedCodePass": true, "exitCode": 0, "evidencePaths": [ "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14" ], "arm": "E", "admission": "NOT_VERIFIED", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "freeze": { "task_sha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "oracle_sha256": "3675736119622cac168d711045426c6678bd08212b9bedf1aab71126a7ef1a76", "starter_sha256": "595a2a41e7bfd09ad28045cb3be95f0126d702eb25e6f53c723c0b77c46b44e2", "frozen_before_execution_at": 1792154967.8028212 }, "mutableRuntimeFiles": [ "budget_ledger.json", "1f3a9675bc6842848c1f00a4579c4ea6.timing.json", "1f3a9675bc6842848c1f00a4579c4ea6.http.json", "1f3a9675bc6842848c1f00a4579c4ea6.request.json", "1f3a9675bc6842848c1f00a4579c4ea6.response.txt" ], "terminalBudgetSnapshot": "budget_ledger_at_terminal.json", "terminalTransportSnapshot": "terminal_evidence/", "rawStatus": "PASS", "metadataTransportDiagnosticCount": 0, "diagnosticOnly": false, "tokenSource": "provider", "accountingComplete": true, "identityVerified": true, "providerCallCount": 1, "partialInputTokens": 822, "partialOutputTokens": 449, "partialTotalTokens": 1271, "partialEstimatedCostUsd": 0.0003478, "reportedAdditiveReasoningTokens": null, "reportedReasoningTokens": null, "allCallsReportReasoning": false, "tokensPerSecond": 138.092, "knownTokensIncludingAdditiveReasoning": 1271, "knownTokensIncludingAdditiveReasoningPerSecond": 138.092, "generationCensored": false, "budgetControls": { "schema": "agent_comparison_rerun_policy.v1", "temperature": 0, "timeout_seconds": 240, "per_request_output_tokens": 8192, "proxy_shared_max_input_tokens": 160000, "proxy_shared_max_output_tokens": 32000, "proxy_reservation": "entire request UTF8 bytes +32/message+128; providerusage settled after eachcall; failclosed ifmissing", "maximum_input_tokens": 160000, "total_token_budget": 400000, "max_turns": 24, "proxy_read_timeout_seconds": 270, "verdict_stream_idle_timeout_seconds": 240, "aider_request_timeout_seconds": 240, "reasoning": { "mode": "effort", "effort": "low" }, "models": "exact pins, no failover", "task_file": "solution.py", "hidden_oracle_fed_to_agent": false, "producer_location": "outside candidate workspace; candidate read denied", "aider_context": "live exact catalog context metadata; history ceiling=min(inputbudget,context/2)", "appendix_model_changed": false }, "reasoningEffort": "low", "sharedProxyCapsEnforced": true, "actualBelowCommonCaps": true, "timeoutSeconds": 240, "terminalError": null, "terminalCompleted": true, "agentFailureReason": null, "sameGatewayInstance": true, "comparisonEligible": true, "publicEvidencePath": "artifacts/agent_comparison_completion_repair_20261016/matrix/public_evidence/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14.json", "publicEvidenceSha256": "8186c5b05ebc86e6a893dfab9b9a4580713144830e90211271a2cc2ee62cce57", "modelSelectionEvidence": "model-selection/selection.json", "catalogEstimatedCostUsd": 0.0003478, "catalogCostEstimateSource": "model-selection/selection.json", "catalogCostEstimateFormula": "(provider inputTokens * catalog inputUsdPerMtok + provider outputTokens * catalog outputUsdPerMtok) / 1000000; no cache discounts; not a charge receipt" }, "sourceFile": "artifacts/agent_comparison_completion_repair_20261016/matrix/reviewed-summary.json", "sourceFileSha256": "cfdb4bb21c1038588f0951f2cbe4774e077c01691d7e0e53434881d3b9a16419", "files": [ { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/1f3a9675bc6842848c1f00a4579c4ea6.http.json", "sha256": "7b7d2ba7da38df349730b844f5a7ee404601546b1255c0860ee52f313dde4f73", "bytes": 138 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/1f3a9675bc6842848c1f00a4579c4ea6.request.json", "sha256": "2b86d420fb1048d892e2812c6fec009717c4cf43a82298828c9ea783ec2eb60d", "bytes": 4516 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/1f3a9675bc6842848c1f00a4579c4ea6.response.txt", "sha256": "18e5631181c6bfc88f926e000b79e68d2d2d7faf4f01f084cfb591f95fec5ff3", "bytes": 105344 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/1f3a9675bc6842848c1f00a4579c4ea6.timing.json", "sha256": "b0d851136aa14aa0b43f5c28eed166047ef7af88974e1441095da82aa56d1e21", "bytes": 306 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/budget_ledger.json", "sha256": "f2aa94d6d1c7af649b300c9c012483571123b8ee2ea16f2852c04580d5ac6a2c", "bytes": 116 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/budget_ledger_at_terminal.json", "sha256": "f2aa94d6d1c7af649b300c9c012483571123b8ee2ea16f2852c04580d5ac6a2c", "bytes": 116 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/freeze.json", "sha256": "56530b5e57c862c90599c6e6ec4a5f5a14dcb2f7b009aa493ef21443f27d67d6", "bytes": 314, "content": "{\n \"task_sha256\": \"6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe\",\n \"oracle_sha256\": \"3675736119622cac168d711045426c6678bd08212b9bedf1aab71126a7ef1a76\",\n \"starter_sha256\": \"595a2a41e7bfd09ad28045cb3be95f0126d702eb25e6f53c723c0b77c46b44e2\",\n \"frozen_before_execution_at\": 1792154967.8028212\n}", "contentRedacted": false, "publishedContentSha256": "56530b5e57c862c90599c6e6ec4a5f5a14dcb2f7b009aa493ef21443f27d67d6" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/health_after.json", "sha256": "38ffd56246f4f77867596fb41aa2995c8f0d5f4f1c7b4bdb4e396b0ee71116fd", "bytes": 1326 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/health_before.json", "sha256": "139a865b03c84746cf0b2af859b2c3cda5c8ded8bd7e7f899f2139e40b453743", "bytes": 1326 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/invocation.json", "sha256": "81b4ba0380850d60e338ca0d5c8556a79853fd7d7f1f51f76f3eea135c573237", "bytes": 1993 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/launch.json", "sha256": "ca77e3183f4eafbf3df0d735fd1edc38ae1f3c3ad9dcdf709493c8090593e7a6", "bytes": 164 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/model_preflight.json", "sha256": "28618c7d7bbe00ae9ffb0d4d7d71a3c80eb0b93a6b63c9d9a4da1ebe5c677f60", "bytes": 2622 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/oracle.stderr.txt", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes": 0, "content": "", "contentRedacted": false, "publishedContentSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/oracle.stdout.txt", "sha256": "5b22a02f39f6be766ee28b0cdfca80c3244449283a14f0786cda512f6659a6fd", "bytes": 1582, "content": "{\"passed\": 20, \"total\": 20, \"cases\": [{\"input\": \"0\", \"expected\": 0, \"actual\": 0, \"pass\": true}, {\"input\": \" 120\\t\", \"expected\": 120, \"actual\": 120, \"pass\": true}, {\"input\": \"0007\", \"expected\": 7, \"actual\": 7, \"pass\": true}, {\"input\": \"-1\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"+1\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"1.5\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"\\uff11\\uff12\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"\\u0661\\u0662\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": null, \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": 4, \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 GMT\", \"expected\": 1, \"actual\": 1, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:01 GMT\", \"expected\": 2, \"actual\": 2, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:27:00 GMT\", \"expected\": 0, \"actual\": 0, \"pass\": true}, {\"input\": \"Tue, 21 Oct 2015 07:28:00 GMT\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 UTC\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"wed, 21 Oct 2015 07:28:00 GMT\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 31 Feb 2015 07:28:00 GMT\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 GMT trailing\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 +0000\", \"expected\": null, \"actual\": null, \"pass\": true}]}\n", "contentRedacted": false, "publishedContentSha256": "5b22a02f39f6be766ee28b0cdfca80c3244449283a14f0786cda512f6659a6fd" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/policy.json", "sha256": "bd7643c61bf02425e5a686a0ff0acdfa4083976b29f6b591dd01f09c243f2fa9", "bytes": 941, "content": "{\n \"schema\": \"agent_comparison_rerun_policy.v1\",\n \"temperature\": 0,\n \"timeout_seconds\": 240,\n \"per_request_output_tokens\": 8192,\n \"proxy_shared_max_input_tokens\": 160000,\n \"proxy_shared_max_output_tokens\": 32000,\n \"proxy_reservation\": \"entire request UTF8 bytes +32/message+128; providerusage settled after eachcall; failclosed ifmissing\",\n \"maximum_input_tokens\": 160000,\n \"total_token_budget\": 400000,\n \"max_turns\": 24,\n \"proxy_read_timeout_seconds\": 270,\n \"verdict_stream_idle_timeout_seconds\": 240,\n \"aider_request_timeout_seconds\": 240,\n \"reasoning\": {\n \"mode\": \"effort\",\n \"effort\": \"low\"\n },\n \"models\": \"exact pins, no failover\",\n \"task_file\": \"solution.py\",\n \"hidden_oracle_fed_to_agent\": false,\n \"producer_location\": \"outside candidate workspace; candidate read denied\",\n \"aider_context\": \"live exact catalog context metadata; history ceiling=min(inputbudget,context/2)\",\n \"appendix_model_changed\": false\n}", "contentRedacted": false, "publishedContentSha256": "bd7643c61bf02425e5a686a0ff0acdfa4083976b29f6b591dd01f09c243f2fa9" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/post-terminal-settlement.json", "sha256": "834a97dadac5d8f4d0a95b00a1857c311bbd5c67f60c876143cdf5956a1b9c48", "bytes": 379 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/product_source.json", "sha256": "0f651a1e7d528643ea06021ec601162cc416d93bb3d0fcca10eae73f4d324b9f", "bytes": 846, "content": "{\n \"head\": \"b4be273425679718d713102505aca87f859e3369\",\n \"files\": {\n \"scripts/benchmarks/agent_comparison_rerun_20261016.py\": \"c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0\",\n \"tools/verdict-code/verdict_code/agent.py\": \"39a9d2f1bb876c8031a257b1f801b8fb4146afb1b14a5434f12906b378d9d9c0\",\n \"tools/verdict-code/verdict_code/tools/__init__.py\": \"609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57\",\n \"tools/verdict-code/verdict_code/main.py\": \"af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab\",\n \"tools/verdict-code/VERSION\": \"361f0472cb741b7864ae144e7abdf93738d129230d117f20c103e1d8ae7bef7c\",\n \"scripts/benchmarks/agent_comparison_completion_repair_20261016.py\": \"827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa\"\n },\n \"retainedSourceBytes\": \"source_snapshot/\"\n}", "contentRedacted": false, "publishedContentSha256": "0f651a1e7d528643ea06021ec601162cc416d93bb3d0fcca10eae73f4d324b9f" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/prompt.txt", "sha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "bytes": 837, "content": "Synthetic benchmark task: Retry-After parsing. Implement `retry_after(value, now)` in solution.py using only Python standard library. now is an aware UTC datetime. Return a nonnegative integer number of seconds or None for invalid input. Strings containing only ASCII decimal digits after surrounding whitespace return that integer. Otherwise accept an RFC HTTP-date of form `Wed, 21 Oct 2015 07:28:00 GMT`, case sensitive, validated weekday must agree with date. For a valid HTTP date return max(0, ceil((date-now).total_seconds())). Reject non-string inputs, signs, fractions, non-ASCII digits, extra text, and non-GMT timezone names. Do not change any other source file. Hidden tests are unavailable. Write the implementation to disk and finish. Do not install dependencies, access network, use other models, or edit unrelated paths.\n", "contentRedacted": false, "publishedContentSha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/reasoning_preflight.json", "sha256": "bf989a20ecee480e99a526bd9bbcec4a110bd3148371f1b6ed6010d2f2ae1926", "bytes": 244 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/result.json", "sha256": "abdc420b9251a4c5a6547790c110f2b240aa63882a8cbe4bd947a509d50aab28", "bytes": 5392 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/sandbox.sb", "sha256": "d1ca1533d90714dd2ff91d3ff013774f18c6d5a820aecb4bc9e5ffcb90615c9f", "bytes": 5672 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/sandbox_preflight.json", "sha256": "8415d156075475eba2b82a04636f833e85484020de5e6424f6140cd0ac61390f", "bytes": 93 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/source_snapshot/scripts/benchmarks/agent_comparison_completion_repair_20261016.py", "sha256": "827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa", "bytes": 13274 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/source_snapshot/scripts/benchmarks/agent_comparison_rerun_20261016.py", "sha256": "c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0", "bytes": 38332 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/source_snapshot/tools/verdict-code/VERSION", "sha256": "361f0472cb741b7864ae144e7abdf93738d129230d117f20c103e1d8ae7bef7c", "bytes": 8 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/source_snapshot/tools/verdict-code/verdict_code/agent.py", "sha256": "39a9d2f1bb876c8031a257b1f801b8fb4146afb1b14a5434f12906b378d9d9c0", "bytes": 764703 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/source_snapshot/tools/verdict-code/verdict_code/main.py", "sha256": "af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab", "bytes": 625279 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/source_snapshot/tools/verdict-code/verdict_code/tools/__init__.py", "sha256": "609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57", "bytes": 269877 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/stderr.txt", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes": 0 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/stdout.txt", "sha256": "34ca41991dad3f3dd5aa2cee9fe5f3cde44db609dc3e7f5690a7a6adb998d241", "bytes": 1744 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/terminal_evidence/1f3a9675bc6842848c1f00a4579c4ea6.http.json", "sha256": "7b7d2ba7da38df349730b844f5a7ee404601546b1255c0860ee52f313dde4f73", "bytes": 138 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/terminal_evidence/1f3a9675bc6842848c1f00a4579c4ea6.request.json", "sha256": "2b86d420fb1048d892e2812c6fec009717c4cf43a82298828c9ea783ec2eb60d", "bytes": 4516 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/terminal_evidence/1f3a9675bc6842848c1f00a4579c4ea6.response.txt", "sha256": "18e5631181c6bfc88f926e000b79e68d2d2d7faf4f01f084cfb591f95fec5ff3", "bytes": 105344 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/terminal_evidence/1f3a9675bc6842848c1f00a4579c4ea6.timing.json", "sha256": "b0d851136aa14aa0b43f5c28eed166047ef7af88974e1441095da82aa56d1e21", "bytes": 306 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/terminal_evidence/budget_ledger.json", "sha256": "f2aa94d6d1c7af649b300c9c012483571123b8ee2ea16f2852c04580d5ac6a2c", "bytes": 116 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/workspace/aider_model_metadata.json", "sha256": "65fdb697b19fba7c305e50470dd2ab65675867ecfd638667e5872bdc1bb8b945", "bytes": 188 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/workspace/aider_settings.yml", "sha256": "c9900354c4edebb24c97ebb443d912410f378725bb9d1d7224555a90706f71a8", "bytes": 196 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/workspace/chat_history.md", "sha256": "2ef74b0ab8baf75e5447c1dc07d4112ddccb7a08fff88471866b37ca3788ea25", "bytes": 3870 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/workspace/solution.py", "sha256": "86a7efc09ad9f1b7dee86616ccc557894cd4df0241acb0247669ab1b9a3fb86a", "bytes": 1330, "content": "import math\nimport re\nfrom datetime import datetime, timezone\n\n_HTTP_DATE_RE = re.compile(\n r\"^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), (\\d{2}) \"\n r\"(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\\d{4}) \"\n r\"(\\d{2}):(\\d{2}):(\\d{2}) GMT$\"\n)\n\n_MONTHS = {\n \"Jan\": 1, \"Feb\": 2, \"Mar\": 3, \"Apr\": 4, \"May\": 5, \"Jun\": 6,\n \"Jul\": 7, \"Aug\": 8, \"Sep\": 9, \"Oct\": 10, \"Nov\": 11, \"Dec\": 12,\n}\n\n_DAYS = [\"Mon\", \"Tue\", \"Wed\", \"Thu\", \"Fri\", \"Sat\", \"Sun\"]\n\n\ndef retry_after(value, now):\n \"\"\"Parse an HTTP Retry-After header according to the task contract.\"\"\"\n if not isinstance(value, str):\n return None\n\n stripped = value.strip()\n if not stripped:\n return None\n\n # ASCII decimal digits only\n if re.fullmatch(r\"[0-9]+\", stripped):\n return int(stripped)\n\n # HTTP-date form\n m = _HTTP_DATE_RE.match(value)\n if not m:\n return None\n\n weekday, day, month, year, hh, mm, ss = (\n m.group(1), int(m.group(2)), m.group(3), int(m.group(4)),\n int(m.group(5)), int(m.group(6)), int(m.group(7)),\n )\n\n try:\n date = datetime(year, _MONTHS[month], day, hh, mm, ss, tzinfo=timezone.utc)\n except ValueError:\n return None\n\n if _DAYS[date.weekday()] != weekday:\n return None\n\n delta = (date - now).total_seconds()\n return max(0, math.ceil(delta))\n", "contentRedacted": false, "publishedContentSha256": "86a7efc09ad9f1b7dee86616ccc557894cd4df0241acb0247669ab1b9a3fb86a" } ], "disclosure": "Synthetic inputs and observed execution. Original-byte hashes retained; selected source and oracle outputs reproduced. Private provider transcripts remain local.", "reviewedEvidence": { "provenance": "Synthetic test input; actual installed CLI execution; reviewed public-safe evidence fixture.", "runId": "20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14", "task": "Synthetic benchmark task: Retry-After parsing. Implement `retry_after(value, now)` in solution.py using only Python standard library. now is an aware UTC datetime. Return a nonnegative integer number of seconds or None for invalid input. Strings containing only ASCII decimal digits after surrounding whitespace return that integer. Otherwise accept an RFC HTTP-date of form `Wed, 21 Oct 2015 07:28:00 GMT`, case sensitive, validated weekday must agree with date. For a valid HTTP date return max(0, ceil((date-now).total_seconds())). Reject non-string inputs, signs, fractions, non-ASCII digits, extra text, and non-GMT timezone names. Do not change any other source file. Hidden tests are unavailable. Write the implementation to disk and finish. Do not install dependencies, access network, use other models, or edit unrelated paths.\n", "freeze": { "task_sha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "oracle_sha256": "3675736119622cac168d711045426c6678bd08212b9bedf1aab71126a7ef1a76", "starter_sha256": "595a2a41e7bfd09ad28045cb3be95f0126d702eb25e6f53c723c0b77c46b44e2", "frozen_before_execution_at": 1792154967.8028212 }, "candidateCode": "import math\nimport re\nfrom datetime import datetime, timezone\n\n_HTTP_DATE_RE = re.compile(\n r\"^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), (\\d{2}) \"\n r\"(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\\d{4}) \"\n r\"(\\d{2}):(\\d{2}):(\\d{2}) GMT$\"\n)\n\n_MONTHS = {\n \"Jan\": 1, \"Feb\": 2, \"Mar\": 3, \"Apr\": 4, \"May\": 5, \"Jun\": 6,\n \"Jul\": 7, \"Aug\": 8, \"Sep\": 9, \"Oct\": 10, \"Nov\": 11, \"Dec\": 12,\n}\n\n_DAYS = [\"Mon\", \"Tue\", \"Wed\", \"Thu\", \"Fri\", \"Sat\", \"Sun\"]\n\n\ndef retry_after(value, now):\n \"\"\"Parse an HTTP Retry-After header according to the task contract.\"\"\"\n if not isinstance(value, str):\n return None\n\n stripped = value.strip()\n if not stripped:\n return None\n\n # ASCII decimal digits only\n if re.fullmatch(r\"[0-9]+\", stripped):\n return int(stripped)\n\n # HTTP-date form\n m = _HTTP_DATE_RE.match(value)\n if not m:\n return None\n\n weekday, day, month, year, hh, mm, ss = (\n m.group(1), int(m.group(2)), m.group(3), int(m.group(4)),\n int(m.group(5)), int(m.group(6)), int(m.group(7)),\n )\n\n try:\n date = datetime(year, _MONTHS[month], day, hh, mm, ss, tzinfo=timezone.utc)\n except ValueError:\n return None\n\n if _DAYS[date.weekday()] != weekday:\n return None\n\n delta = (date - now).total_seconds()\n return max(0, math.ceil(delta))\n", "oracle": { "passed": 20, "total": 20, "cases": [ { "input": "0", "expected": 0, "actual": 0, "pass": true }, { "input": " 120\t", "expected": 120, "actual": 120, "pass": true }, { "input": "0007", "expected": 7, "actual": 7, "pass": true }, { "input": "-1", "expected": null, "actual": null, "pass": true }, { "input": "+1", "expected": null, "actual": null, "pass": true }, { "input": "1.5", "expected": null, "actual": null, "pass": true }, { "input": "１２", "expected": null, "actual": null, "pass": true }, { "input": "١٢", "expected": null, "actual": null, "pass": true }, { "input": "", "expected": null, "actual": null, "pass": true }, { "input": null, "expected": null, "actual": null, "pass": true }, { "input": 4, "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 GMT", "expected": 1, "actual": 1, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:01 GMT", "expected": 2, "actual": 2, "pass": true }, { "input": "Wed, 21 Oct 2015 07:27:00 GMT", "expected": 0, "actual": 0, "pass": true }, { "input": "Tue, 21 Oct 2015 07:28:00 GMT", "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 UTC", "expected": null, "actual": null, "pass": true }, { "input": "wed, 21 Oct 2015 07:28:00 GMT", "expected": null, "actual": null, "pass": true }, { "input": "Wed, 31 Feb 2015 07:28:00 GMT", "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 GMT trailing", "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 +0000", "expected": null, "actual": null, "pass": true } ] }, "providerCalls": [ { "requestEvidence": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14/1f3a9675bc6842848c1f00a4579c4ea6.request.json", "requestSha256": "2b86d420fb1048d892e2812c6fec009717c4cf43a82298828c9ea783ec2eb60d", "responseSha256": "18e5631181c6bfc88f926e000b79e68d2d2d7faf4f01f084cfb591f95fec5ff3", "httpStatus": 200, "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "backendModel": "zai/glm-5.3-flash", "resolutionSource": "explicit_byok_route", "generationControls": { "max_tokens": 8192, "temperature": 0, "reasoning_choice": { "mode": "effort", "effort": "low" }, "auto_failover": false, "stream": true }, "inputTokens": 822, "outputTokens": 449, "totalTokens": 1271, "costUsd": 0.0003478, "energyWh": null, "finishReasons": [ "stop" ], "additiveReasoningTokens": null, "nestedReasoningTokens": null } ], "gatewayBefore": { "status": "healthy", "service": "local_gateway", "mode": "full", "port": 6120, "timestamp": "2026-10-16T12:49:28.277905+00:00", "uptime_seconds": 977.5889370441437, "server_start_time": 1792153990.688971, "git_sha": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "build_time": "2026-10-16T08:32:43-04:00", "version": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "interop_min": 1, "interop_max": 1, "dependencies": { "event_stream": "http://localhost:6102" }, "lightrag": { "enabled": true, "status": "online", "files_indexed": 8020, "url": "http://127.0.0.1:6150" }, "cloud_relay": { "connected": true, "authenticated": true }, "context_status": { "summary": "ok", "target_context": 65536, "providers": [ { "provider": "ollama", "status": "ok", "actual_context": 131072, "target_context": 65536, "model": "deepseek-r1:7b" }, { "provider": "lmstudio", "status": "ok", "actual_context": 262144, "target_context": 65536, "model": "verdict-4-35b-toolterse-v5-mlx" }, { "provider": "llamacpp", "status": "context_unknown", "actual_context": null, "target_context": 65536, "model": null } ] } }, "gatewayAfter": { "status": "healthy", "service": "local_gateway", "mode": "full", "port": 6120, "timestamp": "2026-10-16T12:49:37.741412+00:00", "uptime_seconds": 987.0524480342865, "server_start_time": 1792153990.688971, "git_sha": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "build_time": "2026-10-16T08:32:43-04:00", "version": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "interop_min": 1, "interop_max": 1, "dependencies": { "event_stream": "http://localhost:6102" }, "lightrag": { "enabled": true, "status": "online", "files_indexed": 8020, "url": "http://127.0.0.1:6150" }, "cloud_relay": { "connected": true, "authenticated": true }, "context_status": { "summary": "ok", "target_context": 65536, "providers": [ { "provider": "ollama", "status": "ok", "actual_context": 131072, "target_context": 65536, "model": "deepseek-r1:7b" }, { "provider": "lmstudio", "status": "ok", "actual_context": 262144, "target_context": 65536, "model": "verdict-4-35b-toolterse-v5-mlx" }, { "provider": "llamacpp", "status": "context_unknown", "actual_context": null, "target_context": 65536, "model": null } ] } }, "rawArtifacts": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14", "rawResultSha256": "abdc420b9251a4c5a6547790c110f2b240aa63882a8cbe4bd947a509d50aab28", "rawEvidenceHashes": { "health_before.json": "139a865b03c84746cf0b2af859b2c3cda5c8ded8bd7e7f899f2139e40b453743", "stdout.txt": "34ca41991dad3f3dd5aa2cee9fe5f3cde44db609dc3e7f5690a7a6adb998d241", "policy.json": "bd7643c61bf02425e5a686a0ff0acdfa4083976b29f6b591dd01f09c243f2fa9", "health_after.json": "38ffd56246f4f77867596fb41aa2995c8f0d5f4f1c7b4bdb4e396b0ee71116fd", "model_preflight.json": "28618c7d7bbe00ae9ffb0d4d7d71a3c80eb0b93a6b63c9d9a4da1ebe5c677f60", "sandbox_preflight.json": "8415d156075475eba2b82a04636f833e85484020de5e6424f6140cd0ac61390f", "budget_ledger_at_terminal.json": "f2aa94d6d1c7af649b300c9c012483571123b8ee2ea16f2852c04580d5ac6a2c", "prompt.txt": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "oracle.stdout.txt": "5b22a02f39f6be766ee28b0cdfca80c3244449283a14f0786cda512f6659a6fd", "sandbox.sb": "d1ca1533d90714dd2ff91d3ff013774f18c6d5a820aecb4bc9e5ffcb90615c9f", "stderr.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "reasoning_preflight.json": "bf989a20ecee480e99a526bd9bbcec4a110bd3148371f1b6ed6010d2f2ae1926", "launch.json": "ca77e3183f4eafbf3df0d735fd1edc38ae1f3c3ad9dcdf709493c8090593e7a6", "invocation.json": "81b4ba0380850d60e338ca0d5c8556a79853fd7d7f1f51f76f3eea135c573237", "oracle.stderr.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "freeze.json": "56530b5e57c862c90599c6e6ec4a5f5a14dcb2f7b009aa493ef21443f27d67d6", "product_source.json": "0f651a1e7d528643ea06021ec601162cc416d93bb3d0fcca10eae73f4d324b9f", "post-terminal-settlement.json": "834a97dadac5d8f4d0a95b00a1857c311bbd5c67f60c876143cdf5956a1b9c48", "workspace/aider_model_metadata.json": "65fdb697b19fba7c305e50470dd2ab65675867ecfd638667e5872bdc1bb8b945", "workspace/solution.py": "86a7efc09ad9f1b7dee86616ccc557894cd4df0241acb0247669ab1b9a3fb86a", "workspace/chat_history.md": "2ef74b0ab8baf75e5447c1dc07d4112ddccb7a08fff88471866b37ca3788ea25", "workspace/aider_settings.yml": "c9900354c4edebb24c97ebb443d912410f378725bb9d1d7224555a90706f71a8", "terminal_evidence/1f3a9675bc6842848c1f00a4579c4ea6.timing.json": "b0d851136aa14aa0b43f5c28eed166047ef7af88974e1441095da82aa56d1e21", "terminal_evidence/1f3a9675bc6842848c1f00a4579c4ea6.http.json": "7b7d2ba7da38df349730b844f5a7ee404601546b1255c0860ee52f313dde4f73", "terminal_evidence/1f3a9675bc6842848c1f00a4579c4ea6.request.json": "2b86d420fb1048d892e2812c6fec009717c4cf43a82298828c9ea783ec2eb60d", "terminal_evidence/1f3a9675bc6842848c1f00a4579c4ea6.response.txt": "18e5631181c6bfc88f926e000b79e68d2d2d7faf4f01f084cfb591f95fec5ff3", "terminal_evidence/budget_ledger.json": "f2aa94d6d1c7af649b300c9c012483571123b8ee2ea16f2852c04580d5ac6a2c", "source_snapshot/tools/verdict-code/VERSION": "361f0472cb741b7864ae144e7abdf93738d129230d117f20c103e1d8ae7bef7c", "source_snapshot/tools/verdict-code/verdict_code/agent.py": "39a9d2f1bb876c8031a257b1f801b8fb4146afb1b14a5434f12906b378d9d9c0", "source_snapshot/tools/verdict-code/verdict_code/main.py": "af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab", "source_snapshot/tools/verdict-code/verdict_code/tools/__init__.py": "609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57", "source_snapshot/scripts/benchmarks/agent_comparison_rerun_20261016.py": "c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0", "source_snapshot/scripts/benchmarks/agent_comparison_completion_repair_20261016.py": "827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa" }, "productSource": { "head": "b4be273425679718d713102505aca87f859e3369", "files": { "scripts/benchmarks/agent_comparison_rerun_20261016.py": "c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0", "tools/verdict-code/verdict_code/agent.py": "39a9d2f1bb876c8031a257b1f801b8fb4146afb1b14a5434f12906b378d9d9c0", "tools/verdict-code/verdict_code/tools/__init__.py": "609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57", "tools/verdict-code/verdict_code/main.py": "af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab", "tools/verdict-code/VERSION": "361f0472cb741b7864ae144e7abdf93738d129230d117f20c103e1d8ae7bef7c", "scripts/benchmarks/agent_comparison_completion_repair_20261016.py": "827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa" }, "retainedSourceBytes": "source_snapshot/" }, "protocol": { "schema": "agent_comparison_rerun_policy.v1", "temperature": 0, "timeout_seconds": 240, "per_request_output_tokens": 8192, "proxy_shared_max_input_tokens": 160000, "proxy_shared_max_output_tokens": 32000, "proxy_reservation": "entire request UTF8 bytes +32/message+128; providerusage settled after eachcall; failclosed ifmissing", "maximum_input_tokens": 160000, "total_token_budget": 400000, "max_turns": 24, "proxy_read_timeout_seconds": 270, "verdict_stream_idle_timeout_seconds": 240, "aider_request_timeout_seconds": 240, "reasoning": { "mode": "effort", "effort": "low" }, "models": "exact pins, no failover", "task_file": "solution.py", "hidden_oracle_fed_to_agent": false, "producer_location": "outside candidate workspace; candidate read denied", "aider_context": "live exact catalog context metadata; history ceiling=min(inputbudget,context/2)", "appendix_model_changed": false }, "catalogCostEstimate": { "estimatedUsd": 0.0003478, "inputUsdPerMtok": 0.15, "outputUsdPerMtok": 0.5, "source": "model-selection/selection.json", "formula": "(provider inputTokens * catalog inputUsdPerMtok + provider outputTokens * catalog outputUsdPerMtok) / 1000000; no cache discounts; not a charge receipt" } } }