{ "schema": "agent-comparison-completion-repair.v1", "campaign": "agent-comparison-completion-repair-20261016", "sourceSha": "75fb1c7a5473549983c16143e3cb15482c940d97", "generatedAt": "2026-10-16T13:22:22.547358+00:00", "state": "Completion and latency repair; other task families retain earlier observations.", "protocol": [ "Unchanged frozen parser prompt, starter and hidden oracle; both actual CLIs. The user replaced the obsolete roster with five current inexpensive models, selected from live ladders and catalog added dates; historical runs remain in audit history. Every historical campaign and every new attempt remains available.", "Both agents request temperature zero and retain fixed 240-second, per-request and cumulative token limits. GPT-6 Luna uses Responses with the same provider-default reasoning in both arms; its unsupported custom sampling is omitted upstream, as required by the vendor. No silent model substitution or hidden-check feedback.", "Generic product changes remove redundant read and completion turns, bound repeated self-repair and provide an explicit text tool protocol for models without native calls. Protocol differences are disclosed in receipts.", "The independent hidden scorer runs only after candidate execution. It supplies no examples, diagnostics or expected outputs to candidates or repair decisions.", "Latest scheduled attempt is displayed even if worse or incomplete; no best-of selection. Causal apparatus retries remain in history.", "Other coding and Blueprint families retain prior measured receipts; they were not rerun in this parser campaign.", "Synthetic single-task fixtures, shared hardware and one trial per arm. No statistical or general superiority claim; canonical benchmark-store admission remains unverified." ], "conclusions": [ "The current comparison uses five exact inexpensive models chosen from the live cost, quality and speed ladders and catalog added dates. Earlier model rosters remain only in audit history.", "Haiku 5.5 passed 20/20 in 13.412 s with Verdict Code and 14.012 s with Aider. GPT-6 Luna passed 20/20 in 30.985 s and 30.003 s respectively. These are single observations on one synthetic parser task.", "GLM-5.3-FlashX passed both arms, but Verdict Code took 73.742 s versus 40.713 s. The latest GLM-5.3-Flash run finished in 30.955 s but failed 2 of 20 checks, versus Aider passing 20/20 in 9.204 s. The worse latest result remains primary.", "Qwen 3.8 Flash remains incomplete at the original four-minute limit, despite a partial artifact passing 20/20. Aider completed in 220.804 s. Earlier timeout and stale-credential interruption remain retained; a correct partial artifact is not a completed pass.", "Generic repairs address redundant provider turns, cumulative repair bounds, stale self-check evidence, lost workspace locations, contaminated check imports, and required Responses compatibility. Hidden checks were not supplied to candidate execution or used to tune repairs.", "All original publications remain byte-preserved. Other native-agent, Blueprint and coding-task families are retained prior observations, not fresh reruns. Incomplete accounting stays unknown; these results do not establish a general agent ranking." ], "rows": [ { "id": "20261016T122718Z_aider_byok_openai_gpt-4.1-mini_c41098", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": "byok/openai/gpt-4.1-mini", "wallSeconds": 10.293, "inputTokens": 831, "outputTokens": 583, "totalTokens": 1414, "costUsd": 0.001265, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T122718Z_aider_byok_openai_gpt-4.1-mini_c41098.json", "archived": true }, { "id": "20261016T122849Z_aider_byok_deepseek_deepseek-flash_0893d1", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/deepseek/deepseek-flash", "resolvedModel": "byok/deepseek/deepseek-flash", "wallSeconds": 48.921, "inputTokens": 847, "outputTokens": 8192, "totalTokens": 9039, "costUsd": 0.010084, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Provider output reached the fixed per-request ceiling; censored attempt, not a task-quality failure.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T122849Z_aider_byok_deepseek_deepseek-flash_0893d1.json", "archived": true }, { "id": "20261016T122849Z_aider_byok_qwen_qwen3-coder-flash_0cda6e", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/qwen/qwen3-coder-flash", "resolvedModel": "byok/qwen/qwen3-coder-flash", "wallSeconds": 11.421, "inputTokens": 847, "outputTokens": 529, "totalTokens": 1376, "costUsd": 0.0010475999999999999, "energyWh": null, "assertionsPassed": 16, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T122849Z_aider_byok_qwen_qwen3-coder-flash_0cda6e.json", "archived": true }, { "id": "20261016T122849Z_aider_byok_zai_glm-5.3-flash_417eeb", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 8.01, "inputTokens": 822, "outputTokens": 417, "totalTokens": 1239, "costUsd": 0.0003318, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "low", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T122849Z_aider_byok_zai_glm-5.3-flash_417eeb.json", "archived": false, "supersededBy": "20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14" }, { "id": "20261016T122858Z_aider_byok_google_models_gemini-2.5-flash_5a87e5", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/google/models/gemini-2.5-flash", "resolvedModel": "byok/google/models/gemini-2.5-flash", "wallSeconds": 18.183, "inputTokens": 870, "outputTokens": 3408, "totalTokens": 4278, "costUsd": 0.008781, "energyWh": null, "assertionsPassed": 16, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T122858Z_aider_byok_google_models_gemini-2.5-flash_5a87e5.json", "archived": true }, { "id": "20261016T122901Z_aider_byok_qwen_qwen3.8-flash_c9d466", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": null, "wallSeconds": 240.012, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "TIMEOUT", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "current-model", "resultNote": "Process reached the fixed time limit without completing.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Gateway instance continuity was not proven; excluded from matched-model comparison.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T122901Z_aider_byok_qwen_qwen3.8-flash_c9d466.json", "archived": false, "supersededBy": "20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683" }, { "id": "20261016T122917Z_aider_byok_google_models_gemini-3.8-flash_43aead", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/google/models/gemini-3.8-flash", "resolvedModel": "byok/google/models/gemini-3.8-flash", "wallSeconds": 55.042, "inputTokens": 870, "outputTokens": 8188, "totalTokens": 9058, "costUsd": 0.0313575, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Provider output reached the fixed per-request ceiling; censored attempt, not a task-quality failure.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T122917Z_aider_byok_google_models_gemini-3.8-flash_43aead.json", "archived": true }, { "id": "20261016T123139Z_verdict-code_byok_openai_gpt-4.1-mini_154f5e", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": null, "wallSeconds": 57.415, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "A metadata probe disconnected; inference is assessed separately from captured POST completion and accounting.", "Gateway instance continuity was not proven; excluded from matched-model comparison.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T123139Z_verdict-code_byok_openai_gpt-4.1-mini_154f5e.json", "archived": true, "supersededBy": "20261016T123350Z_verdict-code_byok_openai_gpt-4.1-mini_9b3037" }, { "id": "20261016T123350Z_verdict-code_byok_openai_gpt-4.1-mini_9b3037", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": "byok/openai/gpt-4.1-mini", "wallSeconds": 75.334, "inputTokens": 27387, "outputTokens": 6957, "totalTokens": 34344, "costUsd": 0.022085999999999998, "energyWh": null, "assertionsPassed": 15, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T123350Z_verdict-code_byok_openai_gpt-4.1-mini_9b3037.json", "archived": true, "supersededBy": "20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40" }, { "id": "20261016T123447Z_aider_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_248d97", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "resolvedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "wallSeconds": 7.141, "inputTokens": 886, "outputTokens": 438, "totalTokens": 1324, "costUsd": 0.0, "energyWh": 0.257444, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T123447Z_aider_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_248d97.json", "archived": true }, { "id": "20261016T123456Z_aider_local_lmstudio_qwen_qwen3-coder-30b_95980d", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/qwen/qwen3-coder-30b", "resolvedModel": "local/lmstudio/qwen/qwen3-coder-30b", "wallSeconds": 8.596, "inputTokens": 847, "outputTokens": 574, "totalTokens": 1421, "costUsd": 0.0, "energyWh": 0.276306, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T123456Z_aider_local_lmstudio_qwen_qwen3-coder-30b_95980d.json", "archived": true }, { "id": "20261016T123506Z_aider_local_lmstudio_openai_gpt-oss-20b_55fecb", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/openai/gpt-oss-20b", "resolvedModel": "local/lmstudio/openai/gpt-oss-20b", "wallSeconds": 25.471, "inputTokens": 900, "outputTokens": 340, "totalTokens": 1240, "costUsd": 0.0, "energyWh": 0.241111, "assertionsPassed": 16, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T123506Z_aider_local_lmstudio_openai_gpt-oss-20b_55fecb.json", "archived": true }, { "id": "20261016T123533Z_aider_local_lmstudio_google_gemma-4-26b-a4b_9410a4", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/google/gemma-4-26b-a4b", "resolvedModel": null, "wallSeconds": 240.091, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "TIMEOUT", "tier": "local/LAN", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "historical-model", "resultNote": "Four-minute limit reached; no completed result.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T123533Z_aider_local_lmstudio_google_gemma-4-26b-a4b_9410a4.json", "archived": true }, { "id": "20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": "byok/qwen/qwen3.8-flash", "wallSeconds": 192.155, "inputTokens": 934, "outputTokens": 10861, "totalTokens": 11795, "costUsd": 0.00524477, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683.json", "archived": false, "supersededBy": "20261016T124951Z_aider_byok_qwen_qwen3.8-flash_d64870" }, { "id": "20261016T123719Z_verdict-code_byok_deepseek_deepseek-flash_0e44de", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/deepseek/deepseek-flash", "resolvedModel": "byok/deepseek/deepseek-flash", "wallSeconds": 18.714, "inputTokens": 1253, "outputTokens": 2423, "totalTokens": 3676, "costUsd": 0.003283, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Diagnostic only: request-local initial tool narrowing skipped the caller-authorized behavioral review. Generic harness regression repaired in 1.4.564; independent scorer outcomes are retained but these timings do not establish a successful latency repair." ], "evidenceFile": "20261016T123719Z_verdict-code_byok_deepseek_deepseek-flash_0e44de.json", "archived": true }, { "id": "20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": "byok/openai/gpt-4.1-mini", "wallSeconds": 11.536, "inputTokens": 915, "outputTokens": 478, "totalTokens": 1393, "costUsd": 0.001131, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Diagnostic only: request-local initial tool narrowing skipped the caller-authorized behavioral review. Generic harness regression repaired in 1.4.564; independent scorer outcomes are retained but these timings do not establish a successful latency repair." ], "evidenceFile": "20261016T123719Z_verdict-code_byok_openai_gpt-4.1-mini_123c40.json", "archived": true }, { "id": "20261016T123719Z_verdict-code_byok_zai_glm-5.3-flash_0f8542", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 23.55, "inputTokens": 1102, "outputTokens": 425, "totalTokens": 1527, "costUsd": 0.00037779999999999997, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "low", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Diagnostic only: request-local initial tool narrowing skipped the caller-authorized behavioral review. Generic harness regression repaired in 1.4.564; independent scorer outcomes are retained but these timings do not establish a successful latency repair." ], "evidenceFile": "20261016T123719Z_verdict-code_byok_zai_glm-5.3-flash_0f8542.json", "archived": false, "supersededBy": "20261016T124951Z_verdict-code_byok_zai_glm-5.3-flash_58dcd7" }, { "id": "20261016T123732Z_verdict-code_byok_qwen_qwen3-coder-flash_62aae4", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/qwen/qwen3-coder-flash", "resolvedModel": "byok/qwen/qwen3-coder-flash", "wallSeconds": 19.527, "inputTokens": 3203, "outputTokens": 1058, "totalTokens": 4261, "costUsd": 0.0025478999999999996, "energyWh": null, "assertionsPassed": 16, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Diagnostic only: request-local initial tool narrowing skipped the caller-authorized behavioral review. Generic harness regression repaired in 1.4.564; independent scorer outcomes are retained but these timings do not establish a successful latency repair." ], "evidenceFile": "20261016T123732Z_verdict-code_byok_qwen_qwen3-coder-flash_62aae4.json", "archived": true }, { "id": "20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/google/models/gemini-2.5-flash", "resolvedModel": "byok/google/models/gemini-2.5-flash", "wallSeconds": 19.476, "inputTokens": 992, "outputTokens": 2706, "totalTokens": 3698, "costUsd": 0.0070626000000000005, "energyWh": null, "assertionsPassed": 16, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "historical-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Diagnostic only: request-local initial tool narrowing skipped the caller-authorized behavioral review. Generic harness regression repaired in 1.4.564; independent scorer outcomes are retained but these timings do not establish a successful latency repair." ], "evidenceFile": "20261016T123739Z_verdict-code_byok_google_models_gemini-2.5-flash_049660.json", "archived": true }, { "id": "20261016T123745Z_verdict-code_byok_qwen_qwen3.8-flash_b014b3", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": "byok/qwen/qwen3.8-flash", "wallSeconds": 59.14, "inputTokens": 1280, "outputTokens": 3664, "totalTokens": 4944, "costUsd": 0.0019140799999999999, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Diagnostic only: request-local initial tool narrowing skipped the caller-authorized behavioral review. Generic harness regression repaired in 1.4.564; independent scorer outcomes are retained but these timings do not establish a successful latency repair." ], "evidenceFile": "20261016T123745Z_verdict-code_byok_qwen_qwen3.8-flash_b014b3.json", "archived": false, "supersededBy": "20261016T124951Z_verdict-code_byok_qwen_qwen3.8-flash_029acf" }, { "id": "20261016T123753Z_verdict-code_byok_google_models_gemini-3.8-flash_30a6fd", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/google/models/gemini-3.8-flash", "resolvedModel": null, "wallSeconds": null, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "historical-model", "resultNote": "Interrupted when the user stopped the old model roster. Wall time and completion are unknown; hidden scoring was not run.", "limitations": [ "Interrupted old-roster attempt. No terminal completion, wall measurement or independent scoring claimed. Captured token subtotals, if any, are lower bounds only.", "Provider output reached the fixed per-request ceiling; censored attempt, not a task-quality failure.", "Gateway instance continuity was not proven; excluded from matched-model comparison.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Diagnostic only: request-local initial tool narrowing skipped the caller-authorized behavioral review. Generic harness regression repaired in 1.4.564; independent scorer outcomes are retained but these timings do not establish a successful latency repair." ], "evidenceFile": "20261016T123753Z_verdict-code_byok_google_models_gemini-3.8-flash_30a6fd.json", "archived": true }, { "id": "20261016T124009Z_aider_local_lmstudio_qwen_qwen2.5-coder-1.5b-instruct_0f192d", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/qwen/qwen2.5-coder-1.5b-instruct", "resolvedModel": null, "wallSeconds": null, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "local/LAN", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "historical-model", "resultNote": "Interrupted when the user stopped the old model roster. Wall time and completion are unknown; hidden scoring was not run.", "limitations": [ "Interrupted old-roster attempt. No terminal completion, wall measurement or independent scoring claimed. Captured token subtotals, if any, are lower bounds only.", "Gateway instance continuity was not proven; excluded from matched-model comparison.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T124009Z_aider_local_lmstudio_qwen_qwen2.5-coder-1.5b-instruct_0f192d.json", "archived": true }, { "id": "20261016T124645Z_verdict-code_byok_anthropic_claude-haiku-5-5_677cef", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/anthropic/claude-haiku-5-5", "resolvedModel": "byok/anthropic/claude-haiku-5-5", "wallSeconds": 205.045, "inputTokens": 24556, "outputTokens": 4490, "totalTokens": 29046, "costUsd": null, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Provider cost estimation returned zero despite positive usage and catalog prices. Cost is unavailable; the separately labelled catalog estimate is not an observed charge." ], "evidenceFile": "20261016T124645Z_verdict-code_byok_anthropic_claude-haiku-5-5_677cef.json", "archived": false, "supersededBy": "20261016T125355Z_verdict-code_byok_anthropic_claude-haiku-5-5_62d411" }, { "id": "20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 9.204, "inputTokens": 822, "outputTokens": 449, "totalTokens": 1271, "costUsd": 0.0003478, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "low", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T124926Z_aider_byok_zai_glm-5.3-flash_170c14.json", "archived": false }, { "id": "20261016T124951Z_aider_byok_anthropic_claude-haiku-5-5_7731ed", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/anthropic/claude-haiku-5-5", "resolvedModel": "byok/anthropic/claude-haiku-5-5", "wallSeconds": 14.012, "inputTokens": 1241, "outputTokens": 2617, "totalTokens": 3858, "costUsd": null, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Provider cost estimation returned zero despite positive usage and catalog prices. Cost is unavailable; the separately labelled catalog estimate is not an observed charge." ], "evidenceFile": "20261016T124951Z_aider_byok_anthropic_claude-haiku-5-5_7731ed.json", "archived": false }, { "id": "20261016T124951Z_aider_byok_openai_gpt-6-luna_fd7473", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/openai/gpt-6-luna", "resolvedModel": "byok/openai/gpt-6-luna", "wallSeconds": 21.078, "inputTokens": 836, "outputTokens": 1497, "totalTokens": 2333, "costUsd": 0.000832, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T124951Z_aider_byok_openai_gpt-6-luna_fd7473.json", "archived": false, "supersededBy": "20261016T130549Z_aider_byok_openai_gpt-6-luna_563ba0" }, { "id": "20261016T124951Z_aider_byok_qwen_qwen3.8-flash_d64870", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": "byok/qwen/qwen3.8-flash", "wallSeconds": 220.804, "inputTokens": 934, "outputTokens": 10463, "totalTokens": 11397, "costUsd": 0.00505771, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T124951Z_aider_byok_qwen_qwen3.8-flash_d64870.json", "archived": false }, { "id": "20261016T124951Z_aider_byok_zai_glm-5.3-flashx_ffee9b", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/zai/glm-5.3-flashx", "resolvedModel": "byok/zai/glm-5.3-flashx", "wallSeconds": 40.713, "inputTokens": 822, "outputTokens": 5489, "totalTokens": 6311, "costUsd": 0.007165390000000001, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T124951Z_aider_byok_zai_glm-5.3-flashx_ffee9b.json", "archived": false }, { "id": "20261016T124951Z_verdict-code_byok_openai_gpt-6-luna_ebbc95", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/openai/gpt-6-luna", "resolvedModel": null, "wallSeconds": 3.788, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T124951Z_verdict-code_byok_openai_gpt-6-luna_ebbc95.json", "archived": false, "supersededBy": "20261016T130549Z_verdict-code_byok_openai_gpt-6-luna_4f07c2" }, { "id": "20261016T124951Z_verdict-code_byok_qwen_qwen3.8-flash_029acf", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": null, "wallSeconds": 240.32, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "TIMEOUT", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "current-model", "resultNote": "Process reached the fixed time limit without completing.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T124951Z_verdict-code_byok_qwen_qwen3.8-flash_029acf.json", "archived": false, "supersededBy": "20261016T130549Z_verdict-code_byok_qwen_qwen3.8-flash_fee25f" }, { "id": "20261016T124951Z_verdict-code_byok_zai_glm-5.3-flash_58dcd7", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 84.191, "inputTokens": 20417, "outputTokens": 2526, "totalTokens": 22943, "costUsd": 0.004325549999999999, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "low", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T124951Z_verdict-code_byok_zai_glm-5.3-flash_58dcd7.json", "archived": false, "supersededBy": "20261016T130549Z_verdict-code_byok_zai_glm-5.3-flash_9801e4" }, { "id": "20261016T124951Z_verdict-code_byok_zai_glm-5.3-flashx_5f0d6b", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flashx", "resolvedModel": "byok/zai/glm-5.3-flashx", "wallSeconds": 73.742, "inputTokens": 12279, "outputTokens": 7556, "totalTokens": 19835, "costUsd": 0.01398823, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T124951Z_verdict-code_byok_zai_glm-5.3-flashx_5f0d6b.json", "archived": false }, { "id": "20261016T125355Z_verdict-code_byok_anthropic_claude-haiku-5-5_62d411", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/anthropic/claude-haiku-5-5", "resolvedModel": "byok/anthropic/claude-haiku-5-5", "wallSeconds": 13.412, "inputTokens": 11092, "outputTokens": 1655, "totalTokens": 12747, "costUsd": null, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Provider cost estimation returned zero despite positive usage and catalog prices. Cost is unavailable; the separately labelled catalog estimate is not an observed charge." ], "evidenceFile": "20261016T125355Z_verdict-code_byok_anthropic_claude-haiku-5-5_62d411.json", "archived": false }, { "id": "20261016T130549Z_aider_byok_openai_gpt-6-luna_563ba0", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/openai/gpt-6-luna", "resolvedModel": "byok/openai/gpt-6-luna", "wallSeconds": 30.003, "inputTokens": 836, "outputTokens": 1576, "totalTokens": 2412, "costUsd": 0.000872, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T130549Z_aider_byok_openai_gpt-6-luna_563ba0.json", "archived": false }, { "id": "20261016T130549Z_verdict-code_byok_openai_gpt-6-luna_4f07c2", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/openai/gpt-6-luna", "resolvedModel": "byok/openai/gpt-6-luna", "wallSeconds": 30.985, "inputTokens": 5713, "outputTokens": 1390, "totalTokens": 7103, "costUsd": 0.001266, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T130549Z_verdict-code_byok_openai_gpt-6-luna_4f07c2.json", "archived": false }, { "id": "20261016T130549Z_verdict-code_byok_qwen_qwen3.8-flash_fee25f", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": null, "wallSeconds": 153.084, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T130549Z_verdict-code_byok_qwen_qwen3.8-flash_fee25f.json", "archived": false, "supersededBy": "20261016T131454Z_verdict-code_byok_qwen_qwen3.8-flash_7fd4b1" }, { "id": "20261016T130549Z_verdict-code_byok_zai_glm-5.3-flash_9801e4", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 30.955, "inputTokens": 3984, "outputTokens": 927, "totalTokens": 4911, "costUsd": 0.0010611, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "low", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T130549Z_verdict-code_byok_zai_glm-5.3-flash_9801e4.json", "archived": false }, { "id": "20261016T131454Z_verdict-code_byok_qwen_qwen3.8-flash_7fd4b1", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": null, "wallSeconds": 240.628, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "TIMEOUT", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "current-model", "resultNote": "Process reached the fixed time limit without completing.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T131454Z_verdict-code_byok_qwen_qwen3.8-flash_7fd4b1.json", "archived": false }, { "id": "native-devin-extra-regression-repair", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 10.936, "inputTokens": 72162, "outputTokens": 1086, "totalTokens": 73248, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "swe-2-medium", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter generation_model", "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-devin-extra-regression-repair.json" }, { "id": "native-claude-extra-regression-repair", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Claude Code", "requestedModel": "claude-sonnet-5-5", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 19.102, "inputTokens": 129952, "outputTokens": 1655, "totalTokens": 131607, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "claude-sonnet-5-5", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter modelUsage canonical model", "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-claude-extra-regression-repair.json" }, { "id": "native-codex-extra-regression-repair", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 37.556, "inputTokens": 114952, "outputTokens": 1175, "totalTokens": 116127, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "gpt-6.1-sol", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native-effective-config; provider response identity not exposed", "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-response model identity not exposed; native effective selection is recorded separately and does not change acceptance status." ], "evidenceFile": "native-codex-extra-regression-repair.json" }, { "id": "native-devin-blueprint-explicit-dependencies", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 5.329, "inputTokens": 27466, "outputTokens": 512, "totalTokens": 27978, "costUsd": null, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": "swe-2-medium", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter generation_model", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-devin-blueprint-explicit-dependencies.json" }, { "id": "native-devin-extra-dependency-scope", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 7.391, "inputTokens": 69917, "outputTokens": 563, "totalTokens": 70480, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "swe-2-medium", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter generation_model", "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-devin-extra-dependency-scope.json" }, { "id": "native-devin-blueprint-prose-scope", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 16.52, "inputTokens": 27503, "outputTokens": 781, "totalTokens": 28284, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "swe-2-medium", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter generation_model", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-devin-blueprint-prose-scope.json" }, { "id": "native-codex-extra-dependency-scope", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 25.571, "inputTokens": 91480, "outputTokens": 710, "totalTokens": 92190, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "gpt-6.1-sol", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native-effective-config; provider response identity not exposed", "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-response model identity not exposed; native effective selection is recorded separately and does not change acceptance status." ], "evidenceFile": "native-codex-extra-dependency-scope.json" }, { "id": "native-codex-blueprint-explicit-dependencies", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 17.165, "inputTokens": 48367, "outputTokens": 537, "totalTokens": 48904, "costUsd": null, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": "gpt-6.1-sol", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native-effective-config; provider response identity not exposed", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-response model identity not exposed; native effective selection is recorded separately and does not change acceptance status." ], "evidenceFile": "native-codex-blueprint-explicit-dependencies.json" }, { "id": "native-claude-extra-dependency-scope", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift", "agent": "Claude Code", "requestedModel": "claude-sonnet-5-5", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 16.218, "inputTokens": 85576, "outputTokens": 908, "totalTokens": 86484, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "claude-sonnet-5-5", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter modelUsage canonical model", "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Original raw receipt retained FAIL9/10 because pytest runtime cache was counted as authored code. Independent replay corrected this oracle inconsistency: same candidate9/9 behavior plus source scope passes. No new inference or response replacement." ], "evidenceFile": "native-claude-extra-dependency-scope.json" }, { "id": "native-codex-blueprint-prose-scope", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 17.226, "inputTokens": 48800, "outputTokens": 530, "totalTokens": 49330, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "gpt-6.1-sol", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native-effective-config; provider response identity not exposed", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-response model identity not exposed; native effective selection is recorded separately and does not change acceptance status." ], "evidenceFile": "native-codex-blueprint-prose-scope.json" }, { "id": "native-claude-blueprint-explicit-dependencies", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain", "agent": "Claude Code", "requestedModel": "claude-sonnet-5-5", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 8.83, "inputTokens": 42329, "outputTokens": 849, "totalTokens": 43178, "costUsd": null, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": "claude-sonnet-5-5", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter modelUsage canonical model", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-claude-blueprint-explicit-dependencies.json" }, { "id": "native-claude-blueprint-prose-scope", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria", "agent": "Claude Code", "requestedModel": "claude-sonnet-5-5", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 8.677, "inputTokens": 42355, "outputTokens": 753, "totalTokens": 43108, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "claude-sonnet-5-5", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter modelUsage canonical model", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-claude-blueprint-prose-scope.json" }, { "id": "native-antigravity-preflight", "taskId": "native-access-preflight", "taskTitle": "Repeat isolated native execution access preflight", "agent": "Antigravity", "requestedModel": null, "resolvedModel": null, "wallSeconds": null, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": null, "observedNativeModel": null, "modelIdentityStatus": "NOT_RUN", "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "native", "status": "BLOCKED", "tier": "native subscription", "diagnosticOnly": true, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Access preflight only; no model invocation and no coding failure.", "Installed CLI advertises editor operations, with no isolated headless coding-agent command. Native UI conversations were not disturbed.", "No measured inference wall time, tokens, billing or quality result." ], "evidenceFile": "native-antigravity-preflight.json" }, { "id": "TEST-blueprint-planning-20261016T035726Z-blueprint-explicit-dependencies-blueprint", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain (synthetic PRD input)", "agent": "Blueprint", "requestedModel": null, "resolvedModel": null, "wallSeconds": 0.455, "inputTokens": 0, "outputTokens": 0, "totalTokens": 0, "costUsd": 0, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "deterministic planning", "diagnosticOnly": false, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Heuristic planning, no model invocation; zero tokens are actual estimation inference usage. Forecast tokens/cost/time are excluded from actual run metrics.", "Energy unmeasured.", "Semantic rubric v2 excludes exact task count from acceptance. Count is diagnostic only; exclusion mentions are evaluated for explicit delivery versus reaffirmed non-delivery. Ambiguous wording remains UNVERIFIED.", "Semantic-v2 rerun uses originalunchanged synthetic PRDs. Originalv1 attempts remain published separately. No generated-code execution claim." ], "evidenceFile": "TEST-blueprint-planning-20261016T035726Z-blueprint-explicit-dependencies-blueprint.json" }, { "id": "TEST-blueprint-planning-20261016T035733Z-blueprint-prose-scope-blueprint", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria (synthetic PRD input)", "agent": "Blueprint", "requestedModel": null, "resolvedModel": null, "wallSeconds": 0.958, "inputTokens": 0, "outputTokens": 0, "totalTokens": 0, "costUsd": 0, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "deterministic planning", "diagnosticOnly": false, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Heuristic planning, no model invocation; zero tokens are actual estimation inference usage. Forecast tokens/cost/time are excluded from actual run metrics.", "Energy unmeasured.", "Semantic rubric v2 excludes exact task count from acceptance. Count is diagnostic only; exclusion mentions are evaluated for explicit delivery versus reaffirmed non-delivery. Ambiguous wording remains UNVERIFIED.", "Semantic-v2 rerun uses originalunchanged synthetic PRDs. Originalv1 attempts remain published separately. No generated-code execution claim." ], "evidenceFile": "TEST-blueprint-planning-20261016T035733Z-blueprint-prose-scope-blueprint.json" }, { "id": "TEST-blueprint-planning-20261016T035735Z-blueprint-explicit-dependencies-raw-glm", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain (synthetic PRD input)", "agent": "Raw model planning baseline", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 13.895, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Raw model baseline, not a competing coding agent. Requested model held constant only within this baseline; Blueprint has no inference model.", "Cost is published-rate estimate at $0.15 input/$0.50 output per MTok; provider cash receipt not captured.", "Semantic rubric v2 excludes exact task count from acceptance. Count is diagnostic only; exclusion mentions are evaluated for explicit delivery versus reaffirmed non-delivery. Ambiguous wording remains UNVERIFIED.", "Semantic-v2 rerun uses originalunchanged synthetic PRDs. Originalv1 attempts remain published separately. No generated-code execution claim." ], "evidenceFile": "TEST-blueprint-planning-20261016T035735Z-blueprint-explicit-dependencies-raw-glm.json" }, { "id": "TEST-blueprint-planning-20261016T035735Z-blueprint-prose-scope-raw-glm", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria (synthetic PRD input)", "agent": "Raw model planning baseline", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 34.303, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Raw model baseline, not a competing coding agent. Requested model held constant only within this baseline; Blueprint has no inference model.", "Cost is published-rate estimate at $0.15 input/$0.50 output per MTok; provider cash receipt not captured.", "Semantic rubric v2 excludes exact task count from acceptance. Count is diagnostic only; exclusion mentions are evaluated for explicit delivery versus reaffirmed non-delivery. Ambiguous wording remains UNVERIFIED.", "Semantic-v2 rerun uses originalunchanged synthetic PRDs. Originalv1 attempts remain published separately. No generated-code execution claim." ], "evidenceFile": "TEST-blueprint-planning-20261016T035735Z-blueprint-prose-scope-raw-glm.json" }, { "id": "TEST-extra-dependency-scope-verdict-code-20261016T040002Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": null, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Gateway health preflight raised ConnectError [Errno61] Connection refused before agent/provider invocation. No coding quality result or spend inferred.", "This preflight diagnostic was reconstructed immediately after wrapper failure; elapsed time was not durably captured and remains unknown." ], "evidenceFile": "TEST-extra-dependency-scope-verdict-code-20261016T040002Z.json" }, { "id": "TEST-extra-dependency-scope-verdict-code-20261016T040152Z", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 4.061, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "CLI rejected a non-private 0755 instrumentation directory before inference. Corrected runner uses 0700. Configuration diagnostic, not a coding failure.", "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure." ], "evidenceFile": "TEST-extra-dependency-scope-verdict-code-20261016T040152Z.json" }, { "id": "TEST-extra-dependency-scope-verdict-code-20261016T040534Z", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 56.813, "inputTokens": 25815, "outputTokens": 691, "totalTokens": 26506, "costUsd": 0.004217, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Gateway lifecycle persistence failed for request 19908254-ebcc-4ad1-867e-7dfa65664468: terminal durability 500 ms deadline expired beside a 996 ms Gateway loop stall. No timeout or token-budget failure. Complete usage accounting survived.", "Generated code independently passes 9 behavior tests plus 1 scope check, but the CLI did not terminate successfully. Do not count as completed product PASS.", "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Complete Gateway terminal usage proof; cost is estimated BYOK USD, not a provider cash receipt." ], "evidenceFile": "TEST-extra-dependency-scope-verdict-code-20261016T040534Z.json" }, { "id": "TEST-extra-dependency-scope-verdict-code-20261016T042131Z", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 30.96, "inputTokens": 7064, "outputTokens": 414, "totalTokens": 7478, "costUsd": 0.0012670000000000001, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Protocol update: public --task-file filenames.py --pin-task-files --no-ide metadata explicitly scopes context to the same one-file contract already stated in the unchanged prompt. Earlier full-context attempts remain retained; model/task/oracle are unchanged.", "Complete Gateway terminal usage proof; cost is estimated BYOK USD, not a provider cash receipt." ], "evidenceFile": "TEST-extra-dependency-scope-verdict-code-20261016T042131Z.json" }, { "id": "TEST-extra-regression-repair-verdict-code-20261016T035725Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 156.373, "inputTokens": 83930, "outputTokens": 4190, "totalTokens": 88120, "costUsd": 0.014682999999999998, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Instrumentation diagnostic: candidate read provider traces inside workspace. Completion/code checks are real; timing and token usage are confounded. A mistaken interrupted reconstruction was corrected from original terminal stdout/journal; correction retained.", "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Complete Gateway terminal usage proof; cost is estimated BYOK USD, not a provider cash receipt." ], "evidenceFile": "TEST-extra-regression-repair-verdict-code-20261016T035725Z.json" }, { "id": "TEST-extra-regression-repair-verdict-code-20261016T040029Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": null, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Gateway health preflight raised ConnectError [Errno61] Connection refused before agent/provider invocation. No coding quality result or spend inferred.", "This preflight diagnostic was reconstructed immediately after wrapper failure; elapsed time was not durably captured and remains unknown." ], "evidenceFile": "TEST-extra-regression-repair-verdict-code-20261016T040029Z.json" }, { "id": "TEST-extra-regression-repair-verdict-code-20261016T040146Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 3.768, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "CLI rejected a non-private 0755 instrumentation directory before inference. Corrected runner uses 0700. Configuration diagnostic, not a coding failure.", "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure." ], "evidenceFile": "TEST-extra-regression-repair-verdict-code-20261016T040146Z.json" }, { "id": "TEST-extra-regression-repair-verdict-code-20261016T040304Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 146.71, "inputTokens": 76351, "outputTokens": 3337, "totalTokens": 79688, "costUsd": 0.01312, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Complete Gateway terminal usage proof; cost is estimated BYOK USD, not a provider cash receipt." ], "evidenceFile": "TEST-extra-regression-repair-verdict-code-20261016T040304Z.json" } ], "evidenceRoot": "/benchmarks/agent-comparison-20261016-completion-repair" }