{ "schema_version": 1, "kind": "first_party_snapshot", "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "revision": 3, "run_date": "2026-08-26", "status": "complete", "generated_at": "2026-08-27T03:37:39Z", "published_at": "2026-08-26T23:00:00Z", "question_pack_id": "coding-fast", "question_pack_version": "coding-fast-v4.10", "grader_version": "scoring-mode:semantic_q1_q5_equal_v2", "evaluation_profile": "full", "score_baseline_id": "coding-fast-v4.10:semantic_q1_q5_equal_v2", "runner_commit": "91bc45b3cde120774d0c7f55d2bc2dd3593c5979+runner.sha256.b3751f10726c439d5f7d558653d254531c05999fe531237bb73e5e1fb7b8a215", "pricing_snapshot_id": "pricing-v1-2db5881e647a00cd9b1537918236db7b4696805a74f9ba897e97ee78eee8c80f", "retry_policy": { "schema_version": 1, "mode": "app_rules_v1", "execution_timeout_seconds": 1800, "timeout_retry_count": 1, "max_attempts_per_question": 6, "selective_score_retry": false, "rules": { "missing_usage": { "enabled": true, "action": "retry", "max_retries": 1, "cooldown_seconds": 0 }, "reason_tok_516": { "enabled": true, "action": "retry", "max_retries": 1, "cooldown_seconds": 0 }, "slow_response": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 }, "timeout": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 }, "wrong_answer": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 } }, "max_concurrent_targets": 16, "max_concurrent_targets_by_connection": { "GEMINI": 8, "cloudflare-reference": 8, "deepseek": 8, "glm": 5, "grok-api": 4, "kimi": 1, "minimax": 8, "qwen": 5 }, "execution_cycle_count": 3, "budget_mode": "policy_segments_sum", "policy_segments": [ { "policy_sha256": "sha256:188337ed03309ed60f6d194f1b61ad3e5d51a3c142f03bbf14df53b7f94217a8", "cycle_count": 3, "execution_timeout_seconds": 1800, "timeout_retry_count": 1, "max_attempts_per_question": 2, "selective_score_retry": false, "rules": { "missing_usage": { "enabled": true, "action": "retry", "max_retries": 1, "cooldown_seconds": 0 }, "reason_tok_516": { "enabled": true, "action": "retry", "max_retries": 1, "cooldown_seconds": 0 }, "slow_response": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 }, "timeout": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 }, "wrong_answer": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 } } } ], "max_attempts_per_question_per_cycle": 2 }, "environment": { "os": "macOS 15.4 arm64", "app_version": "cloud-runner-v1", "codex_version": "grok-cli-1.0.5", "machine_profile": "macos-arm64-grok-build-endpoint-v1" }, "grader_replay": { "status": "matched", "method": "independent_regrade", "regraded_at": "2026-08-27T03:37:39Z", "raw_answer_bundle_sha256": "sha256:9bbd2a2a0775e1165f00194975967890c9330e91869f58353e6fbd64df4f139f", "manifest_sha256": "sha256:c05704359e0921fd1cf139e4fa19cb0279978e739782a7d4451c2ea479b888a9" }, "planned_configuration_ids": [ "cloudflare-reference:gpt-5.6-sol:medium", "cloudflare-reference:gpt-5.6-sol:high", "cloudflare-reference:gpt-5.6-sol:xhigh", "cloudflare-reference:gpt-5.6-sol:max", "cloudflare-reference:gpt-5.6-sol:low", "cloudflare-reference:gpt-5.6-luna:medium", "cloudflare-reference:gpt-5.6-luna:high", "cloudflare-reference:gpt-5.6-luna:xhigh", "cloudflare-reference:gpt-5.6-luna:max", "cloudflare-reference:gpt-5.6-luna:low", "cloudflare-reference:gpt-5.6-terra:medium", "cloudflare-reference:gpt-5.6-terra:high", "cloudflare-reference:gpt-5.6-terra:xhigh", "cloudflare-reference:gpt-5.6-terra:max", "cloudflare-reference:gpt-5.6-terra:low", "deepseek:deepseek-v4-flash:high", "deepseek:deepseek-v4-pro:high", "deepseek:deepseek-v4-flash-vision-exp:high", "GEMINI:gemini-3.7-flash-high:default", "glm:glm-5.3:high", "glm:glm-5.3:max", "glm:glm-5.3-flash:high", "glm:glm-5.3-flash:max", "minimax:MiniMax-M3:high", "kimi:k3:high", "qwen:qwen3.8-max:default", "grok-api:grok-4.6:high" ], "evaluation_configuration_ids": [ "cloudflare-reference:gpt-5.6-sol:medium", "cloudflare-reference:gpt-5.6-sol:high", "cloudflare-reference:gpt-5.6-sol:xhigh", "cloudflare-reference:gpt-5.6-sol:max", "cloudflare-reference:gpt-5.6-sol:low", "cloudflare-reference:gpt-5.6-luna:medium", "cloudflare-reference:gpt-5.6-luna:high", "cloudflare-reference:gpt-5.6-luna:xhigh", "cloudflare-reference:gpt-5.6-luna:max", "cloudflare-reference:gpt-5.6-luna:low", "cloudflare-reference:gpt-5.6-terra:medium", "cloudflare-reference:gpt-5.6-terra:high", "cloudflare-reference:gpt-5.6-terra:xhigh", "cloudflare-reference:gpt-5.6-terra:max", "cloudflare-reference:gpt-5.6-terra:low", "deepseek:deepseek-v4-flash:high", "deepseek:deepseek-v4-pro:high", "deepseek:deepseek-v4-flash-vision-exp:high", "grok-api:grok-4.6:high", "GEMINI:gemini-3.7-flash-high:default", "glm:glm-5.3:high", "glm:glm-5.3:max", "glm:glm-5.3-flash:high", "glm:glm-5.3-flash:max", "minimax:MiniMax-M3:high", "kimi:k3:high", "qwen:qwen3.8-max:default" ], "completed_configuration_ids": [ "cloudflare-reference:gpt-5.6-sol:medium", "cloudflare-reference:gpt-5.6-sol:high", "cloudflare-reference:gpt-5.6-sol:xhigh", "cloudflare-reference:gpt-5.6-sol:max", "cloudflare-reference:gpt-5.6-sol:low", "cloudflare-reference:gpt-5.6-luna:medium", "cloudflare-reference:gpt-5.6-luna:high", "cloudflare-reference:gpt-5.6-luna:xhigh", "cloudflare-reference:gpt-5.6-luna:max", "cloudflare-reference:gpt-5.6-luna:low", "cloudflare-reference:gpt-5.6-terra:medium", "cloudflare-reference:gpt-5.6-terra:high", "cloudflare-reference:gpt-5.6-terra:xhigh", "cloudflare-reference:gpt-5.6-terra:max", "cloudflare-reference:gpt-5.6-terra:low", "deepseek:deepseek-v4-flash:high", "deepseek:deepseek-v4-pro:high", "deepseek:deepseek-v4-flash-vision-exp:high", "GEMINI:gemini-3.7-flash-high:default", "glm:glm-5.3:high", "glm:glm-5.3:max", "glm:glm-5.3-flash:high", "glm:glm-5.3-flash:max", "minimax:MiniMax-M3:high", "kimi:k3:high", "qwen:qwen3.8-max:default", "grok-api:grok-4.6:high" ], "omitted_configurations": [], "publication_coverage": { "status": "complete", "planned_count": 27, "completed_count": 27, "failed_count": 0, "success_percent": 100.0, "minimum_success_percent": 90, "maximum_failed_configurations": 5 }, "question_ids": [ "01_session_bundle_repair", "02_code_counterexample_maxgap", "03_ci_optimality_certificate", "04_transaction_regression_design", "05_cache_regression_test_design" ], "provenance": { "kind": "first_party_snapshot", "source": "owner_controlled_supplement", "target_run_id": "run-20260826-125310-20f4d5", "comparison_group_id": "run-20260826-220010-b80992", "public_official_snapshot": true, "evaluation_identity": "sha256:e2f05d9a83598d907bfcbf9226988f46caf0499c19fd0e182a3ccc82a8958d81" }, "entry_count": 27, "entries": [ { "model_configuration_id": "cloudflare-reference:gpt-5.6-sol:medium", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-sol", "canonical_model_id": "gpt-5.6-sol", "display_name": "gpt-5.6-sol / medium", "reasoning_effort": "medium", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:77ef993ab604c7af7a5d55a8c6baad7e6abeb53d86d01e9b3038b1b48b28b4ee", "route_fingerprints": [ "route-v1:sha256:77ef993ab604c7af7a5d55a8c6baad7e6abeb53d86d01e9b3038b1b48b28b4ee" ], "legacy_mixed_route_transition": false, "score": 68, "max_score": 100, "question_scores": { "01_session_bundle_repair": 12.0, "02_code_counterexample_maxgap": 17.0, "03_ci_optimality_certificate": 14.0, "04_transaction_regression_design": 13.0, "05_cache_regression_test_design": 12.0 }, "elapsed_ms": 2617866, "usage": { "input_tokens": 48837, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 23093, "reasoning_tokens": 16770 }, "estimated_api_cost_usd": 0.657208, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:be026b7297e05fc02e72312e14e45891df9ff563abd5b86d53bb7863e9ff1c1b" }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-sol:high", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-sol", "canonical_model_id": "gpt-5.6-sol", "display_name": "gpt-5.6-sol / high", "reasoning_effort": "high", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:9ebd9627467c315b3933411e0b2562f5608be53daf1d9d41b2c1202c0600f23b", "route_fingerprints": [ "route-v1:sha256:9ebd9627467c315b3933411e0b2562f5608be53daf1d9d41b2c1202c0600f23b" ], "legacy_mixed_route_transition": false, "score": 80, "max_score": 100, "question_scores": { "01_session_bundle_repair": 15.0, "02_code_counterexample_maxgap": 17.0, "03_ci_optimality_certificate": 18.0, "04_transaction_regression_design": 18.0, "05_cache_regression_test_design": 12.0 }, "elapsed_ms": 2701896, "usage": { "input_tokens": 48839, "cached_input_tokens": 8960, "cache_write_input_tokens": 0, "output_tokens": 39740, "reasoning_tokens": 31728 }, "estimated_api_cost_usd": 0.9579, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:bd67c07c1b568631b06c5b704fe97d701100cf5428ff9bc1f9bac6298388a0ba" }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-sol:xhigh", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-sol", "canonical_model_id": "gpt-5.6-sol", "display_name": "gpt-5.6-sol / xhigh", "reasoning_effort": "xhigh", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:3dc7e98f34417c7b4f34f5a5feb0ad0e0391a4824ca5d89d09e28d0df58b1b82", "route_fingerprints": [ "route-v1:sha256:3dc7e98f34417c7b4f34f5a5feb0ad0e0391a4824ca5d89d09e28d0df58b1b82" ], "legacy_mixed_route_transition": false, "score": 83, "max_score": 100, "question_scores": { "01_session_bundle_repair": 16.0, "02_code_counterexample_maxgap": 18.0, "03_ci_optimality_certificate": 17.0, "04_transaction_regression_design": 16.0, "05_cache_regression_test_design": 16.0 }, "elapsed_ms": 2748452, "usage": { "input_tokens": 48823, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 53044, "reasoning_tokens": 42278 }, "estimated_api_cost_usd": 1.256172, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:314e11c2d0eef62b233dc1ad95ef94e700390bb670bec501445948679c63cfdf" }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-sol:max", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-sol", "canonical_model_id": "gpt-5.6-sol", "display_name": "gpt-5.6-sol / max", "reasoning_effort": "max", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:86de52f21364b5f6eba46c6440f846b0366771bf6b3fbdc26130b4c3315caa48", "route_fingerprints": [ "route-v1:sha256:86de52f21364b5f6eba46c6440f846b0366771bf6b3fbdc26130b4c3315caa48" ], "legacy_mixed_route_transition": false, "score": 85, "max_score": 100, "question_scores": { "01_session_bundle_repair": 14.0, "02_code_counterexample_maxgap": 20.0, "03_ci_optimality_certificate": 18.0, "04_transaction_regression_design": 17.0, "05_cache_regression_test_design": 16.0 }, "elapsed_ms": 2698077, "usage": { "input_tokens": 70585, "cached_input_tokens": 23296, "cache_write_input_tokens": 0, "output_tokens": 80875, "reasoning_tokens": 70620 }, "estimated_api_cost_usd": 1.815974, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:85fb5a98dff745f104074155f1a53df36f0682c357d69dbae745843991870f5a" }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-sol:low", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-sol", "canonical_model_id": "gpt-5.6-sol", "display_name": "gpt-5.6-sol / low", "reasoning_effort": "low", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:d16dd2dc53ffd8694d53a55581094a5a0e8304a4c56ec00280ebb0b5e579c960", "route_fingerprints": [ "route-v1:sha256:d16dd2dc53ffd8694d53a55581094a5a0e8304a4c56ec00280ebb0b5e579c960" ], "legacy_mixed_route_transition": false, "score": 65, "max_score": 100, "question_scores": { "01_session_bundle_repair": 13.0, "02_code_counterexample_maxgap": 14.0, "03_ci_optimality_certificate": 13.0, "04_transaction_regression_design": 11.0, "05_cache_regression_test_design": 14.0 }, "elapsed_ms": 2345424, "usage": { "input_tokens": 48831, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 12152, "reasoning_tokens": 6415 }, "estimated_api_cost_usd": 0.438364, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:65b47aa6dbdc0d5790104902af5809faf622374e6c2e3204dff5702cfab418cc" }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-luna:medium", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-luna", "canonical_model_id": "gpt-5.6-luna", "display_name": "gpt-5.6-luna / medium", "reasoning_effort": "medium", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:31ee295a86a1f24665cb8dda8221e416750b897aab5b83087d9e226575361887", "route_fingerprints": [ "route-v1:sha256:31ee295a86a1f24665cb8dda8221e416750b897aab5b83087d9e226575361887" ], "legacy_mixed_route_transition": false, "score": 44, "max_score": 100, "question_scores": { "01_session_bundle_repair": 7.0, "02_code_counterexample_maxgap": 13.0, "03_ci_optimality_certificate": 11.0, "04_transaction_regression_design": 0.0, "05_cache_regression_test_design": 13.0 }, "elapsed_ms": 2284255, "usage": { "input_tokens": 41970, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 17999, "reasoning_tokens": 11782 }, "estimated_api_cost_usd": 0.029993, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:3415790cdb1cfdd68f5a5ff406dbcc8c0ed04609cb5b03e8bed731f4aa93bba5" }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-luna:high", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-luna", "canonical_model_id": "gpt-5.6-luna", "display_name": "gpt-5.6-luna / high", "reasoning_effort": "high", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:31f4c77e02358dac1889a4bd58044da5d2f2ca6b64f814127c4286cbd2ccb317", "route_fingerprints": [ "route-v1:sha256:31f4c77e02358dac1889a4bd58044da5d2f2ca6b64f814127c4286cbd2ccb317" ], "legacy_mixed_route_transition": false, "score": 62, "max_score": 100, "question_scores": { "01_session_bundle_repair": 7.0, "02_code_counterexample_maxgap": 15.0, "03_ci_optimality_certificate": 14.0, "04_transaction_regression_design": 12.0, "05_cache_regression_test_design": 14.0 }, "elapsed_ms": 2287412, "usage": { "input_tokens": 41972, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 43362, "reasoning_tokens": 36160 }, "estimated_api_cost_usd": 0.060429, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:934f0a543923f0931de1a3b34cd0f959c8b6c7d29df0d0f0903fde9120e8bb50" }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-luna:xhigh", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-luna", "canonical_model_id": "gpt-5.6-luna", "display_name": "gpt-5.6-luna / xhigh", "reasoning_effort": "xhigh", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:9f3511d7600a2578cce2b10450de0ee143231654af0db9713fa9139085518b7e", "route_fingerprints": [ "route-v1:sha256:9f3511d7600a2578cce2b10450de0ee143231654af0db9713fa9139085518b7e" ], "legacy_mixed_route_transition": false, "score": 64, "max_score": 100, "question_scores": { "01_session_bundle_repair": 9.0, "02_code_counterexample_maxgap": 17.0, "03_ci_optimality_certificate": 14.0, "04_transaction_regression_design": 11.0, "05_cache_regression_test_design": 13.0 }, "elapsed_ms": 2369524, "usage": { "input_tokens": 65270, "cached_input_tokens": 13824, "cache_write_input_tokens": 0, "output_tokens": 50780, "reasoning_tokens": 42272 }, "estimated_api_cost_usd": 0.071502, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:c21700f65e600ac91abc0c191267d4c5a8b06ee9448f38bdca230d23b42ca29d" }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-luna:max", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-luna", "canonical_model_id": "gpt-5.6-luna", "display_name": "gpt-5.6-luna / max", "reasoning_effort": "max", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:13e7069f551603ae729eeb68a2880b7d43a9e226fb0c9a707db3a924907a1821", "route_fingerprints": [ "route-v1:sha256:13e7069f551603ae729eeb68a2880b7d43a9e226fb0c9a707db3a924907a1821" ], "legacy_mixed_route_transition": false, "score": 77, "max_score": 100, "question_scores": { "01_session_bundle_repair": 10.0, "02_code_counterexample_maxgap": 18.0, "03_ci_optimality_certificate": 17.0, "04_transaction_regression_design": 16.0, "05_cache_regression_test_design": 16.0 }, "elapsed_ms": 2408672, "usage": { "input_tokens": 91688, "cached_input_tokens": 7936, "cache_write_input_tokens": 0, "output_tokens": 120204, "reasoning_tokens": 112055 }, "estimated_api_cost_usd": 0.161154, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:2379a893bb6f1ee5b92778a26ee60b433bdb213c279c060f111e826ebeedb425" }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-luna:low", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-luna", "canonical_model_id": "gpt-5.6-luna", "display_name": "gpt-5.6-luna / low", "reasoning_effort": "low", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:7a1f585601864fb9d7fe83d7979fa90c00cb382ea7890f4faddb4df3ed222b2e", "route_fingerprints": [ "route-v1:sha256:7a1f585601864fb9d7fe83d7979fa90c00cb382ea7890f4faddb4df3ed222b2e" ], "legacy_mixed_route_transition": false, "score": 50, "max_score": 100, "question_scores": { "01_session_bundle_repair": 6.0, "02_code_counterexample_maxgap": 14.0, "03_ci_optimality_certificate": 10.0, "04_transaction_regression_design": 9.0, "05_cache_regression_test_design": 11.0 }, "elapsed_ms": 2047853, "usage": { "input_tokens": 41968, "cached_input_tokens": 5888, "cache_write_input_tokens": 0, "output_tokens": 9983, "reasoning_tokens": 4685 }, "estimated_api_cost_usd": 0.019313, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:2bc774be8cce47e54d8be6611848a94a19f4e0c9a7f14ed53ad1f8f113d9cd22" }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-terra:medium", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-terra", "canonical_model_id": "gpt-5.6-terra", "display_name": "gpt-5.6-terra / medium", "reasoning_effort": "medium", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:72d060b92c31032a2757dbeb61f7fd575ba2416f2897d17f0f1e7bd979cc98bd", "route_fingerprints": [ "route-v1:sha256:72d060b92c31032a2757dbeb61f7fd575ba2416f2897d17f0f1e7bd979cc98bd" ], "legacy_mixed_route_transition": false, "score": 48, "max_score": 100, "question_scores": { "01_session_bundle_repair": 10.0, "02_code_counterexample_maxgap": 13.0, "03_ci_optimality_certificate": 15.0, "04_transaction_regression_design": 10.0, "05_cache_regression_test_design": 0.0 }, "elapsed_ms": 2025399, "usage": { "input_tokens": 48831, "cached_input_tokens": 6912, "cache_write_input_tokens": 0, "output_tokens": 15378, "reasoning_tokens": 10199 }, "estimated_api_cost_usd": 0.269756, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:220c2165aceeb46b44787e9dc551cf74a649c36eece1e3f34acc08c829e57684" }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-terra:high", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-terra", "canonical_model_id": "gpt-5.6-terra", "display_name": "gpt-5.6-terra / high", "reasoning_effort": "high", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:c2fd73af85212ca5f777393eb73d25872547d990051c07d9722f5dd27b3a5256", "route_fingerprints": [ "route-v1:sha256:c2fd73af85212ca5f777393eb73d25872547d990051c07d9722f5dd27b3a5256" ], "legacy_mixed_route_transition": false, "score": 62, "max_score": 100, "question_scores": { "01_session_bundle_repair": 9.0, "02_code_counterexample_maxgap": 14.0, "03_ci_optimality_certificate": 15.0, "04_transaction_regression_design": 11.0, "05_cache_regression_test_design": 13.0 }, "elapsed_ms": 2010215, "usage": { "input_tokens": 48833, "cached_input_tokens": 13824, "cache_write_input_tokens": 0, "output_tokens": 21531, "reasoning_tokens": 16630 }, "estimated_api_cost_usd": 0.331155, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:ec4c6f34b12ca64f4dc5ff175632cea156bf86d20a718681dd895a168ace3f2f" }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-terra:xhigh", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-terra", "canonical_model_id": "gpt-5.6-terra", "display_name": "gpt-5.6-terra / xhigh", "reasoning_effort": "xhigh", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:af61163ec7260abd9f6d61faa04438ee95bb03df0df38ef1f3ff5d1219e953b8", "route_fingerprints": [ "route-v1:sha256:af61163ec7260abd9f6d61faa04438ee95bb03df0df38ef1f3ff5d1219e953b8" ], "legacy_mixed_route_transition": false, "score": 73, "max_score": 100, "question_scores": { "01_session_bundle_repair": 12.0, "02_code_counterexample_maxgap": 15.0, "03_ci_optimality_certificate": 19.0, "04_transaction_regression_design": 14.0, "05_cache_regression_test_design": 13.0 }, "elapsed_ms": 1999610, "usage": { "input_tokens": 57635, "cached_input_tokens": 15872, "cache_write_input_tokens": 0, "output_tokens": 32054, "reasoning_tokens": 26467 }, "estimated_api_cost_usd": 0.471348, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:df999b3cc0db2dd4b08d79109e0adf1454e0682ddd61132b77e41769a1eba836" }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-terra:max", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-terra", "canonical_model_id": "gpt-5.6-terra", "display_name": "gpt-5.6-terra / max", "reasoning_effort": "max", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:9a05e0f9287ec4540e552326199558de2ab6a61e24c0e08686040e9afeb8849d", "route_fingerprints": [ "route-v1:sha256:9a05e0f9287ec4540e552326199558de2ab6a61e24c0e08686040e9afeb8849d" ], "legacy_mixed_route_transition": false, "score": 84, "max_score": 100, "question_scores": { "01_session_bundle_repair": 17.0, "02_code_counterexample_maxgap": 17.0, "03_ci_optimality_certificate": 18.0, "04_transaction_regression_design": 18.0, "05_cache_regression_test_design": 14.0 }, "elapsed_ms": 2096850, "usage": { "input_tokens": 54010, "cached_input_tokens": 15872, "cache_write_input_tokens": 0, "output_tokens": 130946, "reasoning_tokens": 121853 }, "estimated_api_cost_usd": 1.650802, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:76f18edd3dab8486ae6276a0f6032d3683897cbbd6fd649b9b494283f0576e72" }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-terra:low", "model_configuration": { "provider_id": "codex", "raw_model_id": "gpt-5.6-terra", "canonical_model_id": "gpt-5.6-terra", "display_name": "gpt-5.6-terra / low", "reasoning_effort": "low", "service_tier": "chatgpt_subscription", "route_type": "official_login" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:dea9d7905707562860e6e16c2e00f02bd5376903ae0b6baf9faf62d21c0f6c71", "route_fingerprints": [ "route-v1:sha256:dea9d7905707562860e6e16c2e00f02bd5376903ae0b6baf9faf62d21c0f6c71" ], "legacy_mixed_route_transition": false, "score": 63, "max_score": 100, "question_scores": { "01_session_bundle_repair": 8.0, "02_code_counterexample_maxgap": 15.0, "03_ci_optimality_certificate": 13.0, "04_transaction_regression_design": 15.0, "05_cache_regression_test_design": 12.0 }, "elapsed_ms": 1732392, "usage": { "input_tokens": 48839, "cached_input_tokens": 13824, "cache_write_input_tokens": 0, "output_tokens": 11691, "reasoning_tokens": 6963 }, "estimated_api_cost_usd": 0.213087, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:45cacc26be3824c2aeef3b1070a138836c84ed7682c49430b18d963c6c3ec894" }, { "model_configuration_id": "deepseek:deepseek-v4-flash:high", "model_configuration": { "provider_id": "deepseek", "raw_model_id": "deepseek-v4-flash", "canonical_model_id": "deepseek-v4-flash", "display_name": "deepseek-v4-flash / high", "reasoning_effort": "high", "service_tier": "owner_controlled_endpoint", "route_type": "custom_endpoint" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:0d07e15a979123ba2e4f522910028d316112e71924e8a1eef24faa5a322c4c8c", "route_fingerprints": [ "route-v1:sha256:0d07e15a979123ba2e4f522910028d316112e71924e8a1eef24faa5a322c4c8c" ], "legacy_mixed_route_transition": false, "score": 66, "max_score": 100, "question_scores": { "01_session_bundle_repair": 10.0, "02_code_counterexample_maxgap": 14.0, "03_ci_optimality_certificate": 13.0, "04_transaction_regression_design": 14.0, "05_cache_regression_test_design": 15.0 }, "elapsed_ms": 1311085, "usage": { "input_tokens": 5499, "cached_input_tokens": 5120, "cache_write_input_tokens": 0, "output_tokens": 187714, "reasoning_tokens": 179384 }, "estimated_api_cost_usd": 0.12401, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:679dfa7751ca0d29dfccfe8a2bf19dc03f517328645b764ef8b68e097e050d6d" }, { "model_configuration_id": "deepseek:deepseek-v4-pro:high", "model_configuration": { "provider_id": "deepseek", "raw_model_id": "deepseek-v4-pro", "canonical_model_id": "deepseek-v4-pro", "display_name": "deepseek-v4-pro / high", "reasoning_effort": "high", "service_tier": "owner_controlled_endpoint", "route_type": "custom_endpoint" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:a10b1ccafd8e556f5d8cd3b510c64b04a3b286bfeba64e83d646186b34d0f9fc", "route_fingerprints": [ "route-v1:sha256:a10b1ccafd8e556f5d8cd3b510c64b04a3b286bfeba64e83d646186b34d0f9fc" ], "legacy_mixed_route_transition": false, "score": 64, "max_score": 100, "question_scores": { "01_session_bundle_repair": 12.0, "02_code_counterexample_maxgap": 13.0, "03_ci_optimality_certificate": 12.0, "04_transaction_regression_design": 14.0, "05_cache_regression_test_design": 13.0 }, "elapsed_ms": 1413898, "usage": { "input_tokens": 5499, "cached_input_tokens": 5120, "cache_write_input_tokens": 0, "output_tokens": 161592, "reasoning_tokens": 151964 }, "estimated_api_cost_usd": 0.320315, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:bb32fe7e925ac1c7027ef1ed3e6f6efa93e42f2422ce00e1fd8930c5fb040452" }, { "model_configuration_id": "deepseek:deepseek-v4-flash-vision-exp:high", "model_configuration": { "provider_id": "deepseek", "raw_model_id": "deepseek-v4-flash-vision-exp", "canonical_model_id": "deepseek-v4-flash-vision-exp", "display_name": "deepseek-v4-flash-vision-exp / high", "reasoning_effort": "high", "service_tier": "owner_controlled_endpoint", "route_type": "custom_endpoint" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:9dff719d4f1ae5d22f9eaaf5346d676118dfd87b436c61e55bd82142da475458", "route_fingerprints": [ "route-v1:sha256:9dff719d4f1ae5d22f9eaaf5346d676118dfd87b436c61e55bd82142da475458" ], "legacy_mixed_route_transition": false, "score": 57, "max_score": 100, "question_scores": { "01_session_bundle_repair": 8.0, "02_code_counterexample_maxgap": 13.0, "03_ci_optimality_certificate": 13.0, "04_transaction_regression_design": 12.0, "05_cache_regression_test_design": 11.0 }, "elapsed_ms": 1309803, "usage": { "input_tokens": 5499, "cached_input_tokens": 5120, "cache_write_input_tokens": 0, "output_tokens": 203671, "reasoning_tokens": 195223 }, "estimated_api_cost_usd": 0.134542, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:2dd557e90a783e9846688991e5bc771b0aa752898bddf65723fb07743aeb9c70" }, { "model_configuration_id": "GEMINI:gemini-3.7-flash-high:default", "model_configuration": { "provider_id": "gemini", "raw_model_id": "gemini-3.7-flash-high", "canonical_model_id": "gemini-3.7-flash-high", "display_name": "gemini-3.7-flash-high / default", "reasoning_effort": "default", "service_tier": "owner_controlled_endpoint", "route_type": "custom_endpoint" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:9d3ec7b183c19a49f9483658c991bd0a8f1033ee259b29ac96cd17e27d3c420a", "route_fingerprints": [ "route-v1:sha256:9d3ec7b183c19a49f9483658c991bd0a8f1033ee259b29ac96cd17e27d3c420a" ], "legacy_mixed_route_transition": false, "score": 55, "max_score": 100, "question_scores": { "01_session_bundle_repair": 10.0, "02_code_counterexample_maxgap": 16.0, "03_ci_optimality_certificate": 16.0, "04_transaction_regression_design": 13.0, "05_cache_regression_test_design": 0.0 }, "elapsed_ms": 457724, "usage": { "input_tokens": 6454, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 14501, "reasoning_tokens": 132351 }, "estimated_api_cost_usd": 0.059219, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:f284fbf1796d5c2b128918ba15eb742427a820b7e374c9672f3d2c5e786afa4c" }, { "model_configuration_id": "glm:glm-5.3:high", "model_configuration": { "provider_id": "z-ai", "raw_model_id": "glm-5.3", "canonical_model_id": "glm-5.3", "display_name": "glm-5.3 / high", "reasoning_effort": "high", "service_tier": "owner_controlled_endpoint", "route_type": "custom_endpoint" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:87408a5e106d1ce7b49055c7018c0d092202b2688fb00a6044369e1e09b7c2a9", "route_fingerprints": [ "route-v1:sha256:87408a5e106d1ce7b49055c7018c0d092202b2688fb00a6044369e1e09b7c2a9" ], "legacy_mixed_route_transition": false, "score": 68, "max_score": 100, "question_scores": { "01_session_bundle_repair": 13.0, "02_code_counterexample_maxgap": 16.0, "03_ci_optimality_certificate": 12.0, "04_transaction_regression_design": 15.0, "05_cache_regression_test_design": 12.0 }, "elapsed_ms": 1415385, "usage": { "input_tokens": 5000, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 93213, "reasoning_tokens": 84811 }, "estimated_api_cost_usd": 0.417137, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:8a3080e17a51e7a28be96d705cbd8ae91c9ea9a7ed3b0c3c3aea0999f90b4696" }, { "model_configuration_id": "glm:glm-5.3:max", "model_configuration": { "provider_id": "z-ai", "raw_model_id": "glm-5.3", "canonical_model_id": "glm-5.3", "display_name": "glm-5.3 / max", "reasoning_effort": "max", "service_tier": "owner_controlled_endpoint", "route_type": "custom_endpoint" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:51bc22d19d1543b8f17342dc08c3b009793efcf8e5ae880eac6efe603cf273ca", "route_fingerprints": [ "route-v1:sha256:51bc22d19d1543b8f17342dc08c3b009793efcf8e5ae880eac6efe603cf273ca" ], "legacy_mixed_route_transition": false, "score": 66, "max_score": 100, "question_scores": { "01_session_bundle_repair": 10.0, "02_code_counterexample_maxgap": 14.0, "03_ci_optimality_certificate": 14.0, "04_transaction_regression_design": 15.0, "05_cache_regression_test_design": 13.0 }, "elapsed_ms": 1506012, "usage": { "input_tokens": 5000, "cached_input_tokens": 1216, "cache_write_input_tokens": 0, "output_tokens": 98661, "reasoning_tokens": 90284 }, "estimated_api_cost_usd": 0.439722, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:5f8538a1fe77692a05503838f11e770195d399b8590d3243b97b907e976c51a3" }, { "model_configuration_id": "glm:glm-5.3-flash:high", "model_configuration": { "provider_id": "z-ai", "raw_model_id": "glm-5.3-flash", "canonical_model_id": "glm-5.3-flash", "display_name": "glm-5.3-flash / high", "reasoning_effort": "high", "service_tier": "owner_controlled_endpoint", "route_type": "custom_endpoint" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:43daab3dc584a6fe142e2a1662e2fb0be1a3764780af59f3560dbe861f391d46", "route_fingerprints": [ "route-v1:sha256:43daab3dc584a6fe142e2a1662e2fb0be1a3764780af59f3560dbe861f391d46" ], "legacy_mixed_route_transition": false, "score": 63, "max_score": 100, "question_scores": { "01_session_bundle_repair": 10.0, "02_code_counterexample_maxgap": 14.0, "03_ci_optimality_certificate": 14.0, "04_transaction_regression_design": 14.0, "05_cache_regression_test_design": 11.0 }, "elapsed_ms": 1381291, "usage": { "input_tokens": 5000, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 40234, "reasoning_tokens": 32422 }, "estimated_api_cost_usd": 0.010434, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:8774ea1e7cd6c4cabad9293e3ccbf516b1acfb0c07e4f88380671b0287700a66" }, { "model_configuration_id": "glm:glm-5.3-flash:max", "model_configuration": { "provider_id": "z-ai", "raw_model_id": "glm-5.3-flash", "canonical_model_id": "glm-5.3-flash", "display_name": "glm-5.3-flash / max", "reasoning_effort": "max", "service_tier": "owner_controlled_endpoint", "route_type": "custom_endpoint" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:a9717539501b5b24e230c3e937c56697475939e8b02bfbaee38ce28322800c43", "route_fingerprints": [ "route-v1:sha256:a9717539501b5b24e230c3e937c56697475939e8b02bfbaee38ce28322800c43" ], "legacy_mixed_route_transition": false, "score": 61, "max_score": 100, "question_scores": { "01_session_bundle_repair": 5.0, "02_code_counterexample_maxgap": 15.0, "03_ci_optimality_certificate": 12.0, "04_transaction_regression_design": 16.0, "05_cache_regression_test_design": 13.0 }, "elapsed_ms": 1276800, "usage": { "input_tokens": 5000, "cached_input_tokens": 4864, "cache_write_input_tokens": 0, "output_tokens": 47986, "reasoning_tokens": 39864 }, "estimated_api_cost_usd": 0.01208, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:6ee9c0da2023e9f075a5dddb08368376b28e1e2e7c9cf5522b12c1e4fa949f12" }, { "model_configuration_id": "minimax:MiniMax-M3:high", "model_configuration": { "provider_id": "minimax", "raw_model_id": "MiniMax-M3", "canonical_model_id": "MiniMax-M3", "display_name": "MiniMax-M3 / high", "reasoning_effort": "high", "service_tier": "owner_controlled_endpoint", "route_type": "custom_endpoint" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:2fb3cc6f382a9f3dac933ae936dbec404ad643ab0437df5be106924f7dc6bcb7", "route_fingerprints": [ "route-v1:sha256:2fb3cc6f382a9f3dac933ae936dbec404ad643ab0437df5be106924f7dc6bcb7" ], "legacy_mixed_route_transition": false, "score": 46, "max_score": 100, "question_scores": { "01_session_bundle_repair": 2.0, "02_code_counterexample_maxgap": 13.0, "03_ci_optimality_certificate": 10.0, "04_transaction_regression_design": 9.0, "05_cache_regression_test_design": 12.0 }, "elapsed_ms": 823293, "usage": { "input_tokens": 5828, "cached_input_tokens": 3709, "cache_write_input_tokens": 0, "output_tokens": 233027, "reasoning_tokens": 103802 }, "estimated_api_cost_usd": 0.280491, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:10571b1eda485c8251376321c30e681e7fd66c43cda9a00f2b90bee7a53c7a61" }, { "model_configuration_id": "kimi:k3:high", "model_configuration": { "provider_id": "moonshot", "raw_model_id": "k3", "canonical_model_id": "k3", "display_name": "k3 / high", "reasoning_effort": "high", "service_tier": "owner_controlled_endpoint", "route_type": "custom_endpoint" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:d206fff528c7cca5d2f70d7d9b667585c3c8b452b259113533836a5c15f53fe2", "route_fingerprints": [ "route-v1:sha256:d206fff528c7cca5d2f70d7d9b667585c3c8b452b259113533836a5c15f53fe2" ], "legacy_mixed_route_transition": false, "score": 61, "max_score": 100, "question_scores": { "01_session_bundle_repair": 9.0, "02_code_counterexample_maxgap": 13.0, "03_ci_optimality_certificate": 12.0, "04_transaction_regression_design": 14.0, "05_cache_regression_test_design": 13.0 }, "elapsed_ms": 1666695, "usage": { "input_tokens": 5373, "cached_input_tokens": 5373, "cache_write_input_tokens": 0, "output_tokens": 29276, "reasoning_tokens": 0 }, "estimated_api_cost_usd": 0.440752, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:50df12cf9a51a9371ae8fece9ab482b169529545beea76060e16cb0125fc6948" }, { "model_configuration_id": "qwen:qwen3.8-max:default", "model_configuration": { "provider_id": "custom", "raw_model_id": "qwen3.8-max", "canonical_model_id": "qwen3.8-max", "display_name": "qwen3.8-max / default", "reasoning_effort": "default", "service_tier": "owner_controlled_endpoint", "route_type": "custom_endpoint" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:b1144b443f282411d25e17267ac48106ff948ea82aa0ed6070e6469fa3891439", "route_fingerprints": [ "route-v1:sha256:b1144b443f282411d25e17267ac48106ff948ea82aa0ed6070e6469fa3891439" ], "legacy_mixed_route_transition": false, "score": 66, "max_score": 100, "question_scores": { "01_session_bundle_repair": 10.0, "02_code_counterexample_maxgap": 17.0, "03_ci_optimality_certificate": 12.0, "04_transaction_regression_design": 13.0, "05_cache_regression_test_design": 14.0 }, "elapsed_ms": 1279496, "usage": { "input_tokens": 5782, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 146374, "reasoning_tokens": 134007 }, "estimated_api_cost_usd": 0.889808, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T22:56:05+00:00", "source_evidence_group_id": "run-20260826-220010-b80992", "run_manifest_sha256": "sha256:3f06265635229c8e4fd56bfbd0362babd9f83d404477dd1f82b9dc7f8e0106ab" }, { "model_configuration_id": "grok-api:grok-4.6:high", "model_configuration": { "provider_id": "xai", "raw_model_id": "grok-4.6", "canonical_model_id": "grok-4.6", "display_name": "grok-4.6 / high", "reasoning_effort": "high", "service_tier": "owner_controlled_endpoint", "route_type": "custom_endpoint" }, "advisor_eligible": true, "score_integrity": "first_party_controlled", "route_identity": "first_party_controlled", "route_fingerprint": "route-v1:sha256:570d33d6b53b3746f45099c68ceae4d9f987eaca0ab7be4e491d3f63abca20c9", "route_fingerprints": [ "route-v1:sha256:570d33d6b53b3746f45099c68ceae4d9f987eaca0ab7be4e491d3f63abca20c9" ], "legacy_mixed_route_transition": false, "score": 75, "max_score": 100, "question_scores": { "01_session_bundle_repair": 10.0, "02_code_counterexample_maxgap": 17.0, "03_ci_optimality_certificate": 17.0, "04_transaction_regression_design": 15.0, "05_cache_regression_test_design": 16.0 }, "elapsed_ms": 1853556, "usage": { "input_tokens": 48856, "cached_input_tokens": 39296, "cache_write_input_tokens": 0, "output_tokens": 115757, "reasoning_tokens": 106500 }, "estimated_api_cost_usd": 0.751059, "cost_coverage": "complete", "attempt_count": 5, "failure_count": 0, "hard_failure_count": 0, "completed_at": "2026-08-26T13:24:12+00:00", "source_evidence_group_id": "run-20260826-125310-20f4d5", "run_manifest_sha256": "sha256:d84d2a02938a74375f004daf4a7ffcb57650d59e5bd000c1b4d8e5decd259426", "elapsed_time_source": "earliest_execution_start_to_latest_execution_finish_v1", "elapsed_time_evidence_sha256": "sha256:0c92f2b6c7f360e5a80a6b69674352fc21ef6c02308fb41ea50563de4bf53e9e", "queue_time_excluded": true } ], "revision_reason": "Add Grok 4.6 High backend score 75 from independently regraded owner-controlled local endpoint evidence", "leaderboard_projection": { "schema_version": 2, "source": "publisher", "ranking_rule": "score_desc_hard_failure_asc_elapsed_asc_cost_asc_identity_v1", "trend_rule": "latest_6_same_configuration_protocol_route_v1", "questions": [ { "id": "01_session_bundle_repair", "short_label": "Q1", "title": "Black-box Regression Audit", "capability_id": "black_box_regression_testing", "capability_label": "契约测试", "detail_label": "边界场景", "ordinal": 1 }, { "id": "02_code_counterexample_maxgap", "short_label": "Q2", "title": "Retry Planner Counterexamples", "capability_id": "debug_counterexample", "capability_label": "反例构造", "detail_label": "重试调度", "ordinal": 2 }, { "id": "03_ci_optimality_certificate", "short_label": "Q3", "title": "CI Adversarial Audit", "capability_id": "ci_plan_audit", "capability_label": "方案审计", "detail_label": "对抗场景", "ordinal": 3 }, { "id": "04_transaction_regression_design", "short_label": "Q4", "title": "Transaction Regression Design", "capability_id": "state_machine_testing", "capability_label": "状态机", "detail_label": "事务回归", "ordinal": 4 }, { "id": "05_cache_regression_test_design", "short_label": "Q5", "title": "Cache Regression Test Design", "capability_id": "regression_validation", "capability_label": "测试设计", "detail_label": "缓存回归", "ordinal": 5 } ], "rows": [ { "model_configuration_id": "cloudflare-reference:gpt-5.6-sol:max", "rank": 1, "target_labels": [ { "id": "highest_score", "label": "Highest score" } ], "decision_tags": [ { "kind": "recommended" } ], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 14.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 20.0 }, { "question_id": "03_ci_optimality_certificate", "score": 18.0 }, { "question_id": "04_transaction_regression_design", "score": 17.0 }, { "question_id": "05_cache_regression_test_design", "score": 16.0 } ], "trend": { "compatibility_key": "sha256:ac01753bb4c77a243cf621c8c990d6037898e70e1938de9c7b15f6f6c754c087", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 87.0, "elapsed_ms": 1827724.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 83.0, "elapsed_ms": 1833850.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 87.0, "elapsed_ms": 2942301.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 87.0, "elapsed_ms": 2942301.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 87.0, "elapsed_ms": 2942301.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 85.0, "elapsed_ms": 2698077.0 } ] } }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-terra:max", "rank": 2, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 17.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 17.0 }, { "question_id": "03_ci_optimality_certificate", "score": 18.0 }, { "question_id": "04_transaction_regression_design", "score": 18.0 }, { "question_id": "05_cache_regression_test_design", "score": 14.0 } ], "trend": { "compatibility_key": "sha256:94a67ee26447fefd8e95dd6ed9af042c5036198b70cf760b857506dd055263dd", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 82.0, "elapsed_ms": 2420348.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 72.0, "elapsed_ms": 1847948.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 80.0, "elapsed_ms": 2288419.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 80.0, "elapsed_ms": 2288419.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 80.0, "elapsed_ms": 2288419.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 84.0, "elapsed_ms": 2096850.0 } ] } }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-sol:xhigh", "rank": 3, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 16.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 18.0 }, { "question_id": "03_ci_optimality_certificate", "score": 17.0 }, { "question_id": "04_transaction_regression_design", "score": 16.0 }, { "question_id": "05_cache_regression_test_design", "score": 16.0 } ], "trend": { "compatibility_key": "sha256:10e949530e788253b49a9790e8140479c2f3cd6e0f1643709477dfb97589d83f", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 73.0, "elapsed_ms": 1385726.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 83.0, "elapsed_ms": 1377594.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 79.0, "elapsed_ms": 1450672.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 79.0, "elapsed_ms": 1450672.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 79.0, "elapsed_ms": 1450672.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 83.0, "elapsed_ms": 2748452.0 } ] } }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-sol:high", "rank": 4, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 15.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 17.0 }, { "question_id": "03_ci_optimality_certificate", "score": 18.0 }, { "question_id": "04_transaction_regression_design", "score": 18.0 }, { "question_id": "05_cache_regression_test_design", "score": 12.0 } ], "trend": { "compatibility_key": "sha256:ab9725c034b70c7ba76355da5970967cd64a716cbff5eed438ddeddacae82d33", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 80.0, "elapsed_ms": 971857.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 75.0, "elapsed_ms": 905266.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 83.0, "elapsed_ms": 848104.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 83.0, "elapsed_ms": 848104.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 83.0, "elapsed_ms": 848104.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 80.0, "elapsed_ms": 2701896.0 } ] } }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-luna:max", "rank": 5, "target_labels": [], "decision_tags": [ { "kind": "value" } ], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 10.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 18.0 }, { "question_id": "03_ci_optimality_certificate", "score": 17.0 }, { "question_id": "04_transaction_regression_design", "score": 16.0 }, { "question_id": "05_cache_regression_test_design", "score": 16.0 } ], "trend": { "compatibility_key": "sha256:5dc6f8d31ee3592ea1456c61af33389e2194f485d5ad0282da75209acf8db36a", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 68.0, "elapsed_ms": 3045078.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 68.0, "elapsed_ms": 1827954.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 68.0, "elapsed_ms": 2949721.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 68.0, "elapsed_ms": 2949721.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 68.0, "elapsed_ms": 2949721.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 77.0, "elapsed_ms": 2408672.0 } ] } }, { "model_configuration_id": "grok-api:grok-4.6:high", "rank": 6, "target_labels": [], "decision_tags": [ { "kind": "speed" } ], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 10.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 17.0 }, { "question_id": "03_ci_optimality_certificate", "score": 17.0 }, { "question_id": "04_transaction_regression_design", "score": 15.0 }, { "question_id": "05_cache_regression_test_design", "score": 16.0 } ], "trend": { "compatibility_key": "sha256:566a27b8aefa94aba9b3528988c75253eeea7312495a96e672692147f6496642", "sample_count": 1, "comparable": false, "stable_ranking_eligible": false, "points": [ { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 75.0, "elapsed_ms": 1853556.0 } ] } }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-terra:xhigh", "rank": 7, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 12.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 15.0 }, { "question_id": "03_ci_optimality_certificate", "score": 19.0 }, { "question_id": "04_transaction_regression_design", "score": 14.0 }, { "question_id": "05_cache_regression_test_design", "score": 13.0 } ], "trend": { "compatibility_key": "sha256:ec22f4263e69c41cdad7c400df1e3db20760f4402d33051f77c748d9fd8ec11e", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 69.0, "elapsed_ms": 930868.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 76.0, "elapsed_ms": 1183469.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 50.0, "elapsed_ms": 914698.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 50.0, "elapsed_ms": 914698.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 50.0, "elapsed_ms": 914698.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 73.0, "elapsed_ms": 1999610.0 } ] } }, { "model_configuration_id": "glm:glm-5.3:high", "rank": 8, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 13.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 16.0 }, { "question_id": "03_ci_optimality_certificate", "score": 12.0 }, { "question_id": "04_transaction_regression_design", "score": 15.0 }, { "question_id": "05_cache_regression_test_design", "score": 12.0 } ], "trend": { "compatibility_key": "sha256:557f3983829a51dbf76149d7057c7a1102e90c4b387fd2e649829f8833b63240", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 66.0, "elapsed_ms": 1315218.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 73.0, "elapsed_ms": 1626759.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 64.0, "elapsed_ms": 1273429.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 64.0, "elapsed_ms": 1273429.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 64.0, "elapsed_ms": 1273429.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 68.0, "elapsed_ms": 1415385.0 } ] } }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-sol:medium", "rank": 9, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 12.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 17.0 }, { "question_id": "03_ci_optimality_certificate", "score": 14.0 }, { "question_id": "04_transaction_regression_design", "score": 13.0 }, { "question_id": "05_cache_regression_test_design", "score": 12.0 } ], "trend": { "compatibility_key": "sha256:a262783e6158c09fe5917e950b66dacb11a061d87be7e56566f94903710af793", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 75.0, "elapsed_ms": 590875.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 75.0, "elapsed_ms": 555973.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 75.0, "elapsed_ms": 507869.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 75.0, "elapsed_ms": 507869.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 75.0, "elapsed_ms": 507869.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 68.0, "elapsed_ms": 2617866.0 } ] } }, { "model_configuration_id": "qwen:qwen3.8-max:default", "rank": 10, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 10.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 17.0 }, { "question_id": "03_ci_optimality_certificate", "score": 12.0 }, { "question_id": "04_transaction_regression_design", "score": 13.0 }, { "question_id": "05_cache_regression_test_design", "score": 14.0 } ], "trend": { "compatibility_key": "sha256:41c0311b2b4ae4054a784c5d37b8dad9f70d37572018b012a184a52b25df352c", "sample_count": 4, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T06-00-00Z", "published_at": "2026-08-24T06:00:00Z", "score": 70.0, "elapsed_ms": 3926110.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 62.0, "elapsed_ms": 4085912.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 62.0, "elapsed_ms": 4085912.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 66.0, "elapsed_ms": 1279496.0 } ] } }, { "model_configuration_id": "deepseek:deepseek-v4-flash:high", "rank": 11, "target_labels": [], "decision_tags": [ { "kind": "lightweight" } ], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 10.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 14.0 }, { "question_id": "03_ci_optimality_certificate", "score": 13.0 }, { "question_id": "04_transaction_regression_design", "score": 14.0 }, { "question_id": "05_cache_regression_test_design", "score": 15.0 } ], "trend": { "compatibility_key": "sha256:421c2595f238a1a35288a5ec8837dcd7e94e8a63657a563b68a49b667fe38762", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 57.0, "elapsed_ms": 1714113.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 64.0, "elapsed_ms": 1981727.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 68.0, "elapsed_ms": 2005797.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 68.0, "elapsed_ms": 2005797.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 68.0, "elapsed_ms": 2005797.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 66.0, "elapsed_ms": 1311085.0 } ] } }, { "model_configuration_id": "glm:glm-5.3:max", "rank": 12, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 10.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 14.0 }, { "question_id": "03_ci_optimality_certificate", "score": 14.0 }, { "question_id": "04_transaction_regression_design", "score": 15.0 }, { "question_id": "05_cache_regression_test_design", "score": 13.0 } ], "trend": { "compatibility_key": "sha256:8bccc5a8758963fcfe1671501e594cebc026529698d16690c998cdab4b9361cc", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 68.0, "elapsed_ms": 1604566.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 65.0, "elapsed_ms": 1126645.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 64.0, "elapsed_ms": 1050099.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 64.0, "elapsed_ms": 1050099.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 64.0, "elapsed_ms": 1050099.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 66.0, "elapsed_ms": 1506012.0 } ] } }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-sol:low", "rank": 13, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 13.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 14.0 }, { "question_id": "03_ci_optimality_certificate", "score": 13.0 }, { "question_id": "04_transaction_regression_design", "score": 11.0 }, { "question_id": "05_cache_regression_test_design", "score": 14.0 } ], "trend": { "compatibility_key": "sha256:c413b9dda586757ff3a2fabdd8c41a01fc75a846739e6d99c37460689986352c", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 69.0, "elapsed_ms": 369238.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 69.0, "elapsed_ms": 478397.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 73.0, "elapsed_ms": 386444.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 73.0, "elapsed_ms": 386444.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 73.0, "elapsed_ms": 386444.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 65.0, "elapsed_ms": 2345424.0 } ] } }, { "model_configuration_id": "deepseek:deepseek-v4-pro:high", "rank": 14, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 12.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 13.0 }, { "question_id": "03_ci_optimality_certificate", "score": 12.0 }, { "question_id": "04_transaction_regression_design", "score": 14.0 }, { "question_id": "05_cache_regression_test_design", "score": 13.0 } ], "trend": { "compatibility_key": "sha256:78765598f3e9349e54ccedc1c7705db7e85d877ebdd8c5c2597d39a879d1f120", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 63.0, "elapsed_ms": 2345486.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 63.0, "elapsed_ms": 2402430.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 64.0, "elapsed_ms": 2322577.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 64.0, "elapsed_ms": 2322577.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 64.0, "elapsed_ms": 2322577.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 64.0, "elapsed_ms": 1413898.0 } ] } }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-luna:xhigh", "rank": 15, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 9.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 17.0 }, { "question_id": "03_ci_optimality_certificate", "score": 14.0 }, { "question_id": "04_transaction_regression_design", "score": 11.0 }, { "question_id": "05_cache_regression_test_design", "score": 13.0 } ], "trend": { "compatibility_key": "sha256:572539b8831e21a5a344216757c00af8fb0c9e631cda31bd45f381b29f609542", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 65.0, "elapsed_ms": 1700941.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 65.0, "elapsed_ms": 1784055.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 68.0, "elapsed_ms": 1661524.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 68.0, "elapsed_ms": 1661524.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 68.0, "elapsed_ms": 1661524.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 64.0, "elapsed_ms": 2369524.0 } ] } }, { "model_configuration_id": "glm:glm-5.3-flash:high", "rank": 16, "target_labels": [ { "id": "lowest_cost", "label": "Lowest cost" } ], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 10.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 14.0 }, { "question_id": "03_ci_optimality_certificate", "score": 14.0 }, { "question_id": "04_transaction_regression_design", "score": 14.0 }, { "question_id": "05_cache_regression_test_design", "score": 11.0 } ], "trend": { "compatibility_key": "sha256:fa8c6d7834602984dd7e069ce04c72c7a1497e37992e66a9e6ad114086a37b81", "sample_count": 2, "comparable": true, "stable_ranking_eligible": false, "points": [ { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 46.0, "elapsed_ms": 384985.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 63.0, "elapsed_ms": 1381291.0 } ] } }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-terra:low", "rank": 17, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 8.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 15.0 }, { "question_id": "03_ci_optimality_certificate", "score": 13.0 }, { "question_id": "04_transaction_regression_design", "score": 15.0 }, { "question_id": "05_cache_regression_test_design", "score": 12.0 } ], "trend": { "compatibility_key": "sha256:705273885c4d8c9940cf48f8af6fea7aa2ffd8b59f362dd0fa8121018583607a", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 43.0, "elapsed_ms": 313786.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 59.0, "elapsed_ms": 360900.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 52.0, "elapsed_ms": 310502.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 52.0, "elapsed_ms": 310502.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 52.0, "elapsed_ms": 310502.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 63.0, "elapsed_ms": 1732392.0 } ] } }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-terra:high", "rank": 18, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 9.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 14.0 }, { "question_id": "03_ci_optimality_certificate", "score": 15.0 }, { "question_id": "04_transaction_regression_design", "score": 11.0 }, { "question_id": "05_cache_regression_test_design", "score": 13.0 } ], "trend": { "compatibility_key": "sha256:28e79f2de0cd5d220c9bac7609011a62baa097d1fcbf9c3d33bdf838483ce69d", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 59.0, "elapsed_ms": 492221.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 65.0, "elapsed_ms": 558254.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 55.0, "elapsed_ms": 542754.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 55.0, "elapsed_ms": 542754.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 55.0, "elapsed_ms": 542754.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 62.0, "elapsed_ms": 2010215.0 } ] } }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-luna:high", "rank": 19, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 7.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 15.0 }, { "question_id": "03_ci_optimality_certificate", "score": 14.0 }, { "question_id": "04_transaction_regression_design", "score": 12.0 }, { "question_id": "05_cache_regression_test_design", "score": 14.0 } ], "trend": { "compatibility_key": "sha256:21697e389df713f314933b10ac10cf662d3c7fdc5bf27d9efcc55b7a0d5d5de7", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 56.0, "elapsed_ms": 838742.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 56.0, "elapsed_ms": 993611.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 56.0, "elapsed_ms": 723820.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 56.0, "elapsed_ms": 723820.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 56.0, "elapsed_ms": 723820.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 62.0, "elapsed_ms": 2287412.0 } ] } }, { "model_configuration_id": "glm:glm-5.3-flash:max", "rank": 20, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 5.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 15.0 }, { "question_id": "03_ci_optimality_certificate", "score": 12.0 }, { "question_id": "04_transaction_regression_design", "score": 16.0 }, { "question_id": "05_cache_regression_test_design", "score": 13.0 } ], "trend": { "compatibility_key": "sha256:4b9194762eeb429e26efdf2d5cccf06272f79184067451532625eed0974efca7", "sample_count": 2, "comparable": true, "stable_ranking_eligible": false, "points": [ { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 59.0, "elapsed_ms": 677501.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 61.0, "elapsed_ms": 1276800.0 } ] } }, { "model_configuration_id": "kimi:k3:high", "rank": 21, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 9.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 13.0 }, { "question_id": "03_ci_optimality_certificate", "score": 12.0 }, { "question_id": "04_transaction_regression_design", "score": 14.0 }, { "question_id": "05_cache_regression_test_design", "score": 13.0 } ], "trend": { "compatibility_key": "sha256:0e2c1f32f6b930b931869723592fb3124561b46a3f8be51e237f68b8e142e2c3", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 64.0, "elapsed_ms": 1474275.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 63.0, "elapsed_ms": 1357171.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 59.0, "elapsed_ms": 1089444.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 59.0, "elapsed_ms": 1089444.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 59.0, "elapsed_ms": 1089444.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 61.0, "elapsed_ms": 1666695.0 } ] } }, { "model_configuration_id": "deepseek:deepseek-v4-flash-vision-exp:high", "rank": 22, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 8.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 13.0 }, { "question_id": "03_ci_optimality_certificate", "score": 13.0 }, { "question_id": "04_transaction_regression_design", "score": 12.0 }, { "question_id": "05_cache_regression_test_design", "score": 11.0 } ], "trend": { "compatibility_key": "sha256:b513088eaa12ac24800d83b50c768ddca5715abdb3fabf15b7b857190a3c4525", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 61.0, "elapsed_ms": 1583947.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 62.0, "elapsed_ms": 1689264.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 59.0, "elapsed_ms": 1587537.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 59.0, "elapsed_ms": 1587537.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 59.0, "elapsed_ms": 1587537.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 57.0, "elapsed_ms": 1309803.0 } ] } }, { "model_configuration_id": "GEMINI:gemini-3.7-flash-high:default", "rank": 23, "target_labels": [ { "id": "fastest", "label": "Fastest" } ], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 10.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 16.0 }, { "question_id": "03_ci_optimality_certificate", "score": 16.0 }, { "question_id": "04_transaction_regression_design", "score": 13.0 }, { "question_id": "05_cache_regression_test_design", "score": 0.0 } ], "trend": { "compatibility_key": "sha256:d0d40cae48e5194924d90e677a73749228ed17feb243e62deeffe6e6d2441a29", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 55.0, "elapsed_ms": 223189.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 51.0, "elapsed_ms": 271043.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 51.0, "elapsed_ms": 217652.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 51.0, "elapsed_ms": 217652.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 51.0, "elapsed_ms": 217652.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 55.0, "elapsed_ms": 457724.0 } ] } }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-luna:low", "rank": 24, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 6.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 14.0 }, { "question_id": "03_ci_optimality_certificate", "score": 10.0 }, { "question_id": "04_transaction_regression_design", "score": 9.0 }, { "question_id": "05_cache_regression_test_design", "score": 11.0 } ], "trend": { "compatibility_key": "sha256:680692bedd1ad91d9318ab9fdcfa39bacd54e826a9cf53aa8917840f473e1941", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 29.0, "elapsed_ms": 258219.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 39.0, "elapsed_ms": 289161.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 43.0, "elapsed_ms": 304284.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 43.0, "elapsed_ms": 304284.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 43.0, "elapsed_ms": 304284.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 50.0, "elapsed_ms": 2047853.0 } ] } }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-terra:medium", "rank": 25, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 10.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 13.0 }, { "question_id": "03_ci_optimality_certificate", "score": 15.0 }, { "question_id": "04_transaction_regression_design", "score": 10.0 }, { "question_id": "05_cache_regression_test_design", "score": 0.0 } ], "trend": { "compatibility_key": "sha256:1a0eb082848083bfc7a02b86d15f33323579e76e76f0592758dcf730a1cb1674", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 57.0, "elapsed_ms": 306625.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 60.0, "elapsed_ms": 366209.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 61.0, "elapsed_ms": 430479.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 61.0, "elapsed_ms": 430479.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 61.0, "elapsed_ms": 430479.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 48.0, "elapsed_ms": 2025399.0 } ] } }, { "model_configuration_id": "minimax:MiniMax-M3:high", "rank": 26, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 2.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 13.0 }, { "question_id": "03_ci_optimality_certificate", "score": 10.0 }, { "question_id": "04_transaction_regression_design", "score": 9.0 }, { "question_id": "05_cache_regression_test_design", "score": 12.0 } ], "trend": { "compatibility_key": "sha256:2f6608cb9662478ab23893b20c38991459a2e80fa7d455e158f09a1bc754001d", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 50.0, "elapsed_ms": 857270.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 47.0, "elapsed_ms": 1591960.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 48.0, "elapsed_ms": 930167.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 48.0, "elapsed_ms": 930167.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 48.0, "elapsed_ms": 930167.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 46.0, "elapsed_ms": 823293.0 } ] } }, { "model_configuration_id": "cloudflare-reference:gpt-5.6-luna:medium", "rank": 27, "target_labels": [], "decision_tags": [], "question_scores": [ { "question_id": "01_session_bundle_repair", "score": 7.0 }, { "question_id": "02_code_counterexample_maxgap", "score": 13.0 }, { "question_id": "03_ci_optimality_certificate", "score": 11.0 }, { "question_id": "04_transaction_regression_design", "score": 0.0 }, { "question_id": "05_cache_regression_test_design", "score": 13.0 } ], "trend": { "compatibility_key": "sha256:cfb92018396048aa711a784675ead958f7bacf4e3ba72d73e66b75aa6ad2803c", "sample_count": 6, "comparable": true, "stable_ranking_eligible": true, "points": [ { "batch_id": "snapshot-2026-08-24T23-00-00Z", "published_at": "2026-08-24T23:00:00Z", "score": 52.0, "elapsed_ms": 433331.0 }, { "batch_id": "snapshot-2026-08-25T05-00-00Z", "published_at": "2026-08-25T05:00:00Z", "score": 45.0, "elapsed_ms": 474963.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z", "published_at": "2026-08-25T23:00:00Z", "score": 52.0, "elapsed_ms": 404225.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r2", "published_at": "2026-08-25T23:00:00Z", "score": 52.0, "elapsed_ms": 404225.0 }, { "batch_id": "snapshot-2026-08-25T23-00-00Z-r3", "published_at": "2026-08-25T23:00:00Z", "score": 52.0, "elapsed_ms": 404225.0 }, { "batch_id": "snapshot-2026-08-26T23-00-00Z-r3", "published_at": "2026-08-26T23:00:00Z", "score": 44.0, "elapsed_ms": 2284255.0 } ] } } ] }, "batch_sha256": "sha256:e4b3b5f4a97ee64b35984b63407b2cb47042d889939da10db54048b95a57dec6", "repricing": { "schema_version": 1, "method": "aggregate_usage_below_long_context_threshold_v1", "source_batch_id": "snapshot-2026-08-26T23-00-00Z", "source_batch_sha256": "sha256:b5174bd2d5fd60dac51cc4548fbbfc94eb72135636365eda0b2525c3372d5adb", "source_pricing_snapshot_id": "pricing-v1-084c3aa793505ecc6bfcbeb5d8c6dab9c7c7361660599fe5017b24a9988844d5", "target_pricing_snapshot_id": "pricing-v1-2db5881e647a00cd9b1537918236db7b4696805a74f9ba897e97ee78eee8c80f", "repriced_at": "2026-08-27T00:35:15Z" }, "supplement_composition": { "schema_version": 2, "base_batch_id": "snapshot-2026-08-26T23-00-00Z-r2", "base_batch_sha256": "sha256:b48991618a77b1287e324d45eb9bd66d6be11e00a978e9e801ed68f2e79281af", "base_revision": 2, "supplement_batch_id": "snapshot-2026-08-26T23-00-00Z", "supplement_batch_sha256": "sha256:209f93408cc59bec524b1515699217295a78a8d1905b4472b3e189ec429dae3d", "added_configuration_ids": [ "grok-api:grok-4.6:high" ], "source_execution_policies": [ { "schema_version": 1, "mode": "app_rules_v1", "execution_timeout_seconds": 1800, "timeout_retry_count": 1, "max_attempts_per_question": 6, "selective_score_retry": false, "rules": { "missing_usage": { "enabled": true, "action": "retry", "max_retries": 1, "cooldown_seconds": 0 }, "reason_tok_516": { "enabled": true, "action": "retry", "max_retries": 1, "cooldown_seconds": 0 }, "slow_response": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 }, "timeout": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 }, "wrong_answer": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 } }, "max_concurrent_targets": 16, "max_concurrent_targets_by_connection": { "GEMINI": 8, "cloudflare-reference": 8, "deepseek": 8, "glm": 5, "grok-api": 4, "kimi": 1, "minimax": 8, "qwen": 5 }, "execution_cycle_count": 3, "budget_mode": "policy_segments_sum", "policy_segments": [ { "policy_sha256": "sha256:188337ed03309ed60f6d194f1b61ad3e5d51a3c142f03bbf14df53b7f94217a8", "cycle_count": 3, "execution_timeout_seconds": 1800, "timeout_retry_count": 1, "max_attempts_per_question": 2, "selective_score_retry": false, "rules": { "missing_usage": { "enabled": true, "action": "retry", "max_retries": 1, "cooldown_seconds": 0 }, "reason_tok_516": { "enabled": true, "action": "retry", "max_retries": 1, "cooldown_seconds": 0 }, "slow_response": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 }, "timeout": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 }, "wrong_answer": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 } } } ], "max_attempts_per_question_per_cycle": 2 }, { "schema_version": 1, "mode": "app_rules_v1", "execution_timeout_seconds": 1800, "timeout_retry_count": 1, "max_attempts_per_question": 2, "selective_score_retry": false, "rules": { "missing_usage": { "enabled": true, "action": "retry", "max_retries": 1, "cooldown_seconds": 0 }, "reason_tok_516": { "enabled": true, "action": "retry", "max_retries": 1, "cooldown_seconds": 0 }, "slow_response": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 }, "timeout": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 }, "wrong_answer": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 } }, "max_concurrent_targets": 16, "max_concurrent_targets_by_connection": { "cloudflare-reference": 8, "grok-api": 2 }, "execution_cycle_count": 1, "budget_mode": "policy_segments_sum", "policy_segments": [ { "policy_sha256": "sha256:fede70183988b8dbad108e3df3868200f837fbf20f0a1138794b49d853f6d7b1", "cycle_count": 1, "execution_timeout_seconds": 1800, "timeout_retry_count": 1, "max_attempts_per_question": 2, "selective_score_retry": false, "rules": { "missing_usage": { "enabled": true, "action": "retry", "max_retries": 1, "cooldown_seconds": 0 }, "reason_tok_516": { "enabled": true, "action": "retry", "max_retries": 1, "cooldown_seconds": 0 }, "slow_response": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 }, "timeout": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 }, "wrong_answer": { "enabled": true, "action": "warn", "max_retries": 0, "cooldown_seconds": 0 } } } ], "max_attempts_per_question_per_cycle": 2 } ], "source_evaluation_identities": [ "sha256:10d3c4df36eaf5e325cc2b4b3dea81686c4be9b3a5ea5f64bd91b5b3bf373af8", "sha256:e2f05d9a83598d907bfcbf9226988f46caf0499c19fd0e182a3ccc82a8958d81" ], "source_runner_commits": [ "91bc45b3cde120774d0c7f55d2bc2dd3593c5979+runner.sha256.ba159796d8d432bf56d204cf0bb46a41e2635f4f25c467092b8b246c4be1440f", "91bc45b3cde120774d0c7f55d2bc2dd3593c5979+runner.sha256.b3751f10726c439d5f7d558653d254531c05999fe531237bb73e5e1fb7b8a215" ], "source_pricing_snapshot_ids": [ "pricing-v1-2db5881e647a00cd9b1537918236db7b4696805a74f9ba897e97ee78eee8c80f", "pricing-v1-2db5881e647a00cd9b1537918236db7b4696805a74f9ba897e97ee78eee8c80f" ], "source_environments": [ { "os": "Linux amd64", "app_version": "cloud-runner-v1", "codex_version": "codex-cli 0.146.0", "machine_profile": "cloudflare-container-linux-amd64-codex-cli-v1" }, { "os": "macOS 15.4 arm64", "app_version": "cloud-runner-v1", "codex_version": "grok-cli-1.0.5", "machine_profile": "macos-arm64-grok-build-endpoint-v1" } ] } }