{ "schema_version": 1, "sample": false, "generated_at": "2026-10-31T20:09:43.009925+00:00", "next_run_at": "2026-11-03T20:30:00+00:00", "schedule": "Every 3 days", "schedule_status": "active", "operator": "Outrun", "endpoint": "https://api.outrun.to/v1", "note": "Real Outrun-operated checks using Meow LLM Detector. Behavioral matches are not identity authentication.", "tool": { "name": "Meow LLM Detector", "version": "4.5.4", "site": "https://meowllm.top/", "repo": "https://github.com/chen-006/meow-llm-detector", "commit": "dc608d5c097abf96575d5b34f49952ddbd27c281" }, "models": [ { "id": "gpt-6.1-sol", "name": "GPT-6.1 Sol", "page": "/model/gpt-6.1-sol", "family": "gpt", "expected": "GPT-6.1 Sol", "run_id": "45c1fe943ccb", "session_id": "45c1fe943ccb4af2a1bb5463a187b4d7", "ran_at": "2026-10-31T19:05:08.866516+00:00", "verdict": "match", "protocol": "Chat Completions", "planned": 128, "valid": 128, "errors": 0, "score": 0.5442121415489577, "line": 0.5007278427691376, "closest": "GPT-6.1 Sol", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-chat.20261003.1", "tier": "Deep, 128 requests", "content_sha256": "5df2963f9e6d7791f36cc273cba6eb14330519d052a35588bebb1ed1664ff1ab" }, "candidates": [ { "id": "gpt-6.1-sol", "name": "GPT-6.1 Sol", "score": 0.5442121415489577, "line": 0.5007278427691376 }, { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.45578785845104225, "line": 0.5007278427691376 }, { "id": "gpt-6-luna", "name": "GPT-6 Luna", "score": 0.4413571553786566, "line": 0.5007278427691376 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.4364723540684961, "line": 0.5068073175882207 }, { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.22982579358127125, "line": 0.5007278427691376 } ], "probes": [ { "id": "gpt__screen067", "prompt": "不要思考，直接回答。随便选一个标点符号，只回答符号。", "english": null, "requests": 64, "valid": 64, "answers": [ [ "！", 39 ], [ "。", 10 ], [ "，", 10 ], [ "；", 3 ], [ "!", 1 ], [ "】【。", 1 ] ] }, { "id": "gpt__screen101", "prompt": "Name a random country. Reply with ONLY the country name.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Uruguay", 13 ], [ "Namibia", 2 ], [ "Portugal", 1 ] ] }, { "id": "gpt__screen108", "prompt": "从17到83中随便选一个整数，只回答数字。", "english": null, "requests": 48, "valid": 48, "answers": [ [ "47", 40 ], [ "46", 6 ], [ "42", 1 ], [ "59", 1 ] ] } ], "history": [ { "run_id": "a8e06ead1e3a", "session_id": "a8e06ead1e3a4bddad68d381dd07ef7b", "ran_at": "2026-10-31T18:09:13.707125+00:00", "verdict": "mismatch", "protocol": "Responses", "planned": 128, "valid": 128, "errors": 0, "score": 0.4760844143989615, "line": 0.5007278427691376, "closest": "GPT-6 Astra", "expected": "GPT-6.1 Sol", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-predictive.20261003.1", "tier": "Deep, 128 requests", "content_sha256": "ff04ec844b5bd63c189748cff4caaf039be9cb908f398cb6c279efab9e8c4aec" }, "candidates": [ { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.5239155856010386, "line": 0.5007278427691376 }, { "id": "gpt-6.1-sol", "name": "GPT-6.1 Sol", "score": 0.4760844143989615, "line": 0.5007278427691376 }, { "id": "gpt-6-luna", "name": "GPT-6 Luna", "score": 0.46374106390609016, "line": 0.5007278427691376 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.4578610096085694, "line": 0.5068073175882207 }, { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.27066090508340807, "line": 0.5007278427691376 } ], "probes": [ { "id": "gpt__screen067", "prompt": "不要思考，直接回答。随便选一个标点符号，只回答符号。", "english": null, "requests": 64, "valid": 64, "answers": [ [ "！", 57 ], [ "。", 5 ], [ "；", 1 ], [ "？", 1 ] ] }, { "id": "gpt__screen101", "prompt": "Name a random country. Reply with ONLY the country name.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Bhutan", 6 ], [ "Namibia", 4 ], [ "Botswana", 2 ], [ "Canada", 2 ], [ "Japan", 1 ], [ "Mongolia", 1 ] ] }, { "id": "gpt__screen108", "prompt": "从17到83中随便选一个整数，只回答数字。", "english": null, "requests": 48, "valid": 48, "answers": [ [ "47", 28 ], [ "46", 16 ], [ "42", 3 ], [ "49", 1 ] ] } ] }, { "run_id": "426187b4ed4b", "session_id": "426187b4ed4b43d38cd2dd26a6f9af99", "ran_at": "2026-10-31T19:05:08.726229+00:00", "verdict": "mismatch", "protocol": "Responses", "planned": 128, "valid": 127, "errors": 1, "score": 0.2946570751966475, "line": 0.5007278427691376, "closest": "Other reference models", "expected": "GPT-6.1 Sol", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-predictive.20261003.1", "tier": "Deep, 128 requests", "content_sha256": "ff04ec844b5bd63c189748cff4caaf039be9cb908f398cb6c279efab9e8c4aec" }, "candidates": [ { "id": "other_known_external", "name": "Other reference models", "score": 0.5861612043791988, "line": 0.5068073175882207 }, { "id": "gpt-6-luna", "name": "GPT-6 Luna", "score": 0.4138387956208011, "line": 0.5007278427691376 }, { "id": "gpt-6.1-sol", "name": "GPT-6.1 Sol", "score": 0.2946570751966475, "line": 0.5007278427691376 }, { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.27422315656402124, "line": 0.5007278427691376 }, { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.23936714678987683, "line": 0.5007278427691376 } ], "probes": [ { "id": "gpt__screen067", "prompt": "不要思考，直接回答。随便选一个标点符号，只回答符号。", "english": null, "requests": 64, "valid": 64, "answers": [ [ "！", 42 ], [ "，", 11 ], [ "。", 7 ], [ "!", 2 ], [ "】【。", 1 ], [ "？", 1 ] ] }, { "id": "gpt__screen101", "prompt": "Name a random country. Reply with ONLY the country name.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Mongolia", 8 ], [ "Bhutan", 7 ], [ "Madagascar", 1 ] ] }, { "id": "gpt__screen108", "prompt": "从17到83中随便选一个整数，只回答数字。", "english": null, "requests": 48, "valid": 47, "answers": [ [ "52", 39 ], [ "56", 3 ], [ "47", 2 ], [ "46", 1 ], [ "57", 1 ], [ "58", 1 ] ] } ] }, { "expected": "GPT-6.1 Sol", "run_id": "45c1fe943ccb", "session_id": "45c1fe943ccb4af2a1bb5463a187b4d7", "ran_at": "2026-10-31T19:05:08.866516+00:00", "verdict": "match", "protocol": "Chat Completions", "planned": 128, "valid": 128, "errors": 0, "score": 0.5442121415489577, "line": 0.5007278427691376, "closest": "GPT-6.1 Sol", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-chat.20261003.1", "tier": "Deep, 128 requests", "content_sha256": "5df2963f9e6d7791f36cc273cba6eb14330519d052a35588bebb1ed1664ff1ab" }, "candidates": [ { "id": "gpt-6.1-sol", "name": "GPT-6.1 Sol", "score": 0.5442121415489577, "line": 0.5007278427691376 }, { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.45578785845104225, "line": 0.5007278427691376 }, { "id": "gpt-6-luna", "name": "GPT-6 Luna", "score": 0.4413571553786566, "line": 0.5007278427691376 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.4364723540684961, "line": 0.5068073175882207 }, { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.22982579358127125, "line": 0.5007278427691376 } ], "probes": [ { "id": "gpt__screen067", "prompt": "不要思考，直接回答。随便选一个标点符号，只回答符号。", "english": null, "requests": 64, "valid": 64, "answers": [ [ "！", 39 ], [ "。", 10 ], [ "，", 10 ], [ "；", 3 ], [ "!", 1 ], [ "】【。", 1 ] ] }, { "id": "gpt__screen101", "prompt": "Name a random country. Reply with ONLY the country name.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Uruguay", 13 ], [ "Namibia", 2 ], [ "Portugal", 1 ] ] }, { "id": "gpt__screen108", "prompt": "从17到83中随便选一个整数，只回答数字。", "english": null, "requests": 48, "valid": 48, "answers": [ [ "47", 40 ], [ "46", 6 ], [ "42", 1 ], [ "59", 1 ] ] } ] } ] }, { "id": "gpt-6-astra", "name": "GPT-6 Astra", "page": "/model/gpt-6-astra", "family": "gpt", "expected": "GPT-6 Astra", "run_id": "647ed38cd97a", "session_id": "647ed38cd97a4b8eb6c2490bcc4ee205", "ran_at": "2026-10-31T18:40:45.717452+00:00", "verdict": "inconsistent", "protocol": "Responses", "planned": 128, "valid": 114, "errors": 14, "score": 0.4915730205871524, "line": 0.5007278427691376, "closest": "Other reference models", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-predictive.20261003.1", "tier": "Deep, 128 requests", "content_sha256": "ff04ec844b5bd63c189748cff4caaf039be9cb908f398cb6c279efab9e8c4aec" }, "candidates": [ { "id": "other_known_external", "name": "Other reference models", "score": 0.5084269794128475, "line": 0.5068073175882207 }, { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.4915730205871524, "line": 0.5007278427691376 }, { "id": "gpt-6-luna", "name": "GPT-6 Luna", "score": 0.4675431533099036, "line": 0.5007278427691376 }, { "id": "gpt-6.1-sol", "name": "GPT-6.1 Sol", "score": 0.4482012388788464, "line": 0.5007278427691376 }, { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.28765573742882394, "line": 0.5007278427691376 } ], "probes": [ { "id": "gpt__screen067", "prompt": "不要思考，直接回答。随便选一个标点符号，只回答符号。", "english": null, "requests": 64, "valid": 55, "answers": [ [ "！", 51 ], [ "。", 3 ], [ "？", 1 ] ] }, { "id": "gpt__screen101", "prompt": "Name a random country. Reply with ONLY the country name.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Bhutan", 7 ], [ "Mongolia", 4 ], [ "Madagascar", 2 ], [ "Japan", 1 ], [ "Kenya", 1 ], [ "Namibia", 1 ] ] }, { "id": "gpt__screen108", "prompt": "从17到83中随便选一个整数，只回答数字。", "english": null, "requests": 48, "valid": 43, "answers": [ [ "46", 19 ], [ "47", 13 ], [ "52", 7 ], [ "42", 2 ], [ "58", 2 ] ] } ], "history": [ { "run_id": "206274a09123", "session_id": "206274a09123415b8015797915a90f57", "ran_at": "2026-10-31T18:15:16.758745+00:00", "verdict": "match", "protocol": "Responses", "planned": 128, "valid": 128, "errors": 0, "score": 0.5560858873503326, "line": 0.5007278427691376, "closest": "GPT-6 Astra", "expected": "GPT-6 Astra", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-predictive.20261003.1", "tier": "Deep, 128 requests", "content_sha256": "ff04ec844b5bd63c189748cff4caaf039be9cb908f398cb6c279efab9e8c4aec" }, "candidates": [ { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.5560858873503326, "line": 0.5007278427691376 }, { "id": "gpt-6.1-sol", "name": "GPT-6.1 Sol", "score": 0.4439141126496674, "line": 0.5007278427691376 }, { "id": "gpt-6-luna", "name": "GPT-6 Luna", "score": 0.43484920017560125, "line": 0.5007278427691376 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.4221500795181847, "line": 0.5068073175882207 }, { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.24900266207236368, "line": 0.5007278427691376 } ], "probes": [ { "id": "gpt__screen067", "prompt": "不要思考，直接回答。随便选一个标点符号，只回答符号。", "english": null, "requests": 64, "valid": 64, "answers": [ [ "！", 61 ], [ "。", 2 ], [ "？", 1 ] ] }, { "id": "gpt__screen101", "prompt": "Name a random country. Reply with ONLY the country name.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Bhutan", 4 ], [ "Kenya", 4 ], [ "Namibia", 4 ], [ "Nepal", 2 ], [ "Chile", 1 ], [ "Laos", 1 ] ] }, { "id": "gpt__screen108", "prompt": "从17到83中随便选一个整数，只回答数字。", "english": null, "requests": 48, "valid": 48, "answers": [ [ "46", 34 ], [ "47", 10 ], [ "49", 2 ], [ "42", 1 ], [ "56", 1 ] ] } ] }, { "run_id": "647ed38cd97a", "session_id": "647ed38cd97a4b8eb6c2490bcc4ee205", "ran_at": "2026-10-31T18:40:45.717452+00:00", "verdict": "mismatch", "protocol": "Responses", "planned": 128, "valid": 114, "errors": 14, "score": 0.4915730205871524, "line": 0.5007278427691376, "closest": "Other reference models", "expected": "GPT-6 Astra", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-predictive.20261003.1", "tier": "Deep, 128 requests", "content_sha256": "ff04ec844b5bd63c189748cff4caaf039be9cb908f398cb6c279efab9e8c4aec" }, "candidates": [ { "id": "other_known_external", "name": "Other reference models", "score": 0.5084269794128475, "line": 0.5068073175882207 }, { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.4915730205871524, "line": 0.5007278427691376 }, { "id": "gpt-6-luna", "name": "GPT-6 Luna", "score": 0.4675431533099036, "line": 0.5007278427691376 }, { "id": "gpt-6.1-sol", "name": "GPT-6.1 Sol", "score": 0.4482012388788464, "line": 0.5007278427691376 }, { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.28765573742882394, "line": 0.5007278427691376 } ], "probes": [ { "id": "gpt__screen067", "prompt": "不要思考，直接回答。随便选一个标点符号，只回答符号。", "english": null, "requests": 64, "valid": 55, "answers": [ [ "！", 51 ], [ "。", 3 ], [ "？", 1 ] ] }, { "id": "gpt__screen101", "prompt": "Name a random country. Reply with ONLY the country name.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Bhutan", 7 ], [ "Mongolia", 4 ], [ "Madagascar", 2 ], [ "Japan", 1 ], [ "Kenya", 1 ], [ "Namibia", 1 ] ] }, { "id": "gpt__screen108", "prompt": "从17到83中随便选一个整数，只回答数字。", "english": null, "requests": 48, "valid": 43, "answers": [ [ "46", 19 ], [ "47", 13 ], [ "52", 7 ], [ "42", 2 ], [ "58", 2 ] ] } ] } ], "review_note": "Repeated checks produced conflicting results. We are investigating; the recorded runs remain available below." }, { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "page": "/model/gpt-5.6-terra", "family": "gpt", "expected": "GPT-5.6 Terra", "run_id": "964e4a6e6830", "session_id": "964e4a6e6830414ca5d0b12c331323f6", "ran_at": "2026-10-31T18:20:29.040789+00:00", "verdict": "match", "protocol": "Responses", "planned": 128, "valid": 128, "errors": 0, "score": 0.5491255424271843, "line": 0.5007278427691376, "closest": "GPT-5.6 Terra", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-predictive.20261003.1", "tier": "Deep, 128 requests", "content_sha256": "ff04ec844b5bd63c189748cff4caaf039be9cb908f398cb6c279efab9e8c4aec" }, "candidates": [ { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.5491255424271843, "line": 0.5007278427691376 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.45087445757281575, "line": 0.5068073175882207 }, { "id": "gpt-6-luna", "name": "GPT-6 Luna", "score": 0.44100242344128404, "line": 0.5007278427691376 }, { "id": "gpt-6.1-sol", "name": "GPT-6.1 Sol", "score": 0.36068577539494184, "line": 0.5007278427691376 }, { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.28267851928522597, "line": 0.5007278427691376 } ], "probes": [ { "id": "gpt__screen067", "prompt": "不要思考，直接回答。随便选一个标点符号，只回答符号。", "english": null, "requests": 64, "valid": 64, "answers": [ [ "。", 35 ], [ "？", 27 ], [ "！", 2 ] ] }, { "id": "gpt__screen101", "prompt": "Name a random country. Reply with ONLY the country name.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Nepal", 4 ], [ "Canada", 3 ], [ "Namibia", 3 ], [ "Bhutan", 2 ], [ "Mongolia", 2 ], [ "Peru", 1 ], [ "Uruguay", 1 ] ] }, { "id": "gpt__screen108", "prompt": "从17到83中随便选一个整数，只回答数字。", "english": null, "requests": 48, "valid": 48, "answers": [ [ "58", 18 ], [ "64", 8 ], [ "42", 7 ], [ "52", 5 ], [ "62", 4 ], [ "56", 3 ], [ "57", 2 ], [ "47", 1 ] ] } ], "history": [ { "expected": "GPT-5.6 Terra", "run_id": "964e4a6e6830", "session_id": "964e4a6e6830414ca5d0b12c331323f6", "ran_at": "2026-10-31T18:20:29.040789+00:00", "verdict": "match", "protocol": "Responses", "planned": 128, "valid": 128, "errors": 0, "score": 0.5491255424271843, "line": 0.5007278427691376, "closest": "GPT-5.6 Terra", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-predictive.20261003.1", "tier": "Deep, 128 requests", "content_sha256": "ff04ec844b5bd63c189748cff4caaf039be9cb908f398cb6c279efab9e8c4aec" }, "candidates": [ { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.5491255424271843, "line": 0.5007278427691376 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.45087445757281575, "line": 0.5068073175882207 }, { "id": "gpt-6-luna", "name": "GPT-6 Luna", "score": 0.44100242344128404, "line": 0.5007278427691376 }, { "id": "gpt-6.1-sol", "name": "GPT-6.1 Sol", "score": 0.36068577539494184, "line": 0.5007278427691376 }, { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.28267851928522597, "line": 0.5007278427691376 } ], "probes": [ { "id": "gpt__screen067", "prompt": "不要思考，直接回答。随便选一个标点符号，只回答符号。", "english": null, "requests": 64, "valid": 64, "answers": [ [ "。", 35 ], [ "？", 27 ], [ "！", 2 ] ] }, { "id": "gpt__screen101", "prompt": "Name a random country. Reply with ONLY the country name.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Nepal", 4 ], [ "Canada", 3 ], [ "Namibia", 3 ], [ "Bhutan", 2 ], [ "Mongolia", 2 ], [ "Peru", 1 ], [ "Uruguay", 1 ] ] }, { "id": "gpt__screen108", "prompt": "从17到83中随便选一个整数，只回答数字。", "english": null, "requests": 48, "valid": 48, "answers": [ [ "58", 18 ], [ "64", 8 ], [ "42", 7 ], [ "52", 5 ], [ "62", 4 ], [ "56", 3 ], [ "57", 2 ], [ "47", 1 ] ] } ] } ] }, { "id": "claude-fable-5-1", "name": "Claude Fable 5.1", "page": "/model/claude-fable-5-1", "family": "claude", "expected": "Claude Fable 5.1", "run_id": "4486c456a64e", "session_id": "4486c456a64e444d90192b3bed9f096d", "ran_at": "2026-10-31T19:15:39.319402+00:00", "verdict": "match", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.5495556433999296, "line": 0.5, "closest": "Claude Fable 5.1", "baseline": { "name": "Claude reference benchmark", "version": "4.5.4-predictive.20260924.1", "tier": "Deep, 120 requests", "content_sha256": "7d61ae9aba3b81b41de1b0dbd2caeb71ea2c53e85511959e160f9ef853d98322" }, "candidates": [ { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.5495556433999296, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.45044435660007043, "line": 0.5 }, { "id": "claude-opus-5.5", "name": "Claude Opus 5.5", "score": 0.39328975271032296, "line": 0.5 }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.37380689642136616, "line": 0.5 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.23353203928490066, "line": 0.5 } ], "probes": [ { "id": "claude__screen008", "prompt": "不要思考，直接回答。从当、咚、铛、叮中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "咚", 16 ], [ "叮", 12 ], [ "铛", 2 ] ] }, { "id": "claude__screen021", "prompt": "不要思考，直接回答。从川、宁、禾、序中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "禾", 21 ], [ "宁", 9 ] ] }, { "id": "claude__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "47", 24 ], [ "42", 5 ], [ "43", 1 ] ] }, { "id": "claude__screen048", "prompt": "不要思考，直接回答。随便选一张麻将牌，只回答牌名。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "五万", 13 ], [ "五筒", 7 ], [ "三万", 3 ], [ "五萬", 3 ], [ "一万", 1 ], [ "七万", 1 ], [ "九万", 1 ], [ "八万", 1 ] ] } ], "history": [ { "expected": "Claude Fable 5.1", "run_id": "4486c456a64e", "session_id": "4486c456a64e444d90192b3bed9f096d", "ran_at": "2026-10-31T19:15:39.319402+00:00", "verdict": "match", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.5495556433999296, "line": 0.5, "closest": "Claude Fable 5.1", "baseline": { "name": "Claude reference benchmark", "version": "4.5.4-predictive.20260924.1", "tier": "Deep, 120 requests", "content_sha256": "7d61ae9aba3b81b41de1b0dbd2caeb71ea2c53e85511959e160f9ef853d98322" }, "candidates": [ { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.5495556433999296, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.45044435660007043, "line": 0.5 }, { "id": "claude-opus-5.5", "name": "Claude Opus 5.5", "score": 0.39328975271032296, "line": 0.5 }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.37380689642136616, "line": 0.5 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.23353203928490066, "line": 0.5 } ], "probes": [ { "id": "claude__screen008", "prompt": "不要思考，直接回答。从当、咚、铛、叮中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "咚", 16 ], [ "叮", 12 ], [ "铛", 2 ] ] }, { "id": "claude__screen021", "prompt": "不要思考，直接回答。从川、宁、禾、序中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "禾", 21 ], [ "宁", 9 ] ] }, { "id": "claude__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "47", 24 ], [ "42", 5 ], [ "43", 1 ] ] }, { "id": "claude__screen048", "prompt": "不要思考，直接回答。随便选一张麻将牌，只回答牌名。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "五万", 13 ], [ "五筒", 7 ], [ "三万", 3 ], [ "五萬", 3 ], [ "一万", 1 ], [ "七万", 1 ], [ "九万", 1 ], [ "八万", 1 ] ] } ] } ] }, { "id": "claude-fable-5-1-2", "name": "Claude Fable 5.1 (2)", "page": "/model/claude-fable-5-1", "family": "claude", "expected": "Claude Fable 5.1", "run_id": "0ed8f3d964a5", "session_id": "0ed8f3d964a5468687ee6cf86f0daf60", "ran_at": "2026-10-31T19:24:40.572377+00:00", "verdict": "match", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.5467648762072775, "line": 0.5, "closest": "Claude Fable 5.1", "baseline": { "name": "Claude reference benchmark", "version": "4.5.4-predictive.20260924.1", "tier": "Deep, 120 requests", "content_sha256": "7d61ae9aba3b81b41de1b0dbd2caeb71ea2c53e85511959e160f9ef853d98322" }, "candidates": [ { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.5467648762072775, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.4532351237927224, "line": 0.5 }, { "id": "claude-opus-5.5", "name": "Claude Opus 5.5", "score": 0.44391147776182976, "line": 0.5 }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.33921259463906034, "line": 0.5 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.3068977211202807, "line": 0.5 } ], "probes": [ { "id": "claude__screen008", "prompt": "不要思考，直接回答。从当、咚、铛、叮中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "咚", 24 ], [ "叮", 3 ], [ "当", 2 ], [ "铛", 1 ] ] }, { "id": "claude__screen021", "prompt": "不要思考，直接回答。从川、宁、禾、序中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "宁", 16 ], [ "禾", 8 ], [ "川", 6 ] ] }, { "id": "claude__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "42", 16 ], [ "47", 10 ], [ "43", 4 ] ] }, { "id": "claude__screen048", "prompt": "不要思考，直接回答。随便选一张麻将牌，只回答牌名。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "五万", 19 ], [ "五萬", 6 ], [ "五筒", 3 ], [ "三万", 1 ], [ "九筒", 1 ] ] } ], "history": [ { "expected": "Claude Fable 5.1", "run_id": "0ed8f3d964a5", "session_id": "0ed8f3d964a5468687ee6cf86f0daf60", "ran_at": "2026-10-31T19:24:40.572377+00:00", "verdict": "match", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.5467648762072775, "line": 0.5, "closest": "Claude Fable 5.1", "baseline": { "name": "Claude reference benchmark", "version": "4.5.4-predictive.20260924.1", "tier": "Deep, 120 requests", "content_sha256": "7d61ae9aba3b81b41de1b0dbd2caeb71ea2c53e85511959e160f9ef853d98322" }, "candidates": [ { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.5467648762072775, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.4532351237927224, "line": 0.5 }, { "id": "claude-opus-5.5", "name": "Claude Opus 5.5", "score": 0.44391147776182976, "line": 0.5 }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.33921259463906034, "line": 0.5 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.3068977211202807, "line": 0.5 } ], "probes": [ { "id": "claude__screen008", "prompt": "不要思考，直接回答。从当、咚、铛、叮中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "咚", 24 ], [ "叮", 3 ], [ "当", 2 ], [ "铛", 1 ] ] }, { "id": "claude__screen021", "prompt": "不要思考，直接回答。从川、宁、禾、序中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "宁", 16 ], [ "禾", 8 ], [ "川", 6 ] ] }, { "id": "claude__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "42", 16 ], [ "47", 10 ], [ "43", 4 ] ] }, { "id": "claude__screen048", "prompt": "不要思考，直接回答。随便选一张麻将牌，只回答牌名。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "五万", 19 ], [ "五萬", 6 ], [ "五筒", 3 ], [ "三万", 1 ], [ "九筒", 1 ] ] } ] } ] }, { "id": "claude-opus-5-5", "name": "Claude Opus 5.5", "page": "/model/claude-opus-5-5", "family": "claude", "expected": "Claude Opus 5.5", "run_id": "446217cc35e5", "session_id": "446217cc35e5470bae4246acc0f920a9", "ran_at": "2026-10-31T18:25:03.735230+00:00", "verdict": "match", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.5886884231741143, "line": 0.5, "closest": "Claude Opus 5.5", "baseline": { "name": "Claude reference benchmark", "version": "4.5.4-predictive.20260924.1", "tier": "Deep, 120 requests", "content_sha256": "7d61ae9aba3b81b41de1b0dbd2caeb71ea2c53e85511959e160f9ef853d98322" }, "candidates": [ { "id": "claude-opus-5.5", "name": "Claude Opus 5.5", "score": 0.5886884231741143, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.4113115768258857, "line": 0.5 }, { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.39449587292301397, "line": 0.5 }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.2965754351637796, "line": 0.5 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.2646141963458414, "line": 0.5 } ], "probes": [ { "id": "claude__screen008", "prompt": "不要思考，直接回答。从当、咚、铛、叮中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "叮", 27 ], [ "咚", 1 ], [ "当", 1 ], [ "铛", 1 ] ] }, { "id": "claude__screen021", "prompt": "不要思考，直接回答。从川、宁、禾、序中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "宁", 30 ] ] }, { "id": "claude__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "42", 28 ], [ "37", 2 ] ] }, { "id": "claude__screen048", "prompt": "不要思考，直接回答。随便选一张麻将牌，只回答牌名。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "五万", 17 ], [ "五筒", 5 ], [ "三万", 2 ], [ "五条", 2 ], [ "红中", 2 ], [ "三条", 1 ], [ "九筒", 1 ] ] } ], "history": [ { "expected": "Claude Opus 5.5", "run_id": "446217cc35e5", "session_id": "446217cc35e5470bae4246acc0f920a9", "ran_at": "2026-10-31T18:25:03.735230+00:00", "verdict": "match", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.5886884231741143, "line": 0.5, "closest": "Claude Opus 5.5", "baseline": { "name": "Claude reference benchmark", "version": "4.5.4-predictive.20260924.1", "tier": "Deep, 120 requests", "content_sha256": "7d61ae9aba3b81b41de1b0dbd2caeb71ea2c53e85511959e160f9ef853d98322" }, "candidates": [ { "id": "claude-opus-5.5", "name": "Claude Opus 5.5", "score": 0.5886884231741143, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.4113115768258857, "line": 0.5 }, { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.39449587292301397, "line": 0.5 }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.2965754351637796, "line": 0.5 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.2646141963458414, "line": 0.5 } ], "probes": [ { "id": "claude__screen008", "prompt": "不要思考，直接回答。从当、咚、铛、叮中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "叮", 27 ], [ "咚", 1 ], [ "当", 1 ], [ "铛", 1 ] ] }, { "id": "claude__screen021", "prompt": "不要思考，直接回答。从川、宁、禾、序中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "宁", 30 ] ] }, { "id": "claude__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "42", 28 ], [ "37", 2 ] ] }, { "id": "claude__screen048", "prompt": "不要思考，直接回答。随便选一张麻将牌，只回答牌名。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "五万", 17 ], [ "五筒", 5 ], [ "三万", 2 ], [ "五条", 2 ], [ "红中", 2 ], [ "三条", 1 ], [ "九筒", 1 ] ] } ] } ] }, { "id": "claude-opus-5-5-2", "name": "Claude Opus 5.5 (2)", "page": "/model/claude-opus-5-5", "family": "claude", "expected": "Claude Opus 5.5", "run_id": "475e4f8282c8", "session_id": "475e4f8282c840499225c031756ac2a5", "ran_at": "2026-10-31T19:20:11.033292+00:00", "verdict": "match", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.5927633089489811, "line": 0.5, "closest": "Claude Opus 5.5", "baseline": { "name": "Claude reference benchmark", "version": "4.5.4-predictive.20260924.1", "tier": "Deep, 120 requests", "content_sha256": "7d61ae9aba3b81b41de1b0dbd2caeb71ea2c53e85511959e160f9ef853d98322" }, "candidates": [ { "id": "claude-opus-5.5", "name": "Claude Opus 5.5", "score": 0.5927633089489811, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.4072366910510188, "line": 0.5 }, { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.4029634676944053, "line": 0.5 }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.32534449274917815, "line": 0.5 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.2739921054344877, "line": 0.5 } ], "probes": [ { "id": "claude__screen008", "prompt": "不要思考，直接回答。从当、咚、铛、叮中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "叮", 29 ], [ "铛", 1 ] ] }, { "id": "claude__screen021", "prompt": "不要思考，直接回答。从川、宁、禾、序中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "宁", 30 ] ] }, { "id": "claude__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "42", 29 ], [ "43", 1 ] ] }, { "id": "claude__screen048", "prompt": "不要思考，直接回答。随便选一张麻将牌，只回答牌名。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "五万", 24 ], [ "五条", 3 ], [ "三万", 2 ], [ "红中", 1 ] ] } ], "history": [ { "expected": "Claude Opus 5.5", "run_id": "475e4f8282c8", "session_id": "475e4f8282c840499225c031756ac2a5", "ran_at": "2026-10-31T19:20:11.033292+00:00", "verdict": "match", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.5927633089489811, "line": 0.5, "closest": "Claude Opus 5.5", "baseline": { "name": "Claude reference benchmark", "version": "4.5.4-predictive.20260924.1", "tier": "Deep, 120 requests", "content_sha256": "7d61ae9aba3b81b41de1b0dbd2caeb71ea2c53e85511959e160f9ef853d98322" }, "candidates": [ { "id": "claude-opus-5.5", "name": "Claude Opus 5.5", "score": 0.5927633089489811, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.4072366910510188, "line": 0.5 }, { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.4029634676944053, "line": 0.5 }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.32534449274917815, "line": 0.5 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.2739921054344877, "line": 0.5 } ], "probes": [ { "id": "claude__screen008", "prompt": "不要思考，直接回答。从当、咚、铛、叮中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "叮", 29 ], [ "铛", 1 ] ] }, { "id": "claude__screen021", "prompt": "不要思考，直接回答。从川、宁、禾、序中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "宁", 30 ] ] }, { "id": "claude__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "42", 29 ], [ "43", 1 ] ] }, { "id": "claude__screen048", "prompt": "不要思考，直接回答。随便选一张麻将牌，只回答牌名。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "五万", 24 ], [ "五条", 3 ], [ "三万", 2 ], [ "红中", 1 ] ] } ] } ] }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "page": "/model/claude-sonnet-5", "family": "claude", "expected": "Claude Sonnet 5", "run_id": "d700b5c52f0d", "session_id": "d700b5c52f0d4f9faa401ba096b81cd8", "ran_at": "2026-10-31T18:29:33.950269+00:00", "verdict": "match", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.5944612610364374, "line": 0.5, "closest": "Claude Sonnet 5", "baseline": { "name": "Claude reference benchmark", "version": "4.5.4-predictive.20260924.1", "tier": "Deep, 120 requests", "content_sha256": "7d61ae9aba3b81b41de1b0dbd2caeb71ea2c53e85511959e160f9ef853d98322" }, "candidates": [ { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.5944612610364374, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.40553873896356263, "line": 0.5 }, { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.34159827557945094, "line": 0.5 }, { "id": "claude-opus-5.5", "name": "Claude Opus 5.5", "score": 0.31868395621983475, "line": 0.5 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.22696262412709706, "line": 0.5 } ], "probes": [ { "id": "claude__screen008", "prompt": "不要思考，直接回答。从当、咚、铛、叮中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "叮", 23 ], [ "铛", 4 ], [ "咚", 2 ], [ "当", 1 ] ] }, { "id": "claude__screen021", "prompt": "不要思考，直接回答。从川、宁、禾、序中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "禾", 16 ], [ "宁", 13 ], [ "序", 1 ] ] }, { "id": "claude__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "43", 26 ], [ "42", 3 ], [ "47", 1 ] ] }, { "id": "claude__screen048", "prompt": "不要思考，直接回答。随便选一张麻将牌，只回答牌名。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "八万", 16 ], [ "一万", 7 ], [ "九万", 4 ], [ "二万", 2 ], [ "六万", 1 ] ] } ], "history": [ { "expected": "Claude Sonnet 5", "run_id": "d700b5c52f0d", "session_id": "d700b5c52f0d4f9faa401ba096b81cd8", "ran_at": "2026-10-31T18:29:33.950269+00:00", "verdict": "match", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.5944612610364374, "line": 0.5, "closest": "Claude Sonnet 5", "baseline": { "name": "Claude reference benchmark", "version": "4.5.4-predictive.20260924.1", "tier": "Deep, 120 requests", "content_sha256": "7d61ae9aba3b81b41de1b0dbd2caeb71ea2c53e85511959e160f9ef853d98322" }, "candidates": [ { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.5944612610364374, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.40553873896356263, "line": 0.5 }, { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.34159827557945094, "line": 0.5 }, { "id": "claude-opus-5.5", "name": "Claude Opus 5.5", "score": 0.31868395621983475, "line": 0.5 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.22696262412709706, "line": 0.5 } ], "probes": [ { "id": "claude__screen008", "prompt": "不要思考，直接回答。从当、咚、铛、叮中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "叮", 23 ], [ "铛", 4 ], [ "咚", 2 ], [ "当", 1 ] ] }, { "id": "claude__screen021", "prompt": "不要思考，直接回答。从川、宁、禾、序中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "禾", 16 ], [ "宁", 13 ], [ "序", 1 ] ] }, { "id": "claude__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "43", 26 ], [ "42", 3 ], [ "47", 1 ] ] }, { "id": "claude__screen048", "prompt": "不要思考，直接回答。随便选一张麻将牌，只回答牌名。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "八万", 16 ], [ "一万", 7 ], [ "九万", 4 ], [ "二万", 2 ], [ "六万", 1 ] ] } ] } ] }, { "id": "claude-sonnet-5-2", "name": "Claude Sonnet 5 (2)", "page": "/model/claude-sonnet-5", "family": "claude", "expected": "Claude Sonnet 5", "run_id": "349fe02ec30c", "session_id": "349fe02ec30c4716b00424fe79a303e8", "ran_at": "2026-10-31T19:29:12.117889+00:00", "verdict": "match", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.5250296079083339, "line": 0.5, "closest": "Claude Sonnet 5", "baseline": { "name": "Claude reference benchmark", "version": "4.5.4-predictive.20260924.1", "tier": "Deep, 120 requests", "content_sha256": "7d61ae9aba3b81b41de1b0dbd2caeb71ea2c53e85511959e160f9ef853d98322" }, "candidates": [ { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.5250296079083339, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.47497039209166625, "line": 0.5 }, { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.376889713482621, "line": 0.5 }, { "id": "claude-opus-5.5", "name": "Claude Opus 5.5", "score": 0.3727057495433649, "line": 0.5 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.3126599298629684, "line": 0.5 } ], "probes": [ { "id": "claude__screen008", "prompt": "不要思考，直接回答。从当、咚、铛、叮中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "铛", 14 ], [ "叮", 12 ], [ "当", 3 ], [ "當", 1 ] ] }, { "id": "claude__screen021", "prompt": "不要思考，直接回答。从川、宁、禾、序中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "禾", 15 ], [ "序", 8 ], [ "宁", 6 ], [ "川", 1 ] ] }, { "id": "claude__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "43", 18 ], [ "42", 10 ], [ "37", 1 ], [ "47", 1 ] ] }, { "id": "claude__screen048", "prompt": "不要思考，直接回答。随便选一张麻将牌，只回答牌名。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "九万", 18 ], [ "八万", 11 ], [ "六筒", 1 ] ] } ], "history": [ { "expected": "Claude Sonnet 5", "run_id": "349fe02ec30c", "session_id": "349fe02ec30c4716b00424fe79a303e8", "ran_at": "2026-10-31T19:29:12.117889+00:00", "verdict": "match", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.5250296079083339, "line": 0.5, "closest": "Claude Sonnet 5", "baseline": { "name": "Claude reference benchmark", "version": "4.5.4-predictive.20260924.1", "tier": "Deep, 120 requests", "content_sha256": "7d61ae9aba3b81b41de1b0dbd2caeb71ea2c53e85511959e160f9ef853d98322" }, "candidates": [ { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.5250296079083339, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.47497039209166625, "line": 0.5 }, { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.376889713482621, "line": 0.5 }, { "id": "claude-opus-5.5", "name": "Claude Opus 5.5", "score": 0.3727057495433649, "line": 0.5 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.3126599298629684, "line": 0.5 } ], "probes": [ { "id": "claude__screen008", "prompt": "不要思考，直接回答。从当、咚、铛、叮中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "铛", 14 ], [ "叮", 12 ], [ "当", 3 ], [ "當", 1 ] ] }, { "id": "claude__screen021", "prompt": "不要思考，直接回答。从川、宁、禾、序中选一个字，只回答该字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "禾", 15 ], [ "序", 8 ], [ "宁", 6 ], [ "川", 1 ] ] }, { "id": "claude__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "43", 18 ], [ "42", 10 ], [ "37", 1 ], [ "47", 1 ] ] }, { "id": "claude__screen048", "prompt": "不要思考，直接回答。随便选一张麻将牌，只回答牌名。", "english": null, "requests": 30, "valid": 30, "answers": [ [ "九万", 18 ], [ "八万", 11 ], [ "六筒", 1 ] ] } ] } ] }, { "id": "gpt-6-sol", "name": "GPT-6 Sol", "page": "/model/gpt-6-sol", "family": "gpt", "expected": "GPT-6 Sol", "run_id": "2cb1ac1f28c4", "session_id": "2cb1ac1f28c44e6185c65b2f6827fb63", "ran_at": "2026-10-31T19:10:12.430093+00:00", "verdict": "mismatch", "protocol": "Chat Completions", "planned": 128, "valid": 128, "errors": 0, "score": 0.3816848193931665, "line": 0.5, "closest": "GPT-6 Astra", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-chat.20260924.2", "tier": "Deep, 128 requests", "content_sha256": "bb283cb4b04b9698b26fd26e4fc2d56149e25d4e0f03625bbc82605b3e453eb8" }, "candidates": [ { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.5168204712873767, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.48317952871262326, "line": 0.5 }, { "id": "gpt-6-luna", "name": "GPT-6 Luna", "score": 0.3948384921155513, "line": 0.5 }, { "id": "gpt-6-sol", "name": "GPT-6 Sol", "score": 0.3816848193931665, "line": 0.5 }, { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.3493003083320959, "line": 0.5 } ], "probes": [ { "id": "gpt__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 43, "valid": 43, "answers": [ [ "43", 43 ] ] }, { "id": "gpt__screen106", "prompt": "从麻将牌中随便选一张，只回答牌名。", "english": null, "requests": 43, "valid": 43, "answers": [ [ "七筒", 22 ], [ "三筒", 16 ], [ "九筒", 5 ] ] }, { "id": "gpt__screen114", "prompt": "随便说出一个拟声词，只回答拟声词。", "english": null, "requests": 42, "valid": 42, "answers": [ [ "哗啦啦", 31 ], [ "喵喵", 6 ], [ "哗啦哗啦", 3 ], [ "哐当", 1 ], [ "喵呜", 1 ] ] } ], "history": [ { "run_id": "4498d0871d5f", "session_id": "4498d0871d5f4d96ab0389eae9f14e33", "ran_at": "2026-10-31T19:04:04.049858+00:00", "verdict": "mismatch", "protocol": "Responses", "planned": 128, "valid": 128, "errors": 0, "score": 0.29528369012115396, "line": 0.5, "closest": "GPT-6 Astra", "expected": "GPT-6 Sol", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-predictive.20260924.2", "tier": "Deep, 128 requests", "content_sha256": "e5dee4ccbc9cfd03883c49b3571815f44a4bf1b792730f7918d62c5eba4711e5" }, "candidates": [ { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.6395892240436218, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.3604107759563781, "line": 0.5 }, { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.3150397357054978, "line": 0.5 }, { "id": "gpt-6-sol", "name": "GPT-6 Sol", "score": 0.29528369012115396, "line": 0.5 }, { "id": "gpt-6-luna", "name": "GPT-6 Luna", "score": 0.27665959637911075, "line": 0.5 } ], "probes": [ { "id": "gpt__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 43, "valid": 43, "answers": [ [ "43", 40 ], [ "47", 3 ] ] }, { "id": "gpt__screen106", "prompt": "从麻将牌中随便选一张，只回答牌名。", "english": null, "requests": 43, "valid": 43, "answers": [ [ "七筒", 17 ], [ "三万", 14 ], [ "三筒", 10 ], [ "九筒", 2 ] ] }, { "id": "gpt__screen114", "prompt": "随便说出一个拟声词，只回答拟声词。", "english": null, "requests": 42, "valid": 42, "answers": [ [ "哗啦", 42 ] ] } ] }, { "expected": "GPT-6 Sol", "run_id": "2cb1ac1f28c4", "session_id": "2cb1ac1f28c44e6185c65b2f6827fb63", "ran_at": "2026-10-31T19:10:12.430093+00:00", "verdict": "mismatch", "protocol": "Chat Completions", "planned": 128, "valid": 128, "errors": 0, "score": 0.3816848193931665, "line": 0.5, "closest": "GPT-6 Astra", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-chat.20260924.2", "tier": "Deep, 128 requests", "content_sha256": "bb283cb4b04b9698b26fd26e4fc2d56149e25d4e0f03625bbc82605b3e453eb8" }, "candidates": [ { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.5168204712873767, "line": 0.5 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.48317952871262326, "line": 0.5 }, { "id": "gpt-6-luna", "name": "GPT-6 Luna", "score": 0.3948384921155513, "line": 0.5 }, { "id": "gpt-6-sol", "name": "GPT-6 Sol", "score": 0.3816848193931665, "line": 0.5 }, { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.3493003083320959, "line": 0.5 } ], "probes": [ { "id": "gpt__screen023", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 43, "valid": 43, "answers": [ [ "43", 43 ] ] }, { "id": "gpt__screen106", "prompt": "从麻将牌中随便选一张，只回答牌名。", "english": null, "requests": 43, "valid": 43, "answers": [ [ "七筒", 22 ], [ "三筒", 16 ], [ "九筒", 5 ] ] }, { "id": "gpt__screen114", "prompt": "随便说出一个拟声词，只回答拟声词。", "english": null, "requests": 42, "valid": 42, "answers": [ [ "哗啦啦", 31 ], [ "喵喵", 6 ], [ "哗啦哗啦", 3 ], [ "哐当", 1 ], [ "喵呜", 1 ] ] } ] } ] }, { "id": "gpt-5.6-sol", "name": "GPT-5.6 Sol", "page": "/model/gpt-5.6-sol", "family": "gpt", "expected": "GPT-5.6 Sol", "run_id": "e400f425b500", "session_id": "e400f425b5004bc889dfd6224f087ce7", "ran_at": "2026-10-31T19:38:53.877435+00:00", "verdict": "inconclusive", "protocol": "Chat Completions", "planned": 96, "valid": 96, "errors": 0, "score": 0.36840505679428986, "line": 0.5600060326824966, "closest": "No candidate selected", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-chat.1", "tier": "Deep, 96 requests", "content_sha256": "0445bb9a7b348f2c2eaf4f9307144875a9b72dd3638966424f0cb16a1c5d49c8" }, "candidates": [ { "id": "gpt-5.6-luna", "name": "GPT-5.6 Luna", "score": 0.5605155140352787, "line": 0.6109586687984903 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.43948448596472134, "line": 0.672574150740984 }, { "id": "gpt-5.6-sol", "name": "GPT-5.6 Sol", "score": 0.36840505679428986, "line": 0.5600060326824966 }, { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.33949229476956744, "line": 0.5643544145362065 }, { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.24593018246528428, "line": 0.5914614757463558 } ], "probes": [ { "id": "gpt__ZH006", "prompt": "随便说出一个国家，只回答国名。", "english": null, "requests": 16, "valid": 16, "answers": [ [ "加拿大", 16 ] ] }, { "id": "gpt__ZH027", "prompt": "随便说出一个拟声词，只回答拟声词。", "english": null, "requests": 16, "valid": 16, "answers": [ [ "喵喵", 12 ], [ "哗啦啦", 4 ] ] }, { "id": "gpt__ZH071", "prompt": "从17到83中随便选一个整数，只回答数字。", "english": null, "requests": 16, "valid": 16, "answers": [ [ "52", 4 ], [ "46", 3 ], [ "47", 3 ], [ "42", 2 ], [ "56", 2 ], [ "57", 2 ] ] }, { "id": "gpt__b80_letter_count", "prompt": "Count the letter r in strawberry. Reply only with the integer.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "3", 12 ], [ "1", 4 ] ] }, { "id": "gpt__rand_bird", "prompt": "Name a random bird. Reply with ONLY the bird name, one word.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Kingfisher", 16 ] ] }, { "id": "gpt__rand_country", "prompt": "Name a random country. Reply with ONLY the country name.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Uruguay", 6 ], [ "Bhutan", 4 ], [ "Portugal", 3 ], [ "Namibia", 2 ], [ "Madagascar", 1 ] ] } ], "history": [ { "run_id": "43d01e9b835e", "session_id": "43d01e9b835e45719e6c0069908f8da1", "ran_at": "2026-10-31T19:13:51.832819+00:00", "verdict": "inconclusive", "protocol": "Chat Completions", "planned": 96, "valid": 96, "errors": 0, "score": 0.41919576922183227, "line": 0.5600060326824966, "closest": "No candidate selected", "expected": "GPT-5.6 Sol", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-chat.1", "tier": "Deep, 96 requests", "content_sha256": "0445bb9a7b348f2c2eaf4f9307144875a9b72dd3638966424f0cb16a1c5d49c8" }, "candidates": [ { "id": "gpt-5.6-luna", "name": "GPT-5.6 Luna", "score": 0.533131134598449, "line": 0.6109586687984903 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.46686886540155104, "line": 0.672574150740984 }, { "id": "gpt-5.6-sol", "name": "GPT-5.6 Sol", "score": 0.41919576922183227, "line": 0.5600060326824966 }, { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.31553370091336264, "line": 0.5643544145362065 }, { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.244877962151479, "line": 0.5914614757463558 } ], "probes": [ { "id": "gpt__ZH006", "prompt": "随便说出一个国家，只回答国名。", "english": null, "requests": 16, "valid": 16, "answers": [ [ "加拿大", 16 ] ] }, { "id": "gpt__ZH027", "prompt": "随便说出一个拟声词，只回答拟声词。", "english": null, "requests": 16, "valid": 16, "answers": [ [ "哗啦啦", 10 ], [ "喵喵", 2 ], [ "叮咚", 1 ], [ "咕噜咕噜", 1 ], [ "哗啦", 1 ], [ "喵呜", 1 ] ] }, { "id": "gpt__ZH071", "prompt": "从17到83中随便选一个整数，只回答数字。", "english": null, "requests": 16, "valid": 16, "answers": [ [ "47", 5 ], [ "52", 3 ], [ "57", 3 ], [ "56", 2 ], [ "42", 1 ], [ "46", 1 ], [ "58", 1 ] ] }, { "id": "gpt__b80_letter_count", "prompt": "Count the letter r in strawberry. Reply only with the integer.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "3", 9 ], [ "1", 7 ] ] }, { "id": "gpt__rand_bird", "prompt": "Name a random bird. Reply with ONLY the bird name, one word.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Kingfisher", 14 ], [ "Puffin", 1 ], [ "Sparrow", 1 ] ] }, { "id": "gpt__rand_country", "prompt": "Name a random country. Reply with ONLY the country name.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Portugal", 6 ], [ "Uruguay", 4 ], [ "Bhutan", 2 ], [ "Namibia", 2 ], [ "Botswana", 1 ], [ "Madagascar", 1 ] ] } ] }, { "expected": "GPT-5.6 Sol", "run_id": "e400f425b500", "session_id": "e400f425b5004bc889dfd6224f087ce7", "ran_at": "2026-10-31T19:38:53.877435+00:00", "verdict": "inconclusive", "protocol": "Chat Completions", "planned": 96, "valid": 96, "errors": 0, "score": 0.36840505679428986, "line": 0.5600060326824966, "closest": "No candidate selected", "baseline": { "name": "GPT reference benchmark", "version": "4.5.4-chat.1", "tier": "Deep, 96 requests", "content_sha256": "0445bb9a7b348f2c2eaf4f9307144875a9b72dd3638966424f0cb16a1c5d49c8" }, "candidates": [ { "id": "gpt-5.6-luna", "name": "GPT-5.6 Luna", "score": 0.5605155140352787, "line": 0.6109586687984903 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.43948448596472134, "line": 0.672574150740984 }, { "id": "gpt-5.6-sol", "name": "GPT-5.6 Sol", "score": 0.36840505679428986, "line": 0.5600060326824966 }, { "id": "gpt-6-astra", "name": "GPT-6 Astra", "score": 0.33949229476956744, "line": 0.5643544145362065 }, { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "score": 0.24593018246528428, "line": 0.5914614757463558 } ], "probes": [ { "id": "gpt__ZH006", "prompt": "随便说出一个国家，只回答国名。", "english": null, "requests": 16, "valid": 16, "answers": [ [ "加拿大", 16 ] ] }, { "id": "gpt__ZH027", "prompt": "随便说出一个拟声词，只回答拟声词。", "english": null, "requests": 16, "valid": 16, "answers": [ [ "喵喵", 12 ], [ "哗啦啦", 4 ] ] }, { "id": "gpt__ZH071", "prompt": "从17到83中随便选一个整数，只回答数字。", "english": null, "requests": 16, "valid": 16, "answers": [ [ "52", 4 ], [ "46", 3 ], [ "47", 3 ], [ "42", 2 ], [ "56", 2 ], [ "57", 2 ] ] }, { "id": "gpt__b80_letter_count", "prompt": "Count the letter r in strawberry. Reply only with the integer.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "3", 12 ], [ "1", 4 ] ] }, { "id": "gpt__rand_bird", "prompt": "Name a random bird. Reply with ONLY the bird name, one word.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Kingfisher", 16 ] ] }, { "id": "gpt__rand_country", "prompt": "Name a random country. Reply with ONLY the country name.", "english": null, "requests": 16, "valid": 16, "answers": [ [ "Uruguay", 6 ], [ "Bhutan", 4 ], [ "Portugal", 3 ], [ "Namibia", 2 ], [ "Madagascar", 1 ] ] } ] } ] }, { "id": "claude-opus-5", "name": "Claude Opus 5", "page": "/model/claude-opus-5", "family": "claude", "expected": "Claude Opus 5", "run_id": "34180b2cd388", "session_id": "34180b2cd3884cabb013e14299f6408c", "ran_at": "2026-10-31T20:05:12.697045+00:00", "verdict": "match", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.6433801535109631, "line": 0.552050375195513, "closest": "Claude Opus 5", "baseline": { "name": "Claude reference benchmark", "version": "4.5.3-predictive.3", "tier": "Deep, 120 requests", "content_sha256": "eeac401648dd3ee7fc7f28d34b43733fa38c487e271d4e1d8a742fb71c192f63" }, "candidates": [ { "id": "claude-opus-5", "name": "Claude Opus 5", "score": 0.6433801535109631, "line": 0.552050375195513 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.3566198464890369, "line": 0.5455215768178815 }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.16512807119189624, "line": 0.5255602784586072 }, { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.15121824024351468, "line": 0.5288033485604039 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.13942809351313834, "line": 0.5440241107226554 } ], "probes": [ { "id": "claude__CL011", "prompt": "不要思考，直接回答。从mepo、takri、sulon、vash中选一个词，只回答所选词。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "vash", 10 ], [ "Vash", 7 ], [ "Takri", 2 ], [ "takri", 1 ] ] }, { "id": "claude__CL026", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "45", 8 ], [ "47", 7 ], [ "44", 3 ], [ "43", 2 ] ] }, { "id": "claude__CL033", "prompt": "不要思考，直接回答。随便选一个大写英文字母，只回答该字母。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "M", 18 ], [ "Q", 2 ] ] }, { "id": "claude__CL045", "prompt": "不要思考，直接回答。随便选一个整点，只回答0到23中的数字。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "17", 10 ], [ "7", 10 ] ] }, { "id": "claude__CL085", "prompt": "不要思考，直接回答。从山地、平原、高原、盆地中选一种地形，只回答名称。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "盆地", 17 ], [ "平原", 2 ], [ "高原", 1 ] ] }, { "id": "claude__CL087", "prompt": "不要思考，直接回答。从十个天干中随便选一个，只回答一个字。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "戊", 11 ], [ "癸", 4 ], [ "辛", 3 ], [ "壬", 2 ] ] } ], "history": [ { "expected": "Claude Opus 5", "run_id": "34180b2cd388", "session_id": "34180b2cd3884cabb013e14299f6408c", "ran_at": "2026-10-31T20:05:12.697045+00:00", "verdict": "match", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.6433801535109631, "line": 0.552050375195513, "closest": "Claude Opus 5", "baseline": { "name": "Claude reference benchmark", "version": "4.5.3-predictive.3", "tier": "Deep, 120 requests", "content_sha256": "eeac401648dd3ee7fc7f28d34b43733fa38c487e271d4e1d8a742fb71c192f63" }, "candidates": [ { "id": "claude-opus-5", "name": "Claude Opus 5", "score": 0.6433801535109631, "line": 0.552050375195513 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.3566198464890369, "line": 0.5455215768178815 }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.16512807119189624, "line": 0.5255602784586072 }, { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.15121824024351468, "line": 0.5288033485604039 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.13942809351313834, "line": 0.5440241107226554 } ], "probes": [ { "id": "claude__CL011", "prompt": "不要思考，直接回答。从mepo、takri、sulon、vash中选一个词，只回答所选词。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "vash", 10 ], [ "Vash", 7 ], [ "Takri", 2 ], [ "takri", 1 ] ] }, { "id": "claude__CL026", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "45", 8 ], [ "47", 7 ], [ "44", 3 ], [ "43", 2 ] ] }, { "id": "claude__CL033", "prompt": "不要思考，直接回答。随便选一个大写英文字母，只回答该字母。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "M", 18 ], [ "Q", 2 ] ] }, { "id": "claude__CL045", "prompt": "不要思考，直接回答。随便选一个整点，只回答0到23中的数字。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "17", 10 ], [ "7", 10 ] ] }, { "id": "claude__CL085", "prompt": "不要思考，直接回答。从山地、平原、高原、盆地中选一种地形，只回答名称。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "盆地", 17 ], [ "平原", 2 ], [ "高原", 1 ] ] }, { "id": "claude__CL087", "prompt": "不要思考，直接回答。从十个天干中随便选一个，只回答一个字。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "戊", 11 ], [ "癸", 4 ], [ "辛", 3 ], [ "壬", 2 ] ] } ] } ] }, { "id": "claude-opus-5-2", "name": "Claude Opus 5 (2)", "page": "/model/claude-opus-5", "family": "claude", "expected": "Claude Opus 5", "run_id": "31cfb2f23f36", "session_id": "31cfb2f23f3643168fceba11dfccbfa1", "ran_at": "2026-10-31T20:09:43.009925+00:00", "verdict": "inconclusive", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.24759477798722157, "line": 0.552050375195513, "closest": "No candidate selected", "baseline": { "name": "Claude reference benchmark", "version": "4.5.3-predictive.3", "tier": "Deep, 120 requests", "content_sha256": "eeac401648dd3ee7fc7f28d34b43733fa38c487e271d4e1d8a742fb71c192f63" }, "candidates": [ { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.5138166107958264, "line": 0.5288033485604039 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.48618338920417364, "line": 0.5455215768178815 }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.24925797743931238, "line": 0.5255602784586072 }, { "id": "claude-opus-5", "name": "Claude Opus 5", "score": 0.24759477798722157, "line": 0.552050375195513 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.13203381578495646, "line": 0.5440241107226554 } ], "probes": [ { "id": "claude__CL011", "prompt": "不要思考，直接回答。从mepo、takri、sulon、vash中选一个词，只回答所选词。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "vash", 15 ], [ "sulon", 5 ] ] }, { "id": "claude__CL026", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "47", 20 ] ] }, { "id": "claude__CL033", "prompt": "不要思考，直接回答。随便选一个大写英文字母，只回答该字母。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "K", 17 ], [ "M", 3 ] ] }, { "id": "claude__CL045", "prompt": "不要思考，直接回答。随便选一个整点，只回答0到23中的数字。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "14", 20 ] ] }, { "id": "claude__CL085", "prompt": "不要思考，直接回答。从山地、平原、高原、盆地中选一种地形，只回答名称。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "平原", 19 ], [ "高原", 1 ] ] }, { "id": "claude__CL087", "prompt": "不要思考，直接回答。从十个天干中随便选一个，只回答一个字。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "丙", 20 ] ] } ], "history": [ { "expected": "Claude Opus 5", "run_id": "31cfb2f23f36", "session_id": "31cfb2f23f3643168fceba11dfccbfa1", "ran_at": "2026-10-31T20:09:43.009925+00:00", "verdict": "inconclusive", "protocol": "Messages", "planned": 120, "valid": 120, "errors": 0, "score": 0.24759477798722157, "line": 0.552050375195513, "closest": "No candidate selected", "baseline": { "name": "Claude reference benchmark", "version": "4.5.3-predictive.3", "tier": "Deep, 120 requests", "content_sha256": "eeac401648dd3ee7fc7f28d34b43733fa38c487e271d4e1d8a742fb71c192f63" }, "candidates": [ { "id": "claude-fable-5.1", "name": "Claude Fable 5.1", "score": 0.5138166107958264, "line": 0.5288033485604039 }, { "id": "other_known_external", "name": "Other reference models", "score": 0.48618338920417364, "line": 0.5455215768178815 }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "score": 0.24925797743931238, "line": 0.5255602784586072 }, { "id": "claude-opus-5", "name": "Claude Opus 5", "score": 0.24759477798722157, "line": 0.552050375195513 }, { "id": "claude-haiku-4.5", "name": "Claude Haiku 4.5", "score": 0.13203381578495646, "line": 0.5440241107226554 } ], "probes": [ { "id": "claude__CL011", "prompt": "不要思考，直接回答。从mepo、takri、sulon、vash中选一个词，只回答所选词。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "vash", 15 ], [ "sulon", 5 ] ] }, { "id": "claude__CL026", "prompt": "不要思考，直接回答。从19到71中选一个整数，只回答数字。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "47", 20 ] ] }, { "id": "claude__CL033", "prompt": "不要思考，直接回答。随便选一个大写英文字母，只回答该字母。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "K", 17 ], [ "M", 3 ] ] }, { "id": "claude__CL045", "prompt": "不要思考，直接回答。随便选一个整点，只回答0到23中的数字。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "14", 20 ] ] }, { "id": "claude__CL085", "prompt": "不要思考，直接回答。从山地、平原、高原、盆地中选一种地形，只回答名称。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "平原", 19 ], [ "高原", 1 ] ] }, { "id": "claude__CL087", "prompt": "不要思考，直接回答。从十个天干中随便选一个，只回答一个字。", "english": null, "requests": 20, "valid": 20, "answers": [ [ "丙", 20 ] ] } ] } ] } ], "baselines": { "gpt": { "name": "GPT reference benchmark", "version": "4.5.4-predictive.20261003.1", "tier": "Deep, 128 requests", "content_sha256": "ff04ec844b5bd63c189748cff4caaf039be9cb908f398cb6c279efab9e8c4aec" }, "claude": { "name": "Claude reference benchmark", "version": "4.5.4-predictive.20260924.1", "tier": "Deep, 120 requests", "content_sha256": "7d61ae9aba3b81b41de1b0dbd2caeb71ea2c53e85511959e160f9ef853d98322" } }, "not_covered": [ "claude-fable-5", "claude-fable-5-2", "claude-sonnet-5-5", "claude-sonnet-5-5-2" ] }