{ "meta": { "note": "Demo-tier fixture batch for the Bench page (#187). Real batches will come from bench/list-experiments (G19, pending). Data shapes mirror dsbench-v2-reference.md §打分合约 (code_result.json = {resolved, score, reason} per run) and the bench-design-pack-160.md §3 kind one-pagers. All numbers are illustrative — nothing here talked to a real model.", "generatedAt": 1721160000000, "version": 1 }, "experiments": [ { "id": "matrix-deepseek-vs-baseline-onProm28", "name": "deepseek-vs-baseline-onProm28", "kind": "matrix", "status": "running", "progress": { "done": 62, "total": 84 }, "createdAt": 1720982400000, "N": 3, "promptSet": { "id": "prom-28", "version": "v7", "count": 28 }, "models": [ { "id": "deepseek-chat", "label": "deepseek-chat" }, { "id": "deepseek-reasoner", "label": "deepseek-reasoner" }, { "id": "baseline-oss", "label": "baseline-oss" } ], "config": { "profile": "stdio-deepseek", "rubric": { "id": "code-grade-v3", "kind": "code-grade" }, "temperature": 0.2 }, "matrix": { "prompts": [ { "id": "sum-of-primes", "label": "sum-of-primes" }, { "id": "unify-json-diff", "label": "unify-json-diff" }, { "id": "parse-jsonc", "label": "parse-jsonc" }, { "id": "topo-sort-tasks", "label": "topo-sort-tasks" } ], "models": [ { "id": "deepseek-chat", "label": "deepseek-chat" }, { "id": "deepseek-reasoner", "label": "deepseek-reasoner" }, { "id": "baseline-oss", "label": "baseline-oss" } ], "cells": { "sum-of-primes|deepseek-chat": { "promptId": "sum-of-primes", "modelId": "deepseek-chat", "resolvedCount": 3, "N": 3, "status": "ok", "score": 0.87, "latencyMs": 1700, "tokens": { "in": 640, "out": 320 }, "cost": 0.021, "runs": [ { "runIdx": 1, "resolved": true, "score": 0.88, "reason": "passes tests", "latencyMs": 1680, "tokens": { "in": 640, "out": 320 }, "cost": 0.007, "sessionId": "sess-mchat-1" }, { "runIdx": 2, "resolved": true, "score": 0.86, "reason": "passes tests", "latencyMs": 1720, "tokens": { "in": 640, "out": 322 }, "cost": 0.007, "sessionId": "sess-mchat-2" }, { "runIdx": 3, "resolved": true, "score": 0.87, "reason": "passes tests", "latencyMs": 1700, "tokens": { "in": 640, "out": 318 }, "cost": 0.007, "sessionId": "sess-mchat-3" } ] }, "sum-of-primes|deepseek-reasoner": { "promptId": "sum-of-primes", "modelId": "deepseek-reasoner", "resolvedCount": 3, "N": 3, "status": "ok", "score": 0.91, "latencyMs": 3200, "tokens": { "in": 640, "out": 890 }, "cost": 0.048, "runs": [ { "runIdx": 1, "resolved": true, "score": 0.91, "reason": "reasoning-block traced", "latencyMs": 3100, "tokens": { "in": 640, "out": 880 }, "cost": 0.016, "sessionId": "sess-mreas-1" }, { "runIdx": 2, "resolved": true, "score": 0.90, "reason": "reasoning-block traced", "latencyMs": 3200, "tokens": { "in": 640, "out": 900 }, "cost": 0.016, "sessionId": "sess-mreas-2" }, { "runIdx": 3, "resolved": true, "score": 0.92, "reason": "reasoning-block traced", "latencyMs": 3300, "tokens": { "in": 640, "out": 890 }, "cost": 0.016, "sessionId": "sess-mreas-3" } ] }, "sum-of-primes|baseline-oss": { "promptId": "sum-of-primes", "modelId": "baseline-oss", "resolvedCount": 0, "N": 3, "status": "fail", "score": 0.28, "latencyMs": 3400, "tokens": { "in": 640, "out": 220 }, "cost": 0.006, "runs": [ { "runIdx": 1, "resolved": false, "score": 0.30, "reason": "syntax error in output", "latencyMs": 3400, "tokens": { "in": 640, "out": 210 }, "cost": 0.002, "sessionId": "sess-mbase-1" }, { "runIdx": 2, "resolved": false, "score": 0.28, "reason": "syntax error in output", "latencyMs": 3400, "tokens": { "in": 640, "out": 230 }, "cost": 0.002, "sessionId": "sess-mbase-2" }, { "runIdx": 3, "resolved": false, "score": 0.26, "reason": "syntax error in output", "latencyMs": 3400, "tokens": { "in": 640, "out": 220 }, "cost": 0.002, "sessionId": "sess-mbase-3" } ] }, "unify-json-diff|deepseek-chat": { "promptId": "unify-json-diff", "modelId": "deepseek-chat", "resolvedCount": 3, "N": 3, "status": "ok", "score": 0.91, "latencyMs": 1900, "tokens": { "in": 720, "out": 410 }, "cost": 0.024, "runs": [ { "runIdx": 1, "resolved": true, "score": 0.92, "reason": "passes tests", "latencyMs": 1900, "tokens": { "in": 720, "out": 410 }, "cost": 0.008 }, { "runIdx": 2, "resolved": true, "score": 0.90, "reason": "passes tests", "latencyMs": 1900, "tokens": { "in": 720, "out": 410 }, "cost": 0.008 }, { "runIdx": 3, "resolved": true, "score": 0.91, "reason": "passes tests", "latencyMs": 1900, "tokens": { "in": 720, "out": 410 }, "cost": 0.008 } ] }, "unify-json-diff|deepseek-reasoner": { "promptId": "unify-json-diff", "modelId": "deepseek-reasoner", "resolvedCount": 3, "N": 3, "status": "ok", "score": 0.85, "latencyMs": 3600, "tokens": { "in": 720, "out": 950 }, "cost": 0.051, "runs": [ { "runIdx": 1, "resolved": true, "score": 0.85, "reason": "passes tests", "latencyMs": 3600, "tokens": { "in": 720, "out": 950 }, "cost": 0.017 }, { "runIdx": 2, "resolved": true, "score": 0.85, "reason": "passes tests", "latencyMs": 3600, "tokens": { "in": 720, "out": 950 }, "cost": 0.017 }, { "runIdx": 3, "resolved": true, "score": 0.85, "reason": "passes tests", "latencyMs": 3600, "tokens": { "in": 720, "out": 950 }, "cost": 0.017 } ] }, "unify-json-diff|baseline-oss": { "promptId": "unify-json-diff", "modelId": "baseline-oss", "resolvedCount": 1, "N": 3, "status": "ok", "score": 0.44, "latencyMs": 3200, "tokens": { "in": 720, "out": 260 }, "cost": 0.007, "runs": [ { "runIdx": 1, "resolved": true, "score": 0.60, "reason": "one lucky pass", "latencyMs": 3200, "tokens": { "in": 720, "out": 260 }, "cost": 0.002 }, { "runIdx": 2, "resolved": false, "score": 0.42, "reason": "wrong diff format", "latencyMs": 3200, "tokens": { "in": 720, "out": 260 }, "cost": 0.002 }, { "runIdx": 3, "resolved": false, "score": 0.30, "reason": "wrong diff format", "latencyMs": 3200, "tokens": { "in": 720, "out": 260 }, "cost": 0.002 } ] }, "parse-jsonc|deepseek-chat": { "promptId": "parse-jsonc", "modelId": "deepseek-chat", "resolvedCount": 3, "N": 3, "status": "ok", "score": 0.79, "latencyMs": 1500, "tokens": { "in": 580, "out": 290 }, "cost": 0.018, "runs": [ { "runIdx": 1, "resolved": true, "score": 0.80, "reason": "passes tests", "latencyMs": 1500, "tokens": { "in": 580, "out": 290 }, "cost": 0.006 }, { "runIdx": 2, "resolved": true, "score": 0.78, "reason": "passes tests", "latencyMs": 1500, "tokens": { "in": 580, "out": 290 }, "cost": 0.006 }, { "runIdx": 3, "resolved": true, "score": 0.79, "reason": "passes tests", "latencyMs": 1500, "tokens": { "in": 580, "out": 290 }, "cost": 0.006 } ] }, "parse-jsonc|deepseek-reasoner": { "promptId": "parse-jsonc", "modelId": "deepseek-reasoner", "resolvedCount": 1, "N": 3, "status": "running", "score": 0.68, "latencyMs": 3000, "tokens": { "in": 580, "out": 720 }, "cost": 0.041, "runs": [ { "runIdx": 1, "resolved": true, "score": 0.68, "reason": "partial trace", "latencyMs": 3000, "tokens": { "in": 580, "out": 720 }, "cost": 0.014 } ] }, "parse-jsonc|baseline-oss": { "promptId": "parse-jsonc", "modelId": "baseline-oss", "resolvedCount": 0, "N": 0, "status": "queued", "score": 0, "latencyMs": null, "tokens": null, "cost": null, "runs": [] }, "topo-sort-tasks|deepseek-chat": { "promptId": "topo-sort-tasks", "modelId": "deepseek-chat", "resolvedCount": 3, "N": 3, "status": "ok", "score": 0.94, "latencyMs": 2100, "tokens": { "in": 810, "out": 470 }, "cost": 0.026, "runs": [ { "runIdx": 1, "resolved": true, "score": 0.94, "reason": "passes tests", "latencyMs": 2100, "tokens": { "in": 810, "out": 470 }, "cost": 0.009 }, { "runIdx": 2, "resolved": true, "score": 0.94, "reason": "passes tests", "latencyMs": 2100, "tokens": { "in": 810, "out": 470 }, "cost": 0.009 }, { "runIdx": 3, "resolved": true, "score": 0.94, "reason": "passes tests", "latencyMs": 2100, "tokens": { "in": 810, "out": 470 }, "cost": 0.008 } ] }, "topo-sort-tasks|deepseek-reasoner": { "promptId": "topo-sort-tasks", "modelId": "deepseek-reasoner", "resolvedCount": 2, "N": 3, "status": "ok", "score": 0.68, "latencyMs": 3700, "tokens": { "in": 810, "out": 1020 }, "cost": 0.054, "runs": [ { "runIdx": 1, "resolved": true, "score": 0.72, "reason": "passes tests", "latencyMs": 3700, "tokens": { "in": 810, "out": 1020 }, "cost": 0.018 }, { "runIdx": 2, "resolved": true, "score": 0.68, "reason": "passes tests", "latencyMs": 3700, "tokens": { "in": 810, "out": 1020 }, "cost": 0.018 }, { "runIdx": 3, "resolved": false, "score": 0.64, "reason": "cycle undetected", "latencyMs": 3700, "tokens": { "in": 810, "out": 1020 }, "cost": 0.018 } ] }, "topo-sort-tasks|baseline-oss": { "promptId": "topo-sort-tasks", "modelId": "baseline-oss", "resolvedCount": 0, "N": 0, "status": "queued", "score": 0, "latencyMs": null, "tokens": null, "cost": null, "runs": [] } } } }, { "id": "ab-memory-plugin-v2-vs-v1", "name": "memory-plugin-v2-vs-v1", "kind": "ab", "status": "done", "progress": { "done": 48, "total": 48 }, "createdAt": 1720886400000, "N": 24, "promptSet": { "id": "prom-memory-24", "version": "v3", "count": 24 }, "models": [{ "id": "deepseek-chat", "label": "deepseek-chat" }], "config": { "profile": "stdio-deepseek", "rubric": { "id": "memory-recall-v1", "kind": "code-grade" }, "plugin": { "id": "memory-plugin", "vary": "version" } }, "ab": { "variantA": { "label": "v2", "plugin": "memory-plugin", "version": "v2" }, "variantB": { "label": "v1", "plugin": "memory-plugin", "version": "v1" }, "rows": [ { "promptId": "recall-file-context", "a": { "resolved": true, "score": 0.87, "latencyMs": 1800, "tokens": { "in": 1200, "out": 320 } }, "b": { "resolved": true, "score": 0.79, "latencyMs": 2100, "tokens": { "in": 1400, "out": 350 } } }, { "promptId": "recall-conversation", "a": { "resolved": true, "score": 0.91, "latencyMs": 1900, "tokens": { "in": 1300, "out": 340 } }, "b": { "resolved": true, "score": 0.88, "latencyMs": 2200, "tokens": { "in": 1500, "out": 370 } } }, { "promptId": "recall-across-forks", "a": { "resolved": false, "score": 0.32, "latencyMs": 2400, "tokens": { "in": 1500, "out": 280 } }, "b": { "resolved": true, "score": 0.68, "latencyMs": 2600, "tokens": { "in": 1700, "out": 400 } } }, { "promptId": "recall-with-compact", "a": { "resolved": true, "score": 0.82, "latencyMs": 2000, "tokens": { "in": 1400, "out": 360 } }, "b": { "resolved": true, "score": 0.74, "latencyMs": 2300, "tokens": { "in": 1600, "out": 390 } } }, { "promptId": "recall-multi-file", "a": { "resolved": true, "score": 0.90, "latencyMs": 2100, "tokens": { "in": 1450, "out": 380 } }, "b": { "resolved": true, "score": 0.85, "latencyMs": 2400, "tokens": { "in": 1650, "out": 410 } } }, { "promptId": "recall-code-symbol", "a": { "resolved": true, "score": 0.88, "latencyMs": 1750, "tokens": { "in": 1150, "out": 300 } }, "b": { "resolved": true, "score": 0.86, "latencyMs": 2050, "tokens": { "in": 1350, "out": 330 } } }, { "promptId": "recall-with-shadowed", "a": { "resolved": true, "score": 0.75, "latencyMs": 2200, "tokens": { "in": 1550, "out": 400 } }, "b": { "resolved": false, "score": 0.42, "latencyMs": 2500, "tokens": { "in": 1750, "out": 420 } } }, { "promptId": "recall-stale-context", "a": { "resolved": true, "score": 0.80, "latencyMs": 1950, "tokens": { "in": 1350, "out": 350 } }, "b": { "resolved": true, "score": 0.78, "latencyMs": 2250, "tokens": { "in": 1550, "out": 380 } } } ] } }, { "id": "rep-parse-jsonc-edge-cases", "name": "parse-jsonc-edge-cases N=5", "kind": "rep", "status": "done", "progress": { "done": 5, "total": 5 }, "createdAt": 1720800000000, "N": 5, "models": [{ "id": "deepseek-chat", "label": "deepseek-chat" }], "config": { "profile": "stdio-deepseek", "rubric": { "id": "code-grade-v3", "kind": "code-grade" }, "temperature": 0.7 }, "rep": { "input": "Parse this JSONC with mixed //-comments, /* */-blocks, trailing commas, and unquoted keys.", "dims": [ { "id": "code_correctness", "label": "code_correctness", "kind": "score" }, { "id": "passes_tests", "label": "passes_tests", "kind": "boolean" }, { "id": "latency", "label": "latency", "kind": "latency" }, { "id": "tokens_out", "label": "tokens (out)", "kind": "tokens" } ], "repetitions": [ { "idx": 1, "resolved": false, "score": 0.62, "latencyMs": 1400, "tokens": { "in": 580, "out": 543 }, "sessionId": "sess-rep-1", "dimensions": { "code_correctness": 0.62, "passes_tests": false, "latency": 1400, "tokens_out": 543 } }, { "idx": 2, "resolved": true, "score": 0.85, "latencyMs": 900, "tokens": { "in": 580, "out": 488 }, "sessionId": "sess-rep-2", "dimensions": { "code_correctness": 0.85, "passes_tests": true, "latency": 900, "tokens_out": 488 } }, { "idx": 3, "resolved": true, "score": 0.79, "latencyMs": 1100, "tokens": { "in": 580, "out": 510 }, "sessionId": "sess-rep-3", "dimensions": { "code_correctness": 0.79, "passes_tests": true, "latency": 1100, "tokens_out": 510 } }, { "idx": 4, "resolved": true, "score": 0.91, "latencyMs": 1300, "tokens": { "in": 580, "out": 521 }, "sessionId": "sess-rep-4", "dimensions": { "code_correctness": 0.91, "passes_tests": true, "latency": 1300, "tokens_out": 521 } }, { "idx": 5, "resolved": true, "score": 0.72, "latencyMs": 1200, "tokens": { "in": 580, "out": 499 }, "sessionId": "sess-rep-5", "dimensions": { "code_correctness": 0.72, "passes_tests": true, "latency": 1200, "tokens_out": 499 } } ] } }, { "id": "matrix-reasoner-vs-chat-on-swe-lite", "name": "reasoner-vs-chat-on-swe-bench-lite", "kind": "matrix", "status": "queued", "progress": { "done": 0, "total": 60 }, "createdAt": 1720720000000, "N": 3, "promptSet": { "id": "swe-bench-lite", "version": "2024.11", "count": 20 }, "models": [ { "id": "deepseek-reasoner", "label": "deepseek-reasoner" }, { "id": "deepseek-chat", "label": "deepseek-chat" } ], "config": { "profile": "stdio-deepseek", "rubric": { "id": "swe-lite-grader", "kind": "code-grade" } }, "matrix": { "prompts": [], "models": [], "cells": {} } } ] }