mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
1fd9e2a9ab
Every session.jsonl fixture embedded the full composed system prompt and
complete tool-schema list in its request/header event (~8 KB on one line,
identical across the suite), so any prompt or tool-schema edit forced a
re-record or hand-edit of every fixture — see the dynamic-workflows PR for
the churn pattern this removes.
Now exactly one scenario (text-turn, flagged pinsHeader) commits and
compares that content verbatim; every other fixture stores and compares it
as {{system}}/{{tools}} tokens via the new pure scrubRequestHeaders
normalizer (applied to both compare sides and to record-mode writes, so a
re-record cannot reintroduce the content). request/header-delta payloads
are scrubbed the same way; config/reason stay verbatim — a model swap
SHOULD churn every fixture, a prompt edit should not. Replay is unaffected:
script derivation reads only assistant/chunk events.
Fixture meta-guards enforce the split: non-pinning fixtures must be fixed
points of the scrub, the pinning fixture must not be, and exactly one
scenario pins. Committed fixtures migrated through the same function.
Docs: pinned-header RFC (implemented/testing), base snapshot RFC + testing
policy + llm-replay module doc/README updated.
50 lines
7.5 KiB
JSON
50 lines
7.5 KiB
JSON
{"type":"session","version":0,"id":"6a477a04-6b95-4197-ba20-7a9a30072f68","createdAt":1783279463841,"cwd":"/tmp/acp-snap-cwd-dXMGno"}
|
|
{"type":"turn/start","seq":0,"time":1783279463845,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}}
|
|
{"type":"hook/invoked","seq":1,"time":1783279463846,"data":{"turn":1,"point":"UserPromptSubmit","dialect":"codex","handlerId":"codex:UserPromptSubmit:1"}}
|
|
{"type":"hook/result","seq":2,"time":1783279463864,"data":{"turn":1,"point":"UserPromptSubmit","handlerId":"codex:UserPromptSubmit:1","decision":"pass","exitCode":0,"durationMs":17.464082999999846}}
|
|
{"type":"user/message","seq":3,"time":1783279463864,"data":{"content":[{"type":"text","text":"What is my favorite color? Reply with just the color and stop. Do not use any tools."}],"source":{"kind":"user"}},"surfaceOp":"append"}
|
|
{"type":"context/message","seq":4,"time":1783279463864,"data":{"content":[{"type":"text","text":"The user has previously stated their favorite color is teal."}],"source":{"kind":"plugin","plugin":"hooks-codex"}},"surfaceOp":"append"}
|
|
{"type":"step/start","seq":5,"time":1783279463865,"data":{"turn":1,"step":1}}
|
|
{"type":"request/header","seq":6,"time":1783279463866,"data":{"header":{"config":{"model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}
|
|
{"type":"assistant/chunk","seq":7,"time":1783279464538,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}}
|
|
{"type":"assistant/chunk","seq":8,"time":1783279464539,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}}
|
|
{"type":"assistant/chunk","seq":9,"time":1783279464680,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}}
|
|
{"type":"assistant/chunk","seq":10,"time":1783279464707,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" asked"}}}
|
|
{"type":"assistant/chunk","seq":11,"time":1783279464736,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" about"}}}
|
|
{"type":"assistant/chunk","seq":12,"time":1783279464736,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" their"}}}
|
|
{"type":"assistant/chunk","seq":13,"time":1783279464764,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" favorite"}}}
|
|
{"type":"assistant/chunk","seq":14,"time":1783279464764,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" color"}}}
|
|
{"type":"assistant/chunk","seq":15,"time":1783279464765,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":","}}}
|
|
{"type":"assistant/chunk","seq":16,"time":1783279464765,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" and"}}}
|
|
{"type":"assistant/chunk","seq":17,"time":1783279464792,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" the"}}}
|
|
{"type":"assistant/chunk","seq":18,"time":1783279464792,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" context"}}}
|
|
{"type":"assistant/chunk","seq":19,"time":1783279464792,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" tells"}}}
|
|
{"type":"assistant/chunk","seq":20,"time":1783279464820,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}}
|
|
{"type":"assistant/chunk","seq":21,"time":1783279464821,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" they"}}}
|
|
{"type":"assistant/chunk","seq":22,"time":1783279464847,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" previously"}}}
|
|
{"type":"assistant/chunk","seq":23,"time":1783279464848,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" stated"}}}
|
|
{"type":"assistant/chunk","seq":24,"time":1783279464848,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" it"}}}
|
|
{"type":"assistant/chunk","seq":25,"time":1783279464878,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"'s"}}}
|
|
{"type":"assistant/chunk","seq":26,"time":1783279464878,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" te"}}}
|
|
{"type":"assistant/chunk","seq":27,"time":1783279464878,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"al"}}}
|
|
{"type":"assistant/chunk","seq":28,"time":1783279464878,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}}
|
|
{"type":"assistant/chunk","seq":29,"time":1783279464878,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" They"}}}
|
|
{"type":"assistant/chunk","seq":30,"time":1783279464904,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" want"}}}
|
|
{"type":"assistant/chunk","seq":31,"time":1783279464932,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" just"}}}
|
|
{"type":"assistant/chunk","seq":32,"time":1783279464961,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" the"}}}
|
|
{"type":"assistant/chunk","seq":33,"time":1783279464961,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" color"}}}
|
|
{"type":"assistant/chunk","seq":34,"time":1783279464961,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" and"}}}
|
|
{"type":"assistant/chunk","seq":35,"time":1783279464987,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" no"}}}
|
|
{"type":"assistant/chunk","seq":36,"time":1783279465015,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" tools"}}}
|
|
{"type":"assistant/chunk","seq":37,"time":1783279465016,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}}
|
|
{"type":"assistant/chunk","seq":38,"time":1783279465047,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"text"}}}
|
|
{"type":"assistant/chunk","seq":39,"time":1783279465047,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":1,"text":"te"}}}
|
|
{"type":"assistant/chunk","seq":40,"time":1783279465047,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":1,"text":"al"}}}
|
|
{"type":"assistant/chunk","seq":41,"time":1783279465048,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user asked about their favorite color, and the context tells me they previously stated it's teal. They want just the color and no tools."}}}}
|
|
{"type":"assistant/chunk","seq":42,"time":1783279465048,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"text","text":"teal"}}}}
|
|
{"type":"assistant/chunk","seq":43,"time":1783279465048,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":2119,"outputTokens":33,"cacheReadTokens":0,"reasoningTokens":30}}}}
|
|
{"type":"assistant/chunk","seq":44,"time":1783279465048,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}
|
|
{"type":"assistant/message","seq":45,"time":1783279465050,"data":{"turn":1,"step":1,"content":[{"type":"reasoning","text":"The user asked about their favorite color, and the context tells me they previously stated it's teal. They want just the color and no tools."},{"type":"text","text":"teal"}],"usage":{"inputTokens":2119,"outputTokens":33,"cacheReadTokens":0,"reasoningTokens":30}},"sourceEventSeqs":[7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44],"surfaceOp":"append"}
|
|
{"type":"step/end","seq":46,"time":1783279465050,"data":{"turn":1,"step":1}}
|
|
{"type":"turn/end","seq":47,"time":1783279465050,"data":{"turn":1,"reason":{"kind":"completed"}}}
|