test(e2e): drive the TUI keyless smoke through the cross-platform PTY harness

The smoke's inline Python pty driver only ran on POSIX (no termios on
Windows). Rebuild every scenario — banner sweep, scripted conversation with
model switch, /skill:, Code Mode overlay, resume failure, and the dsh CLI
suite (default boot, personal overlay, invalid overlay, --resume flag,
source-path prompt) — as marker-gated action lists on pty-harness.ts, which
drives ConPTY via node-pty on Windows and the Python driver elsewhere. The
harness gains configArgs (bins with built-in default configs), prepare
(workspace seeding), and inspect (post-run log assertions); examples/
declares the session-title provider the shipped cordis.yml now mounts.
This commit is contained in:
Turtle
2026-07-22 15:19:35 +08:00
parent 2a9f248594
commit 06827eb868
5 changed files with 260 additions and 362 deletions

View File

@@ -37,6 +37,7 @@
"@deepseek-ai/dsh-spill-local": "workspace:*",
"@deepseek-ai/dsh-spill-policy": "workspace:*",
"@deepseek-ai/dsh-tui-demo": "workspace:*",
"@deepseek-ai/dsh-session-title-first-message-llm": "workspace:*",
"@deepseek-ai/dsh-subagent": "workspace:*",
"@deepseek-ai/dsh-subagent-acp": "workspace:*",
"@deepseek-ai/dsh-subagent-fork": "workspace:*",

View File

@@ -1,10 +1,9 @@
import { spawn } from 'node:child_process'
import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { describe, expect, it } from 'vitest'
import { LOADER_SMOKE_TEST_TIMEOUT_MS, resolveExampleLaunch } from '@deepseek-ai/dsh-loader-smoke'
import { LOADER_SMOKE_TEST_TIMEOUT_MS } from '@deepseek-ai/dsh-loader-smoke'
import { runTuiPtySmoke, type TuiPtySmokeOptions } from './pty-harness.ts'
const binScript = fileURLToPath(new URL('../../../packages/examples/tui-demo/src/bin.ts', import.meta.url))
const dshBinScript = fileURLToPath(new URL('../../../apps/cli/src/bin.ts', import.meta.url))
@@ -13,204 +12,25 @@ const codeModeConfigPath = fileURLToPath(new URL('../code-mode.cordis.yml', impo
const scriptedConfigPath = fileURLToPath(new URL('./fixtures/tui-scripted.cordis.yml', import.meta.url))
const tsconfigPath = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url))
const PTY_DRIVER = String.raw`
import errno, json, os, pty, select, signal, sys, time
node, launch_args_json, launch_env_json, cwd, resume_session_id, scenario, boot_marker = sys.argv[1:]
env = os.environ.copy()
env.update(json.loads(launch_env_json))
env.update({
"COLUMNS": "100",
"LINES": "30",
})
# Deterministic banner: a developer shell's COLORTERM=truecolor would switch the
# banner to the per-letter gradient (one SGR per letter), breaking the literal
# DEEPSEEK assertions. The gradient path has its own unit and snapshot coverage.
env.pop("COLORTERM", None)
if resume_session_id:
env["RESUME_SESSION_ID"] = resume_session_id
pid, fd = pty.fork()
if pid == 0:
os.chdir(cwd)
os.execvpe(node, [node, *json.loads(launch_args_json)], env)
output = bytearray()
answered_question = False
opened_selector = False
selected_model = False
sent_prompt = False
sent_exit = False
deadline = time.monotonic() + 25
status = None
while time.monotonic() < deadline:
ready, _, _ = select.select([fd], [], [], 0.05)
if ready:
try:
chunk = os.read(fd, 65536)
except OSError as error:
if error.errno != errno.EIO:
raise
chunk = b""
if chunk:
output.extend(chunk)
if scenario == "conversation" and not opened_selector and b"scripted TUI ready." in output:
os.write(fd, b"/model\r")
opened_selector = True
if scenario == "conversation" and opened_selector and not selected_model and b"Select model" in output:
os.write(fd, b"\x1b[B\r")
selected_model = True
if scenario == "conversation" and selected_model and not sent_prompt and b"Model selected: tui-scripted/tui-scripted-model-pro." in output:
os.write(fd, b"exercise the TUI\r")
sent_prompt = True
if scenario == "conversation" and sent_prompt and not answered_question and b"How should the scripted run proceed?" in output:
os.write(fd, b"\r")
answered_question = True
if scenario == "conversation" and answered_question and not sent_exit and b"Decision received. Scripted TUI run complete." in output:
os.write(fd, b"/exit\r")
sent_exit = True
if scenario == "skill" and not selected_model and b"scripted TUI ready." in output:
os.write(fd, b"/model tui-scripted/tui-scripted-model-pro\r")
selected_model = True
if scenario == "skill" and selected_model and not sent_prompt and b"Model selected: tui-scripted/tui-scripted-model-pro." in output:
os.write(fd, b"/skill:scripted-skill\r")
sent_prompt = True
if scenario == "skill" and sent_prompt and not sent_exit and b"Scripted skill body received." in output:
os.write(fd, b"/exit\r")
sent_exit = True
if scenario == "boot" and not sent_exit and boot_marker.encode() in output:
os.write(fd, b"/exit\r")
sent_exit = True
waited, candidate = os.waitpid(pid, os.WNOHANG)
if waited == pid:
status = candidate
break
if status is None:
os.kill(pid, signal.SIGKILL)
_, status = os.waitpid(pid, 0)
sys.stdout.buffer.write(output)
if scenario == "resume-failure":
if b'ui-tui: session "missing-session" failed to start:' not in output:
sys.stderr.write("TUI did not render the startup failure before timeout\n")
sys.exit(126)
if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 1:
sys.stderr.write("TUI startup failure did not exit with status 1\n")
sys.exit(127)
elif scenario == "conversation":
if not sent_prompt:
sys.stderr.write("TUI did not render the scripted welcome marker before timeout\n")
sys.exit(128)
if not answered_question:
sys.stderr.write("TUI did not render the user-question dialog before timeout\n")
sys.exit(129)
if not sent_exit:
sys.stderr.write("TUI did not finish the scripted tool round-trip before timeout\n")
sys.exit(130)
if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0:
sys.stderr.write("TUI scripted conversation did not exit cleanly\n")
sys.exit(131)
elif scenario == "skill":
if not sent_prompt:
sys.stderr.write("TUI did not render the scripted welcome marker before typing /skill:\n")
sys.exit(132)
if b"Scripted skill body received." not in output:
sys.stderr.write("TUI did not deliver the loaded skill body to the model before timeout\n")
sys.exit(133)
if not sent_exit:
sys.stderr.write("TUI did not reach idle to accept /exit after the skill turn\n")
sys.exit(134)
if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0:
sys.stderr.write("TUI skill scenario did not exit cleanly\n")
sys.exit(135)
else:
if not sent_exit:
sys.stderr.write("TUI did not render its welcome marker before timeout\n")
sys.exit(124)
if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0:
sys.stderr.write("TUI child did not exit cleanly\n")
sys.exit(125)
`
interface TuiLoaderSmokeOptions {
config?: string
resumeSessionId?: string
scenario?: 'boot' | 'conversation' | 'resume-failure' | 'skill'
/** Welcome text the boot scenario waits for before sending `/exit`. */
bootMarker?: string
/** Bin to boot; defaults to the tui-demo bin (the dsh CLI tests override). */
srcBin?: string
/** Argument vector for the bin; defaults to `[config]`. */
configArgs?: string[]
/** Files written into the isolated Harness home (`$DSH_HOME`) before launch. */
personalFiles?: Record<string, string>
/** Skill bundles written under the isolated agents home (`.agents/skills/`) before launch, keyed by path below that root. */
skillFiles?: Record<string, string>
/** Runs against the workspace `cwd` after a clean exit, before it is removed. */
inspect?: (cwd: string) => Promise<void>
}
async function runTuiLoaderSmoke(options: TuiLoaderSmokeOptions = {}): Promise<string> {
const cwd = await mkdtemp(join(tmpdir(), 'tui-agent-smoke-'))
try {
// Personal config is always isolated from the developer's real ~/.dsh;
// a test opts into an overlay by supplying files under the Harness home.
const dshHome = join(cwd, '.dsh')
for (const [name, content] of Object.entries(options.personalFiles ?? {})) {
await mkdir(dshHome, { recursive: true })
await writeFile(join(dshHome, name), content)
}
// The child chdirs to this cwd and the scripted config roots fs-local here,
// so a skill dropped under DSH_AGENTS_HOME's `skills/` root is discoverable
// and its body readable through the same tree the model-facing stack uses.
const skillsRoot = join(cwd, '.agents', 'skills')
for (const [name, content] of Object.entries(options.skillFiles ?? {})) {
const file = join(skillsRoot, name)
/**
* Seed the harness workspace: personal files land in the isolated Harness home
* (`.dsh`), skill bundles under the agents home's `skills/` root — the same
* trees `$DSH_HOME` / `$DSH_AGENTS_HOME` point the child at.
*/
function seedWorkspace(
files: { personal?: Record<string, string>; skills?: Record<string, string> },
): (cwd: string) => Promise<void> {
return async (cwd) => {
for (const [name, content] of Object.entries(files.personal ?? {})) {
const file = join(cwd, '.dsh', name)
await mkdir(dirname(file), { recursive: true })
await writeFile(file, content)
}
for (const [name, content] of Object.entries(files.skills ?? {})) {
const file = join(cwd, '.agents', 'skills', name)
await mkdir(dirname(file), { recursive: true })
await writeFile(file, content)
}
const launch = resolveExampleLaunch({
srcBin: options.srcBin ?? binScript,
configArgs: options.configArgs ?? [options.config ?? configPath],
tsconfigPath,
exposeInternals: true,
env: {
DEEPSEEK_API_KEY: 'keyless-tui-no-call',
DSH_HOME: dshHome,
DSH_AGENTS_HOME: join(cwd, '.agents'),
},
})
return await new Promise((resolve, reject) => {
const child = spawn('python3', [
'-c',
PTY_DRIVER,
launch.command,
JSON.stringify(launch.args),
JSON.stringify(launch.env),
cwd,
options.resumeSessionId ?? '',
options.scenario ?? 'boot',
// With no configured welcome the borderless banner sweeps in; its
// detail line's session id (`main-session-<uuid>`) renders only once
// the sweep reaches it, so it marks a settled banner.
options.bootMarker ?? 'main-session-',
], { stdio: ['ignore', 'pipe', 'pipe'] })
let stdout = ''
let stderr = ''
child.stdout.setEncoding('utf8')
child.stdout.on('data', (chunk: string) => { stdout += chunk })
child.stderr.setEncoding('utf8')
child.stderr.on('data', (chunk: string) => { stderr += chunk })
child.once('error', reject)
child.once('exit', (code) => {
if (code !== 0) {
reject(new Error(`TUI PTY smoke exited ${String(code)}. stdout:\n${stdout}\nstderr:\n${stderr}`))
return
}
// Inspect the workspace before `finally` removes it (e.g. the session log).
void (options.inspect?.(cwd) ?? Promise.resolve()).then(() => { resolve(stdout) }, reject)
})
})
} finally {
await rm(cwd, { recursive: true, force: true })
}
}
@@ -229,12 +49,35 @@ async function readLoggedSystemPrompt(cwd: string): Promise<string> {
throw new Error(`session log ${logRelPath} has no request/header event`)
}
/** Shared defaults: the keyless key, the tui-demo bin, and the live cordis.yml. */
function smoke(overrides: Partial<TuiPtySmokeOptions> & { label: string }): Promise<string> {
return runTuiPtySmoke({
tempDirPrefix: 'tui-agent-smoke-',
binScript,
configPath,
tsconfigPath,
env: { DEEPSEEK_API_KEY: 'keyless-tui-no-call' },
...overrides,
})
}
// The scripted conversation switches to the pro model first: the scripted
// adapter proves routing + prompt variables by rejecting tool-ful calls on any
// other route (see fixtures/tui-scripted-llm.ts).
const SELECT_PRO_MODEL = [
{ waitFor: 'scripted TUI ready.', send: '/model\r' },
{ waitFor: 'Select model', send: '\x1b[B\r' },
] as const
describe('tui-agent keyless smoke (real Loader tree in a PTY)', () => {
it('boots pi-tui, sweeps the borderless banner in, accepts /exit, and restores the terminal', async () => {
const output = await runTuiLoaderSmoke()
// With no configured welcome the borderless banner sweeps in left-to-right;
// the boot scenario waits for the detail line's session id, which renders
// only once the sweep reaches it.
// the detail line's session id (`main-session-<uuid>`) renders only once
// the sweep reaches it, so it marks a settled banner.
const output = await smoke({
label: 'tui-agent boot',
actions: [{ waitFor: 'main-session-', send: '/exit\r' }],
})
expect(output).toContain('DEEPSEEK')
expect(output).toContain('HARNESS')
expect(output).toContain('main-session-')
@@ -244,8 +87,24 @@ describe('tui-agent keyless smoke (real Loader tree in a PTY)', () => {
expect(output).toContain('\u001B[?2004l')
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
it('streams a response, answers a user-question dialog, completes the tool round-trip, and exits cleanly', async () => {
const output = await runTuiLoaderSmoke({ config: scriptedConfigPath, scenario: 'conversation' })
it('switches models, streams a response, answers a user-question dialog, and exits cleanly', async () => {
const output = await smoke({
label: 'tui-agent conversation',
tempDirPrefix: 'tui-agent-conversation-',
configPath: scriptedConfigPath,
actions: [
...SELECT_PRO_MODEL,
{ waitFor: 'Model selected: tui-scripted/tui-scripted-model-pro.', send: 'exercise the TUI\r' },
{ waitFor: 'How should the scripted run proceed?', send: '\r' },
{ waitFor: 'Decision received. Scripted TUI run complete.', send: '' },
// Session title: the first user message drives the first-message-llm
// provider's tool-less title call; the scripted adapter answers it, the
// accepted title lands in the log, and the TUI renders the terminal
// window title as `<session title> — <configured title>` via OSC 0.
// Gating /exit on it keeps the assertion race-free.
{ waitFor: 'scripted session title — DeepSeek Harness', send: '/exit\r' },
],
})
expect(output).toContain('I need one decision before I continue.')
expect(output).toContain(String.raw`\x1b]2;MODEL_CONTROLLED\x07`)
expect(output).toContain(String.raw`\x1b[999CMODEL_CURSOR`)
@@ -253,13 +112,7 @@ describe('tui-agent keyless smoke (real Loader tree in a PTY)', () => {
expect(output).not.toContain('\u001B]2;MODEL_CONTROLLED\u0007')
expect(output).not.toContain('\u001B[999CMODEL_CURSOR')
expect(output).not.toContain('\u009B31mMODEL_C1')
expect(output).toContain('How should the scripted run proceed?')
expect(output).toContain('Safe')
expect(output).toContain('Decision received. Scripted TUI run complete.')
// Session title: the first user message drives the first-message-llm
// provider's tool-less title call; the scripted adapter answers it, the
// accepted title lands in the log, and the TUI renders the terminal window
// title as `<session title> — <configured title>` via OSC 0.
expect(output).toContain('\u001B]0;scripted session title — DeepSeek Harness\u0007')
expect(output).toContain('\u001B[?2004l')
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
@@ -270,20 +123,28 @@ describe('tui-agent keyless smoke (real Loader tree in a PTY)', () => {
// the local provider loads `scripted-skill` from the agents home, and the
// rendered `<skill name="…">` block reaches the model — proven by the
// scripted adapter echoing the fixture's body marker only when it arrives.
const output = await runTuiLoaderSmoke({
config: scriptedConfigPath,
scenario: 'skill',
skillFiles: {
'scripted-skill/SKILL.md': [
'---',
'name: scripted-skill',
'description: Keyless PTY proof that the skill command loads a local skill into the conversation.',
'---',
'',
'SCRIPTED SKILL BODY MARKER',
'',
].join('\n'),
},
const output = await smoke({
label: 'tui-agent skill',
tempDirPrefix: 'tui-agent-skill-',
configPath: scriptedConfigPath,
prepare: seedWorkspace({
skills: {
'scripted-skill/SKILL.md': [
'---',
'name: scripted-skill',
'description: Keyless PTY proof that the skill command loads a local skill into the conversation.',
'---',
'',
'SCRIPTED SKILL BODY MARKER',
'',
].join('\n'),
},
}),
actions: [
...SELECT_PRO_MODEL,
{ waitFor: 'Model selected: tui-scripted/tui-scripted-model-pro.', send: '/skill:scripted-skill\r' },
{ waitFor: 'Scripted skill body received.', send: '/exit\r' },
],
})
expect(output).toContain('Scripted skill body received.')
expect(output).toContain('\u001B[?2004l')
@@ -292,23 +153,39 @@ describe('tui-agent keyless smoke (real Loader tree in a PTY)', () => {
it('boots the Code Mode overlay tree, renders its banner, and exits cleanly', async () => {
// The overlay's only keyless composition proof: the include+patch tree,
// worker code runtime, and one-tool registry all mount before the banner.
const output = await runTuiLoaderSmoke({
config: codeModeConfigPath,
bootMarker: 'TUI Code Mode ready.',
const output = await smoke({
label: 'tui-agent code mode',
tempDirPrefix: 'tui-agent-code-mode-',
configPath: codeModeConfigPath,
actions: [{ waitFor: 'TUI Code Mode ready.', send: '/exit\r' }],
})
expect(output).toContain('TUI Code Mode ready.')
expect(output).toContain('\u001B[?2004l')
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
it('prints a config-resume failure and exits instead of leaving a blank terminal', async () => {
const output = await runTuiLoaderSmoke({ resumeSessionId: 'missing-session', scenario: 'resume-failure' })
const output = await smoke({
label: 'tui-agent resume failure',
tempDirPrefix: 'tui-agent-resume-',
env: {
DEEPSEEK_API_KEY: 'keyless-tui-no-call',
RESUME_SESSION_ID: 'missing-session',
},
expectedExitCode: 1,
})
expect(output).toContain('ui-tui: session "missing-session" failed to start:')
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
})
describe('dsh CLI keyless smoke (apps/cli through the same PTY)', () => {
it('boots the shipped default config with no arguments and no personal overlay', async () => {
const output = await runTuiLoaderSmoke({ srcBin: dshBinScript, configArgs: [] })
const output = await smoke({
label: 'dsh default boot',
tempDirPrefix: 'dsh-default-boot-',
binScript: dshBinScript,
configArgs: [],
actions: [{ waitFor: 'main-session-', send: '/exit\r' }],
})
expect(output).toContain('DEEPSEEK')
expect(output).toContain('main-session-')
expect(output).not.toContain('╭')
@@ -320,45 +197,55 @@ describe('dsh CLI keyless smoke (apps/cli through the same PTY)', () => {
// The whole personal-config chain in one boot: the personal .env supplies
// the variable, config.yaml patches the tui-agent entry with a `!!js`
// reference to it, and the banner renders the patched welcome verbatim.
const output = await runTuiLoaderSmoke({
srcBin: dshBinScript,
const output = await smoke({
label: 'dsh personal overlay',
tempDirPrefix: 'dsh-personal-overlay-',
binScript: dshBinScript,
configArgs: [],
bootMarker: 'PERSONAL OVERLAY READY.',
personalFiles: {
'.env': 'DSH_PERSONAL_WELCOME=PERSONAL OVERLAY READY.\n',
'config.yaml': [
'- id: tui-agent',
" name: '@deepseek-ai/dsh-tui-demo'",
' config:',
' provider: deepseek',
' model: deepseek-v4-flash',
' workspaceContext: false',
' welcome: !!js process.env.DSH_PERSONAL_WELCOME',
'',
].join('\n'),
},
prepare: seedWorkspace({
personal: {
'.env': 'DSH_PERSONAL_WELCOME=PERSONAL OVERLAY READY.\n',
'config.yaml': [
'- id: tui-agent',
" name: '@deepseek-ai/dsh-tui-demo'",
' config:',
' provider: deepseek',
' model: deepseek-v4-flash',
' workspaceContext: false',
' welcome: !!js process.env.DSH_PERSONAL_WELCOME',
'',
].join('\n'),
},
}),
actions: [{ waitFor: 'PERSONAL OVERLAY READY.', send: '/exit\r' }],
})
expect(output).toContain('PERSONAL OVERLAY READY.')
expect(output).toContain('\u001B[?2004l')
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
it('fails loud instead of booting when the personal config.yaml is invalid', async () => {
await expect(runTuiLoaderSmoke({
srcBin: dshBinScript,
const output = await smoke({
label: 'dsh invalid personal config',
tempDirPrefix: 'dsh-invalid-personal-',
binScript: dshBinScript,
configArgs: [],
personalFiles: { 'config.yaml': 'id: not-a-list\n' },
})).rejects.toThrow('must be a top-level YAML array of loader patch entries')
prepare: seedWorkspace({ personal: { 'config.yaml': 'id: not-a-list\n' } }),
expectedExitCode: 1,
})
expect(output).toContain('must be a top-level YAML array of loader patch entries')
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
it('routes the --resume flag into the config resume intake, failing loud on a missing id', async () => {
// The flag path end to end: apps/cli parses `--resume missing-session` and
// sets RESUME_SESSION_ID (the PTY driver does NOT here), the shipped
// config's `!!js` reads it, and the resume fails loud — proving the printed
// `dsh --resume <id>` hint reaches the same intake as the env var.
const output = await runTuiLoaderSmoke({
srcBin: dshBinScript,
// sets RESUME_SESSION_ID, the shipped config's `!!js` reads it, and the
// resume fails loud — proving the printed `dsh --resume <id>` hint reaches
// the same intake as the env var.
const output = await smoke({
label: 'dsh resume flag failure',
tempDirPrefix: 'dsh-resume-flag-',
binScript: dshBinScript,
configArgs: ['--resume', 'missing-session'],
scenario: 'resume-failure',
expectedExitCode: 1,
})
expect(output).toContain('ui-tui: session "missing-session" failed to start:')
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
@@ -368,10 +255,17 @@ describe('dsh CLI keyless smoke (apps/cli through the same PTY)', () => {
// this test file sits an equal depth under the same root, so the same hop applies.
const sourceRoot = fileURLToPath(new URL('../../..', import.meta.url))
let loggedSystem = ''
await runTuiLoaderSmoke({
srcBin: dshBinScript,
await smoke({
label: 'dsh source-path prompt',
tempDirPrefix: 'dsh-source-path-',
binScript: dshBinScript,
configArgs: [scriptedConfigPath],
scenario: 'conversation',
actions: [
...SELECT_PRO_MODEL,
{ waitFor: 'Model selected: tui-scripted/tui-scripted-model-pro.', send: 'exercise the TUI\r' },
{ waitFor: 'How should the scripted run proceed?', send: '\r' },
{ waitFor: 'Decision received. Scripted TUI run complete.', send: '/exit\r' },
],
inspect: async (cwd) => { loggedSystem = await readLoggedSystemPrompt(cwd) },
})
expect(loggedSystem).toContain(`Your own source code is the checkout at ${sourceRoot}; you can read it there to learn how dsh works and how to extend it.`)