mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
Merge remote-tracking branch 'origin/master' into codex/trim-ai-prose
# Conflicts: # docs/config-catalog.md # docs/event-producer-consumer.md # docs/rfc/implemented/feature/2026-07-05-dynamic-workflows.md # examples/acp-agent/tests/acp.snapshot.ts # packages/code-runtime/code-runtime-worker/tests/built-lib.e2e.ts # packages/code-runtime/code-runtime-worker/tsdown.config.ts
This commit is contained in:
Tianyi Cui
392
scripts/build-exe-for-python-sdk.ts
Normal file
392
scripts/build-exe-for-python-sdk.ts
Normal file
@@ -0,0 +1,392 @@
|
||||
/**
|
||||
* Build the SDK runtime executables and Python node carrier. The fixed
|
||||
* `@yao-pkg/pkg --sea` route, deploy flags, and artifact layout are owned by
|
||||
* docs/rfc/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md.
|
||||
* The staged closure is symlink-free, and whole-tree assets cover Cordis's
|
||||
* runtime imports that pkg cannot discover statically.
|
||||
*/
|
||||
|
||||
import { spawn } from 'node:child_process'
|
||||
import { existsSync, mkdirSync, statSync } from 'node:fs'
|
||||
import { copyFile, readFile, rm, writeFile } from 'node:fs/promises'
|
||||
import { basename, join, resolve, sep } from 'node:path'
|
||||
import { parseArgs } from 'node:util'
|
||||
|
||||
const root = resolve(import.meta.dirname, '..')
|
||||
|
||||
/** The closure manifest whose dependencies define the executable. */
|
||||
const DEPLOY_ROOT_PACKAGE = 'dsh-jsonrpc-agent-pkg'
|
||||
/** The app entry inside the deployed closure. */
|
||||
const ENTRY_BIN = 'node_modules/@deepseek-ai/dsh-jsonrpc-agent/lib/bin.js'
|
||||
const OUTPUT_BASENAME = 'dsh-jsonrpc-agent-pkg'
|
||||
/** Default Node major; SEA mode requires at least Node 22. */
|
||||
const DEFAULT_NODE_RANGE = 'node24'
|
||||
/** Pinned for reproducible builds. */
|
||||
const PKG_SPEC = '@yao-pkg/pkg@6.21.0'
|
||||
const OUT_DIR = 'dist-exe'
|
||||
/** Python package destination; created when absent. */
|
||||
const PYTHON_RUNTIME_DIR = 'python/sdk-runtime/src/deepseek_harness_runtime/runtime'
|
||||
/** The deployed closure doubles as the node-mode carrier. */
|
||||
const PYTHON_NODE_SUBDIR = 'node'
|
||||
/** Documentation excluded from the generated runtime directory. */
|
||||
const DEPLOY_ONLY_DOCS = ['README.md', 'README.zh.md', 'README.i18n.yaml']
|
||||
|
||||
/**
|
||||
* Whole-tree assets cover Cordis's runtime bare-package imports, which pkg's
|
||||
* static analysis cannot see. Package manifests are explicit because bare-name
|
||||
* resolution depends on them.
|
||||
*/
|
||||
const ASSET_GLOBS = [
|
||||
'package.json',
|
||||
'node_modules/**/*.js',
|
||||
'node_modules/**/*.cjs',
|
||||
'node_modules/**/*.mjs',
|
||||
'node_modules/**/package.json',
|
||||
'node_modules/**/*.json',
|
||||
'node_modules/**/*.node',
|
||||
'node_modules/**/*.wasm',
|
||||
]
|
||||
|
||||
const PLATFORMS = ['linux', 'macos'] as const
|
||||
const ARCHES = ['x64', 'arm64'] as const
|
||||
type Platform = (typeof PLATFORMS)[number]
|
||||
type Arch = (typeof ARCHES)[number]
|
||||
|
||||
function isPlatform(value: string): value is Platform {
|
||||
return (PLATFORMS as readonly string[]).includes(value)
|
||||
}
|
||||
|
||||
function isArch(value: string): value is Arch {
|
||||
return (ARCHES as readonly string[]).includes(value)
|
||||
}
|
||||
|
||||
/**
|
||||
* A validated pkg target triple, constructed from `--targets` or the host.
|
||||
*/
|
||||
class Target {
|
||||
private constructor(
|
||||
/** pkg Node range (`node<major>`). */
|
||||
readonly nodeRange: string,
|
||||
/**
|
||||
* pkg platform tag. Windows is a documented non-goal
|
||||
* (docs/rfc/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md).
|
||||
*/
|
||||
readonly platform: Platform,
|
||||
/** pkg CPU tag. */
|
||||
readonly arch: Arch,
|
||||
) {}
|
||||
|
||||
/** The pkg `--targets` spec string `<nodeRange>-<platform>-<arch>`. */
|
||||
get spec(): string {
|
||||
return `${this.nodeRange}-${this.platform}-${this.arch}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one target spec, rejecting malformed or unsupported components.
|
||||
* @param spec - the raw triple, e.g. `node24-linux-x64`.
|
||||
* @returns the parsed target.
|
||||
*/
|
||||
static parse(spec: string): Target {
|
||||
const parts = spec.split('-')
|
||||
const [nodeRange, platform, arch] = parts
|
||||
if (parts.length !== 3 || nodeRange === undefined || platform === undefined || arch === undefined) {
|
||||
throw new Error(`build-exe-for-python-sdk: target ${JSON.stringify(spec)} must be <nodeRange>-<platform>-<arch>, e.g. node24-linux-x64.`)
|
||||
}
|
||||
if (!/^node\d+$/.test(nodeRange)) {
|
||||
throw new Error(`build-exe-for-python-sdk: target ${JSON.stringify(spec)}: node range must look like node24, got ${JSON.stringify(nodeRange)}.`)
|
||||
}
|
||||
if (!isPlatform(platform)) {
|
||||
throw new Error(`build-exe-for-python-sdk: target ${JSON.stringify(spec)}: platform must be one of ${PLATFORMS.join(', ')}, got ${JSON.stringify(platform)}.`)
|
||||
}
|
||||
if (!isArch(arch)) {
|
||||
throw new Error(`build-exe-for-python-sdk: target ${JSON.stringify(spec)}: arch must be one of ${ARCHES.join(', ')}, got ${JSON.stringify(arch)}.`)
|
||||
}
|
||||
return new Target(nodeRange, platform, arch)
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the host-platform default on Node 24.
|
||||
* @returns the host target; throws on an unsupported host platform or arch.
|
||||
*/
|
||||
static host(): Target {
|
||||
const platform = process.platform === 'darwin' ? 'macos' : process.platform === 'linux' ? 'linux' : undefined
|
||||
if (platform === undefined) {
|
||||
throw new Error(`build-exe-for-python-sdk: unsupported host platform ${process.platform}; pass --targets explicitly.`)
|
||||
}
|
||||
const arch = process.arch === 'x64' || process.arch === 'arm64' ? process.arch : undefined
|
||||
if (arch === undefined) {
|
||||
throw new Error(`build-exe-for-python-sdk: unsupported host arch ${process.arch}; pass --targets explicitly.`)
|
||||
}
|
||||
return new Target(DEFAULT_NODE_RANGE, platform, arch)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validated CLI configuration; construction owns help and parse-error exits.
|
||||
*/
|
||||
class BuildCli {
|
||||
private constructor(
|
||||
/** Build targets; defaults to the host platform only. */
|
||||
readonly targets: readonly Target[],
|
||||
/** Skip step 1 (`pnpm run build`); lib/ artifacts must already exist. */
|
||||
readonly skipBuild: boolean,
|
||||
/** Print every command and config patch instead of executing. */
|
||||
readonly dryRun: boolean,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Parse argv. Help exits 0; malformed flags exit 1; invalid or colliding
|
||||
* targets throw.
|
||||
* @param argv - the raw arguments (`process.argv.slice(2)`).
|
||||
* @returns the parsed, validated configuration.
|
||||
*/
|
||||
static parse(argv: string[]): BuildCli {
|
||||
let values: ReturnType<typeof BuildCli.parseRaw>
|
||||
try {
|
||||
values = BuildCli.parseRaw(argv)
|
||||
} catch (error) {
|
||||
console.error(`build-exe-for-python-sdk: ${error instanceof Error ? error.message : String(error)}\n`)
|
||||
console.error(BuildCli.usage())
|
||||
process.exit(1)
|
||||
}
|
||||
if (values.help) {
|
||||
console.log(BuildCli.usage())
|
||||
process.exit(0)
|
||||
}
|
||||
const targets = values.targets === undefined
|
||||
? [Target.host()]
|
||||
: values.targets.split(',').map(part => part.trim()).filter(part => part !== '').map(spec => Target.parse(spec))
|
||||
if (targets.length === 0) throw new Error('build-exe-for-python-sdk: --targets is empty.')
|
||||
const seen = new Set<string>()
|
||||
for (const target of targets) {
|
||||
const key = `${target.platform}-${target.arch}`
|
||||
if (seen.has(key)) {
|
||||
throw new Error(`build-exe-for-python-sdk: duplicate platform-arch ${key} in --targets; canonical product names would collide.`)
|
||||
}
|
||||
seen.add(key)
|
||||
}
|
||||
return new BuildCli(targets, values['skip-build'], values['dry-run'])
|
||||
}
|
||||
|
||||
private static parseRaw(argv: string[]) {
|
||||
return parseArgs({
|
||||
args: argv,
|
||||
options: {
|
||||
'targets': { type: 'string' },
|
||||
'skip-build': { type: 'boolean', default: false },
|
||||
'dry-run': { type: 'boolean', default: false },
|
||||
'help': { type: 'boolean', default: false },
|
||||
},
|
||||
}).values
|
||||
}
|
||||
|
||||
private static usage(): string {
|
||||
return [
|
||||
'Usage: pnpm exec tsx scripts/build-exe-for-python-sdk.ts [flags]',
|
||||
'',
|
||||
' --targets=<t1,t2,...> pkg targets, e.g. node24-linux-x64,node24-linux-arm64,node24-macos-arm64.',
|
||||
' Default: the host platform only (on node24).',
|
||||
' --skip-build skip `pnpm run build` (lib/ artifacts must already exist).',
|
||||
' --dry-run print every command and config patch without executing.',
|
||||
' --help print this help.',
|
||||
'',
|
||||
`Build route: ${PKG_SPEC} --sea; see docs/rfc/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md.`,
|
||||
`Stages the node carrier in ${PYTHON_RUNTIME_DIR}/${PYTHON_NODE_SUBDIR} and writes executables to ${OUT_DIR}/.`,
|
||||
].join('\n')
|
||||
}
|
||||
}
|
||||
|
||||
function pnpmBin(): string {
|
||||
return process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm'
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a command for logs and errors, quoting arguments with spaces.
|
||||
* @param command - the executable.
|
||||
* @param args - its arguments.
|
||||
* @returns the printable command line.
|
||||
*/
|
||||
function formatCommand(command: string, args: string[]): string {
|
||||
return [command, ...args].map(part => (part.includes(' ') ? JSON.stringify(part) : part)).join(' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* Sequential build pipeline. Subprocesses inherit stdio and errors include
|
||||
* the command; dry runs print commands and filesystem changes.
|
||||
*/
|
||||
class SingleExeBuild {
|
||||
/**
|
||||
* The cleared deploy target, pkg input, and Python node-mode carrier. The
|
||||
* checked-in default `cordis.yml` remains in its parent directory.
|
||||
*/
|
||||
readonly staging = resolve(root, PYTHON_RUNTIME_DIR, PYTHON_NODE_SUBDIR)
|
||||
private readonly outDir = resolve(root, OUT_DIR)
|
||||
|
||||
constructor(private readonly cli: BuildCli) {}
|
||||
|
||||
/** Verify the closure before compiling or packaging. */
|
||||
async verifyClosure(): Promise<void> {
|
||||
await this.run('runtime dependency closure', pnpmBin(), ['run', 'verify-runtime-closure'])
|
||||
}
|
||||
|
||||
/** Build all package artifacts unless `--skip-build` was passed. */
|
||||
async build(): Promise<void> {
|
||||
if (this.cli.skipBuild) {
|
||||
console.log('build-exe-for-python-sdk: skipping pnpm run build (--skip-build)')
|
||||
return
|
||||
}
|
||||
await this.run('build', pnpmBin(), ['run', 'build'])
|
||||
}
|
||||
|
||||
/** Clear and deploy the runtime closure into the node carrier. */
|
||||
async deployStaging(): Promise<void> {
|
||||
if (this.staging === root || root.startsWith(this.staging + sep)) {
|
||||
throw new Error(`build-exe-for-python-sdk: refusing to clear staging dir ${this.staging}: it contains the repo root.`)
|
||||
}
|
||||
if (this.cli.dryRun) console.log(`build-exe-for-python-sdk: [dry-run] rm -rf ${this.staging}`)
|
||||
else await rm(this.staging, { recursive: true, force: true })
|
||||
await this.run('deploy', pnpmBin(), [
|
||||
'--filter',
|
||||
DEPLOY_ROOT_PACKAGE,
|
||||
'deploy',
|
||||
'--legacy',
|
||||
'--prod',
|
||||
'--config.node-linker=hoisted',
|
||||
'--config.auto-install-peers=false',
|
||||
'--config.link-workspace-packages=true',
|
||||
this.staging,
|
||||
])
|
||||
if (this.cli.dryRun) {
|
||||
for (const name of DEPLOY_ONLY_DOCS) console.log(`build-exe-for-python-sdk: [dry-run] rm -f ${join(this.staging, name)}`)
|
||||
} else {
|
||||
await Promise.all(DEPLOY_ONLY_DOCS.map(name => rm(join(this.staging, name), { force: true })))
|
||||
}
|
||||
}
|
||||
|
||||
/** Add the executable entry and pkg assets to the staged manifest. */
|
||||
async injectPkgConfig(): Promise<void> {
|
||||
const patch = { bin: ENTRY_BIN, pkg: { assets: ASSET_GLOBS } }
|
||||
const manifestPath = join(this.staging, 'package.json')
|
||||
if (this.cli.dryRun) {
|
||||
console.log(`build-exe-for-python-sdk: [dry-run] patch ${manifestPath} with ${JSON.stringify(patch)}`)
|
||||
return
|
||||
}
|
||||
if (!existsSync(manifestPath)) {
|
||||
throw new Error(`build-exe-for-python-sdk: ${manifestPath} missing — pnpm deploy did not produce a staged package.`)
|
||||
}
|
||||
if (!existsSync(join(this.staging, ENTRY_BIN))) {
|
||||
throw new Error(`build-exe-for-python-sdk: ${join(this.staging, ENTRY_BIN)} missing — run without --skip-build so lib/ artifacts exist.`)
|
||||
}
|
||||
const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) as Record<string, unknown>
|
||||
await writeFile(manifestPath, `${JSON.stringify({ ...manifest, ...patch }, null, 2)}\n`)
|
||||
console.log(`build-exe-for-python-sdk: injected pkg config into ${manifestPath}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Package one target; SEA mode accepts one target per invocation.
|
||||
* @param target - the pkg target triple to build.
|
||||
* @returns the canonical product path `<out>/dsh-jsonrpc-agent-pkg-<platform>-<arch>`.
|
||||
*/
|
||||
async pack(target: Target): Promise<string> {
|
||||
const product = join(this.outDir, `${OUTPUT_BASENAME}-${target.platform}-${target.arch}`)
|
||||
if (!this.cli.dryRun) mkdirSync(this.outDir, { recursive: true })
|
||||
await this.run(`pkg ${target.spec}`, pnpmBin(), [
|
||||
'dlx',
|
||||
PKG_SPEC,
|
||||
this.staging,
|
||||
'--sea',
|
||||
'--targets',
|
||||
target.spec,
|
||||
'--output',
|
||||
product,
|
||||
])
|
||||
if (!this.cli.dryRun && !existsSync(product)) {
|
||||
throw new Error(`build-exe-for-python-sdk: product ${product} is missing after the pkg run; inspect ${this.outDir}.`)
|
||||
}
|
||||
return product
|
||||
}
|
||||
|
||||
/**
|
||||
* Print each product path and, outside dry-run mode, its size.
|
||||
* @param products - the product paths returned by {@link pack}.
|
||||
*/
|
||||
printProducts(products: string[]): void {
|
||||
console.log(this.cli.dryRun ? 'build-exe-for-python-sdk: [dry-run] would produce:' : 'build-exe-for-python-sdk: products:')
|
||||
for (const product of products) {
|
||||
if (this.cli.dryRun) {
|
||||
console.log(` ${product}`)
|
||||
continue
|
||||
}
|
||||
const megabytes = statSync(product).size / (1024 * 1024)
|
||||
console.log(` ${product} (${megabytes.toFixed(1)} MB)`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy each executable into the Python runtime package. The deployed node
|
||||
* carrier is already in place, and `dist-exe/` retains upload copies.
|
||||
* @param products - the product paths returned by {@link pack}.
|
||||
*/
|
||||
async syncToPythonRuntime(products: string[]): Promise<void> {
|
||||
const destDir = resolve(root, PYTHON_RUNTIME_DIR)
|
||||
if (this.cli.dryRun) {
|
||||
for (const product of products) {
|
||||
console.log(`build-exe-for-python-sdk: [dry-run] cp ${product} ${join(destDir, basename(product))}`)
|
||||
}
|
||||
return
|
||||
}
|
||||
mkdirSync(destDir, { recursive: true })
|
||||
for (const product of products) {
|
||||
const destination = join(destDir, basename(product))
|
||||
await copyFile(product, destination)
|
||||
console.log(`build-exe-for-python-sdk: synced ${destination}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one subprocess with inherited stdio. Spawn and non-zero-exit errors
|
||||
* include the command; dry runs only print it.
|
||||
* @param label - the step name used in logs and error messages.
|
||||
* @param command - the executable.
|
||||
* @param args - its arguments.
|
||||
*/
|
||||
private async run(label: string, command: string, args: string[]): Promise<void> {
|
||||
const printable = formatCommand(command, args)
|
||||
if (this.cli.dryRun) {
|
||||
console.log(`build-exe-for-python-sdk: [dry-run] ${printable}`)
|
||||
return
|
||||
}
|
||||
console.log(`build-exe-for-python-sdk: ${label}: ${printable}`)
|
||||
await new Promise<void>((resolvePromise, reject) => {
|
||||
const child = spawn(command, args, { cwd: root, stdio: 'inherit' })
|
||||
child.once('error', (error) => {
|
||||
reject(new Error(`build-exe-for-python-sdk: ${label} failed to spawn: ${error.message} (${printable})`))
|
||||
})
|
||||
child.once('exit', (code, signal) => {
|
||||
if (code === 0) {
|
||||
resolvePromise()
|
||||
return
|
||||
}
|
||||
const cause = code === null ? `signal ${signal ?? 'unknown'}` : `exit code ${code}`
|
||||
reject(new Error(`build-exe-for-python-sdk: ${label} failed (${cause}): ${printable}`))
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const cli = BuildCli.parse(process.argv.slice(2))
|
||||
const pipeline = new SingleExeBuild(cli)
|
||||
console.log(`build-exe-for-python-sdk: targets: ${cli.targets.map(target => target.spec).join(', ')}`)
|
||||
console.log(`build-exe-for-python-sdk: staging: ${pipeline.staging}`)
|
||||
await pipeline.verifyClosure()
|
||||
await pipeline.build()
|
||||
await pipeline.deployStaging()
|
||||
await pipeline.injectPkgConfig()
|
||||
const products: string[] = []
|
||||
for (const target of cli.targets) products.push(await pipeline.pack(target))
|
||||
pipeline.printProducts(products)
|
||||
await pipeline.syncToPythonRuntime(products)
|
||||
}
|
||||
|
||||
await main()
|
||||
Reference in New Issue
Block a user