DeepSeek/deepseek-harness
1
0
Fork 0
mirror of https://github.com/deepseek-ai/deepseek-harness synced 2026-08-15 21:04:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

cleanup(build): collapse native runtime payloads

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-30 01:09:54 +08:00
parent 0e53b7a8aa
commit 7118b4cfe1
4 changed files with 60 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
python/sdk-runtime/hatch_build.py
View File

@@ -39,31 +39,24 @@ class RuntimeBuildHook(BuildHookInterface):
)
platform_tag = os.environ.get("DSH_RUNTIME_PLATFORM_TAG") or _host_platform_tag()
matches = [(key, value) for key, value in _PLATFORMS.items() if value[0] == platform_tag]
matches = [value for value in _PLATFORMS.values() if value[0] == platform_tag]
if len(matches) != 1:
supported = ", ".join(value[0] for value in _PLATFORMS.values())
raise RuntimeError(
f"unsupported DSH_RUNTIME_PLATFORM_TAG {platform_tag!r}; expected one of {supported}"
)
expected_target, (_, expected_executable) = matches[0]
expected_executable = matches[0][1]
runtime_dir = Path(self.root) / "src" / "deepseek_harness_runtime" / "runtime"
runtime_files = sorted(runtime_dir.glob("dsh-jsonrpc-agent-pkg-*") if runtime_dir.is_dir() else [])
executables = [path for path in runtime_files if not path.name.endswith(_SPAWN_HELPER_SUFFIX)]
helpers = [path for path in runtime_files if path.name.endswith(_SPAWN_HELPER_SUFFIX)]
if [path.name for path in executables] != [expected_executable]:
found = ", ".join(path.name for path in executables) or "none"
expected_files = [expected_executable]
if "-macos-" in expected_executable:
expected_files.append(f"{expected_executable}{_SPAWN_HELPER_SUFFIX}")
found_files = [path.name for path in runtime_files]
if found_files != expected_files:
raise RuntimeError(
f"runtime wheel {platform_tag} must contain only {expected_executable}; found {found}"
f"runtime wheel {platform_tag} payload must be {expected_files}; found {found_files}"
)
expected_helper = f"{expected_executable}{_SPAWN_HELPER_SUFFIX}"
expected_helpers = [expected_helper] if expected_target.startswith("macos-") else []
if [path.name for path in helpers] != expected_helpers:
expected = ", ".join(expected_helpers) or "none"
found = ", ".join(path.name for path in helpers) or "none"
raise RuntimeError(
f"runtime wheel {platform_tag} helper payload mismatch: expected {expected}; found {found}"
)
for executable in [executables[0], *helpers]:
for executable in runtime_files:
if executable.stat().st_mode & stat.S_IXUSR == 0:
raise RuntimeError(f"runtime executable is not executable: {executable}")
build_data["pure_python"] = False

30
python/sdk/tests/test_release_version.py
View File

@@ -68,7 +68,7 @@ def test_stage_runtime_rejects_missing_spawn_helper(tmp_path: Path) -> None:
executable.write_bytes(b"runtime")
executable.chmod(0o755)
with pytest.raises(FileNotFoundError, match="spawn helper"):
with pytest.raises(FileNotFoundError, match="spawn-helper"):
build_python_release.stage_runtime(
tmp_path / "staging",
"1.2.3",
@@ -77,32 +77,8 @@ def test_stage_runtime_rejects_missing_spawn_helper(tmp_path: Path) -> None:
)
def test_stage_runtime_rejects_unsupported_executable_name(tmp_path: Path) -> None:
executable = tmp_path / "custom-runtime"
executable.write_bytes(b"runtime")
executable.chmod(0o755)
with pytest.raises(
ValueError,
match=(
"unsupported runtime executable 'custom-runtime'; expected one of: "
"dsh-jsonrpc-agent-pkg-linux-arm64, dsh-jsonrpc-agent-pkg-linux-x64, "
"dsh-jsonrpc-agent-pkg-macos-arm64"
),
):
build_python_release.stage_runtime(
tmp_path / "staging",
"1.2.3",
executable,
executable.name,
)
@pytest.mark.parametrize("target", ["linux-x64", "linux-arm64"])
def test_stage_runtime_copies_linux_executable_without_spawn_helper(
tmp_path: Path, target: str
) -> None:
executable = tmp_path / f"dsh-jsonrpc-agent-pkg-{target}"
def test_stage_runtime_copies_linux_executable_without_spawn_helper(tmp_path: Path) -> None:
executable = tmp_path / "dsh-jsonrpc-agent-pkg-linux-x64"
executable.write_bytes(b"runtime")
executable.chmod(0o755)
destination = tmp_path / "staging"

75
scripts/build-exe-for-python-sdk.ts
View File

@@ -53,15 +53,6 @@ const ARCHES = ['x64', 'arm64'] as const
type Platform = (typeof PLATFORMS)[number]
type Arch = (typeof ARCHES)[number]
interface RuntimeProduct {
executable: string
spawnHelper?: string
}
function runtimeProductFiles(product: RuntimeProduct): string[] {
return [product.executable, ...(product.spawnHelper === undefined ? [] : [product.spawnHelper])]
}
function isPlatform(value: string): value is Platform {
return (PLATFORMS as readonly string[]).includes(value)
}
@@ -299,9 +290,9 @@ class SingleExeBuild {
/**
* Package one target; SEA mode accepts one target per invocation.
* @param target - the pkg target triple to build.
* @returns the canonical product path `<out>/dsh-jsonrpc-agent-pkg-<platform>-<arch>`.
* @returns the executable path and, on macOS, its helper path.
*/
async pack(target: Target): Promise<RuntimeProduct> {
async pack(target: Target): Promise<string[]> {
const product = join(this.outDir, `${OUTPUT_BASENAME}-${target.platform}-${target.arch}`)
await this.prepareNativePty(target)
if (!this.cli.dryRun) mkdirSync(this.outDir, { recursive: true })
@@ -318,7 +309,7 @@ class SingleExeBuild {
if (!this.cli.dryRun && !existsSync(product)) {
throw new Error(`build-exe-for-python-sdk: product ${product} is missing after the pkg run; inspect ${this.outDir}.`)
}
if (target.platform !== 'macos') return { executable: product }
if (target.platform !== 'macos') return [product]
const spawnHelper = `${product}${SPAWN_HELPER_SUFFIX}`
if (this.cli.dryRun) {
console.log(`build-exe-for-python-sdk: [dry-run] copy target node-pty spawn-helper to ${spawnHelper}`)
@@ -327,7 +318,7 @@ class SingleExeBuild {
await copyFile(source, spawnHelper)
await chmod(spawnHelper, statSync(source).mode & 0o777)
}
return { executable: product, spawnHelper }
return [product, spawnHelper]
}
/**
@@ -341,21 +332,19 @@ class SingleExeBuild {
if (this.cli.dryRun) console.log(`build-exe-for-python-sdk: [dry-run] rm -rf ${stagedBuild}`)
else await rm(stagedBuild, { recursive: true, force: true })
const nativePlatform = target.platform === 'macos' ? 'darwin' : 'linux'
const prebuilt = join(stagedRoot, 'prebuilds', `${nativePlatform}-${target.arch}`, 'pty.node')
const source = join(root, 'packages', 'pty', 'pty-local', 'node_modules', 'node-pty', 'build', 'Release', 'pty.node')
const destination = join(stagedBuild, 'Release', 'pty.node')
if (this.cli.dryRun) {
if (target.platform === 'linux') console.log(`build-exe-for-python-sdk: [dry-run] cp ${source} ${destination}`)
return
}
if (existsSync(prebuilt)) return
if (target.platform === 'macos') return
const host = Target.host()
if (target.platform !== host.platform || target.arch !== host.arch || !existsSync(source)) {
throw new Error(
`build-exe-for-python-sdk: node-pty native addon for ${target.platform}-${target.arch} is missing; `
+ `checked ${prebuilt}, ${source}. Build the Linux runtime on its target architecture.`,
+ `checked ${source}. Build the Linux runtime on its target architecture.`,
)
}
await mkdir(dirname(destination), { recursive: true })
@@ -368,19 +357,11 @@ class SingleExeBuild {
* @returns a physical executable outside pkg's virtual snapshot.
*/
private resolveSpawnHelper(target: Target): string {
const nodePtyRoot = join(this.staging, 'node_modules', 'node-pty')
const candidates = [
join(nodePtyRoot, 'prebuilds', `darwin-${target.arch}`, 'spawn-helper'),
]
const host = Target.host()
if (target.platform === host.platform && target.arch === host.arch) {
candidates.push(join(root, 'packages', 'pty', 'pty-local', 'node_modules', 'node-pty', 'build', 'Release', 'spawn-helper'))
}
const helper = candidates.find(candidate => existsSync(candidate))
if (helper === undefined) {
const helper = join(this.staging, 'node_modules', 'node-pty', 'prebuilds', `darwin-${target.arch}`, 'spawn-helper')
if (!existsSync(helper)) {
throw new Error(
`build-exe-for-python-sdk: node-pty spawn-helper for ${target.platform}-${target.arch} is missing; `
+ `checked ${candidates.join(', ')}. Build each runtime on its target platform and architecture.`,
+ `checked ${helper}. Build each runtime on its target platform and architecture.`,
)
}
if (statSync(helper).mode & 0o111) return helper
@@ -391,43 +372,37 @@ class SingleExeBuild {
* Print each product path and, outside dry-run mode, its size.
* @param products - the product paths returned by {@link pack}.
*/
printProducts(products: RuntimeProduct[]): void {
printProducts(products: string[]): void {
console.log(this.cli.dryRun ? 'build-exe-for-python-sdk: [dry-run] would produce:' : 'build-exe-for-python-sdk: products:')
for (const product of products) {
for (const path of products) {
if (this.cli.dryRun) {
for (const path of runtimeProductFiles(product)) console.log(` ${path}`)
console.log(` ${path}`)
continue
}
for (const path of runtimeProductFiles(product)) {
const megabytes = statSync(path).size / (1024 * 1024)
console.log(` ${path} (${megabytes.toFixed(1)} MB)`)
}
const megabytes = statSync(path).size / (1024 * 1024)
console.log(` ${path} (${megabytes.toFixed(1)} MB)`)
}
}
/**
* Copy each executable into the Python runtime package. The deployed node
* Copy each product into the Python runtime package. The deployed node
* carrier is already in place, and `dist-exe/` retains upload copies.
* @param products - the product paths returned by {@link pack}.
*/
async syncToPythonRuntime(products: RuntimeProduct[]): Promise<void> {
async syncToPythonRuntime(products: string[]): Promise<void> {
const destDir = resolve(root, PYTHON_RUNTIME_DIR)
if (this.cli.dryRun) {
for (const product of products) {
for (const path of runtimeProductFiles(product)) {
console.log(`build-exe-for-python-sdk: [dry-run] cp ${path} ${join(destDir, basename(path))}`)
}
for (const path of products) {
console.log(`build-exe-for-python-sdk: [dry-run] cp ${path} ${join(destDir, basename(path))}`)
}
return
}
mkdirSync(destDir, { recursive: true })
for (const product of products) {
for (const path of runtimeProductFiles(product)) {
const destination = join(destDir, basename(path))
await copyFile(path, destination)
await chmod(destination, statSync(path).mode & 0o777)
console.log(`build-exe-for-python-sdk: synced ${destination}`)
}
for (const path of products) {
const destination = join(destDir, basename(path))
await copyFile(path, destination)
await chmod(destination, statSync(path).mode & 0o777)
console.log(`build-exe-for-python-sdk: synced ${destination}`)
}
}
@@ -476,8 +451,8 @@ async function main(): Promise<void> {
await pipeline.build()
await pipeline.deployStaging()
await pipeline.injectPkgConfig()
const products: RuntimeProduct[] = []
for (const target of cli.targets) products.push(await pipeline.pack(target))
const products: string[] = []
for (const target of cli.targets) products.push(...await pipeline.pack(target))
pipeline.printProducts(products)
await pipeline.syncToPythonRuntime(products)
}

69
scripts/build-python-release.py
View File

@@ -23,17 +23,6 @@ PLATFORMS = {
"macos-arm64": ("macosx_11_0_arm64", "dsh-jsonrpc-agent-pkg-macos-arm64"),
}
SPAWN_HELPER_SUFFIX = "-spawn-helper"
EXECUTABLE_TARGETS = {value[1]: key for key, value in PLATFORMS.items()}
def executable_target(executable_name: str) -> str:
try:
return EXECUTABLE_TARGETS[executable_name]
except KeyError as error:
supported = ", ".join(sorted(EXECUTABLE_TARGETS))
raise ValueError(
f"unsupported runtime executable {executable_name!r}; expected one of: {supported}"
) from error
def main() -> None:
@@ -144,28 +133,24 @@ def stage_sdk(destination: Path, version: str) -> None:
def stage_runtime(destination: Path, version: str, executable: Path, executable_name: str) -> None:
if not executable.is_file():
raise FileNotFoundError(f"runtime executable does not exist: {executable}")
if executable.stat().st_mode & stat.S_IXUSR == 0:
raise PermissionError(f"runtime executable is not executable: {executable}")
expected_target = executable_target(executable_name)
spawn_helper = Path(f"{executable}{SPAWN_HELPER_SUFFIX}")
if expected_target.startswith("macos-"):
if not spawn_helper.is_file():
raise FileNotFoundError(f"runtime spawn helper does not exist: {spawn_helper}")
if spawn_helper.stat().st_mode & stat.S_IXUSR == 0:
raise PermissionError(f"runtime spawn helper is not executable: {spawn_helper}")
payload = [(executable, executable_name)]
if "-macos-" in executable_name:
payload.append(
(Path(f"{executable}{SPAWN_HELPER_SUFFIX}"), f"{executable_name}{SPAWN_HELPER_SUFFIX}")
)
for source, _ in payload:
if not source.is_file():
raise FileNotFoundError(f"runtime file does not exist: {source}")
if source.stat().st_mode & stat.S_IXUSR == 0:
raise PermissionError(f"runtime file is not executable: {source}")
copy_package(ROOT / "python" / "sdk-runtime", destination)
rewrite_version(destination / "pyproject.toml", version)
runtime_dir = destination / "src" / "deepseek_harness_runtime" / "runtime"
runtime_dir.mkdir(parents=True, exist_ok=True)
destination_executable = runtime_dir / executable_name
shutil.copyfile(executable, destination_executable)
destination_executable.chmod(executable.stat().st_mode & 0o777)
if expected_target.startswith("macos-"):
destination_helper = runtime_dir / f"{executable_name}{SPAWN_HELPER_SUFFIX}"
shutil.copyfile(spawn_helper, destination_helper)
destination_helper.chmod(spawn_helper.stat().st_mode & 0o777)
for source, name in payload:
target = runtime_dir / name
shutil.copyfile(source, target)
target.chmod(source.stat().st_mode & 0o777)
def verify_wheel(
@@ -187,26 +172,18 @@ def verify_wheel(
runtime_files = [
name for name in archive.namelist() if "/runtime/dsh-jsonrpc-agent-pkg-" in name
]
helpers = [name for name in runtime_files if name.endswith(SPAWN_HELPER_SUFFIX)]
executables = [name for name in runtime_files if not name.endswith(SPAWN_HELPER_SUFFIX)]
if package == "runtime":
assert platform is not None
if len(executables) != 1 or not executables[0].endswith(f"/runtime/{platform[1]}"):
raise RuntimeError(f"{wheel} must contain exactly {platform[1]}, found {executables}")
expected_target = executable_target(platform[1])
expected_helper = f"{platform[1]}{SPAWN_HELPER_SUFFIX}"
expected_helpers = [expected_helper] if expected_target.startswith("macos-") else []
found_helpers = [Path(helper).name for helper in helpers]
if found_helpers != expected_helpers:
expected = ", ".join(expected_helpers) or "none"
found = ", ".join(found_helpers) or "none"
raise RuntimeError(
f"{wheel} runtime helper payload mismatch: expected {expected}; found {found}"
)
for executable in [executables[0], *helpers]:
mode = archive.getinfo(executable).external_attr >> 16
expected_files = [platform[1]]
if "-macos-" in platform[1]:
expected_files.append(f"{platform[1]}{SPAWN_HELPER_SUFFIX}")
found_files = sorted(Path(name).name for name in runtime_files)
if found_files != expected_files:
raise RuntimeError(f"{wheel} runtime payload must be {expected_files}, found {found_files}")
for runtime_file in runtime_files:
mode = archive.getinfo(runtime_file).external_attr >> 16
if mode & stat.S_IXUSR == 0:
raise RuntimeError(f"{wheel} runtime executable lost its executable bit: {executable}")
raise RuntimeError(f"{wheel} runtime executable lost its executable bit: {runtime_file}")
elif runtime_files:
raise RuntimeError(f"SDK wheel unexpectedly contains runtime executables: {runtime_files}")
if package == "sdk":