mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
346 lines
17 KiB
TypeScript
346 lines
17 KiB
TypeScript
import { readFileSync } from 'node:fs'
|
|
import { resolve } from 'node:path'
|
|
import * as yaml from 'js-yaml'
|
|
import { describe, expect, it } from 'vitest'
|
|
|
|
const root = resolve(import.meta.dirname, '..')
|
|
const runnerPrivatePnpmDestination = '${{ runner.temp }}/setup-pnpm'
|
|
|
|
describe('CI workflow', () => {
|
|
it('isolates every pnpm action setup destination per runner', () => {
|
|
const workflow: unknown = yaml.load(readFileSync(resolve(root, '.github/workflows/ci.yml'), 'utf8'))
|
|
if (!isRecord(workflow) || !isRecord(workflow.jobs)) throw new TypeError('CI workflow must define jobs')
|
|
|
|
const setups = Object.entries(workflow.jobs).flatMap(([jobName, job]) => {
|
|
if (!isRecord(job) || !Array.isArray(job.steps)) return []
|
|
return job.steps.flatMap((step) => {
|
|
if (!isRecord(step) || typeof step.uses !== 'string' || !step.uses.startsWith('pnpm/action-setup@')) return []
|
|
return [{ jobName, step }]
|
|
})
|
|
})
|
|
|
|
expect(setups.length).toBeGreaterThan(0)
|
|
for (const { jobName, step } of setups) {
|
|
expect(step, `${jobName} must not share pnpm/action-setup's default destination`).toMatchObject({
|
|
with: { dest: runnerPrivatePnpmDestination },
|
|
})
|
|
}
|
|
})
|
|
|
|
it('keeps a required Wine Windows job, a non-blocking native Windows job with failover, and a master-only standby', () => {
|
|
const workflow = loadWorkflow('.github/workflows/ci.yml')
|
|
if (!isRecord(workflow.jobs)
|
|
|| !isRecord(workflow.jobs.windows)
|
|
|| !isRecord(workflow.jobs['windows-native'])
|
|
|| !isRecord(workflow.jobs['wine-apt-cache'])
|
|
|| !isRecord(workflow.jobs['serial-windows'])
|
|
|| !isRecord(workflow.jobs['all-checks-passed'])) {
|
|
throw new TypeError('CI workflow must define windows, windows-native, wine-apt-cache, serial-windows, and all-checks-passed jobs')
|
|
}
|
|
|
|
const windows = workflow.jobs.windows
|
|
const windowsNative = workflow.jobs['windows-native']
|
|
const wineAptCache = workflow.jobs['wine-apt-cache']
|
|
const serialWindows = workflow.jobs['serial-windows']
|
|
const aggregate = workflow.jobs['all-checks-passed']
|
|
if (!Array.isArray(windows.steps) || !Array.isArray(aggregate.needs)) {
|
|
throw new TypeError('Windows job must define steps and the aggregate must define needs')
|
|
}
|
|
const commandSteps = windows.steps.filter((step): step is Record<string, unknown> & { run: string } => (
|
|
isRecord(step) && typeof step.run === 'string'
|
|
))
|
|
|
|
// Required PR job: Wine on ubuntu-latest, runs wine-windows-gates.sh.
|
|
expect(windows['runs-on']).toBe('ubuntu-latest')
|
|
expect(windows.name).toBe('windows node 24 / wine blocking')
|
|
expect(windows.if).toBe("github.event_name == 'pull_request'")
|
|
expect(commandSteps.some(step => step.run.includes('wine-windows-gates.sh'))).toBe(true)
|
|
|
|
// windows-native: non-blocking native job with failover, runs windows-complete.
|
|
expect(typeof windowsNative['runs-on']).toBe('string')
|
|
expect(windowsNative['runs-on']).toContain('DSH_CI_FAILOVER')
|
|
expect(windowsNative['runs-on']).toContain('self-hosted')
|
|
expect(windowsNative['runs-on']).toContain('dsh-win-ci')
|
|
expect(windowsNative['runs-on']).toContain('dsh-windows-2025-16core')
|
|
expect(windowsNative.name).toBe('windows node 24 / native complete')
|
|
expect(windowsNative.if).toBe("github.event_name == 'pull_request'")
|
|
const nativeCommandSteps = (windowsNative.steps as unknown[]).filter((step): step is Record<string, unknown> & { run: string } => (
|
|
isRecord(step) && typeof step.run === 'string'
|
|
))
|
|
expect(nativeCommandSteps.map(step => step.run)).toContain('pnpm run check:ci:windows-complete')
|
|
|
|
// wine-apt-cache: master-only, seeds the Wine apt cache.
|
|
expect(wineAptCache.if).toBe("github.event_name == 'push' && github.ref == 'refs/heads/master'")
|
|
expect(wineAptCache['runs-on']).toBe('ubuntu-latest')
|
|
|
|
// serial-windows: master-only standby, self-hosted, non-blocking.
|
|
expect(serialWindows.if).toBe("github.event_name == 'push' && github.ref == 'refs/heads/master'")
|
|
expect(serialWindows['runs-on']).toEqual(['self-hosted', 'dsh-win-ci', 'windows'])
|
|
expect(serialWindows.name).toBe('serial / windows (self-hosted standby)')
|
|
|
|
// Aggregate: Wine `windows` required, native `windows-native` excluded.
|
|
expect(aggregate.needs).toContain('windows')
|
|
expect(aggregate.needs).not.toContain('windows-native')
|
|
expect(aggregate.needs).not.toContain('serial-windows')
|
|
})
|
|
|
|
it('keeps supported LSP source under native Windows coverage', () => {
|
|
const config = readFileSync(resolve(root, 'vitest.config.ts'), 'utf8')
|
|
|
|
expect(config).not.toContain('packages/lsp/lsp-local/src/connection.ts')
|
|
expect(config).not.toContain('packages/lsp/lsp-local/src/index.ts')
|
|
expect(config).not.toContain('packages/lsp/lsp-local/src/instance.ts')
|
|
})
|
|
|
|
it('requires one release-shaped Python runtime target on every pull request', () => {
|
|
const workflow = loadWorkflow('.github/workflows/ci.yml')
|
|
const pythonRuntime = workflowJob(workflow, 'python-runtime')
|
|
const aggregate = workflowJob(workflow, 'all-checks-passed')
|
|
if (!Array.isArray(aggregate.needs)) {
|
|
throw new TypeError('CI aggregate must define required job dependencies')
|
|
}
|
|
|
|
expect(pythonRuntime).toMatchObject({
|
|
if: "github.event_name == 'pull_request'",
|
|
name: 'python runtime / release-shaped Linux x64',
|
|
uses: './.github/workflows/build-exe-for-python-sdk.yml',
|
|
with: {
|
|
targets: 'node24-linux-x64',
|
|
ci: true,
|
|
},
|
|
})
|
|
expect(aggregate.needs).toContain('python-runtime')
|
|
})
|
|
|
|
it('keeps every Vitest project process-isolated on native Windows', () => {
|
|
const config = readFileSync(resolve(root, 'vitest.config.ts'), 'utf8')
|
|
|
|
expect(config).not.toContain("pool: process.platform === 'win32' ? 'threads' : 'forks'")
|
|
expect(config.match(/pool: 'forks'/g)).toHaveLength(2)
|
|
})
|
|
})
|
|
|
|
describe('E2B e2e workflow', () => {
|
|
it('is manual-only and fails loud before running the focused live suite', () => {
|
|
const workflow = loadWorkflow('.github/workflows/e2b-e2e.yml')
|
|
expect(workflow.on).toEqual({ workflow_dispatch: null })
|
|
if (!isRecord(workflow.jobs) || !isRecord(workflow.jobs.e2b) || !Array.isArray(workflow.jobs.e2b.steps)) {
|
|
throw new TypeError('E2B e2e workflow must define the e2b job steps')
|
|
}
|
|
|
|
const steps = workflow.jobs.e2b.steps.filter(isRecord)
|
|
const preflight = steps.find(step => step.name === 'Preflight (require E2B API key)')
|
|
const e2b = steps.find(step => step.name === 'E2B tests (live sandbox)')
|
|
|
|
expect(preflight).toMatchObject({
|
|
env: { E2B_API_KEY: '${{ secrets.E2B_API_KEY_EXTERNAL }}' },
|
|
})
|
|
expect(preflight?.run).toContain('E2B_API_KEY_EXTERNAL repository secret')
|
|
expect(e2b).toMatchObject({
|
|
env: {
|
|
E2B_API_KEY: '${{ secrets.E2B_API_KEY_EXTERNAL }}',
|
|
DSH_E2E_MAX_WORKERS: '1',
|
|
DSH_EXAMPLE_MODE: 'lib',
|
|
},
|
|
})
|
|
expect(e2b?.run).toContain('packages/e2b/e2b/tests/composition.e2e.ts')
|
|
})
|
|
})
|
|
|
|
describe('Python release workflows', () => {
|
|
it('keeps complete wheel validation separate from protected public publication', () => {
|
|
const workflow = loadWorkflow('.github/workflows/python-release.yml')
|
|
const dispatch = workflowEvent(workflow, 'workflow_dispatch')
|
|
const pullRequest = workflowEvent(workflow, 'pull_request')
|
|
const build = workflowJob(workflow, 'build')
|
|
const pythonCompat = workflowJob(workflow, 'python-compat')
|
|
const validate = workflowJob(workflow, 'validate')
|
|
const publishRuntime = workflowJob(workflow, 'publish-runtime')
|
|
const publishSdk = workflowJob(workflow, 'publish-sdk')
|
|
if (!isRecord(dispatch.inputs)
|
|
|| !isRecord(dispatch.inputs.publish)
|
|
|| !Array.isArray(pythonCompat.steps)
|
|
|| !Array.isArray(validate.steps)
|
|
|| !Array.isArray(publishRuntime.steps)
|
|
|| !Array.isArray(publishSdk.steps)) {
|
|
throw new TypeError('Python release workflow must define publish input and release steps')
|
|
}
|
|
|
|
expect(dispatch.inputs.publish).toMatchObject({ type: 'boolean', default: false })
|
|
expect(pullRequest).toEqual({ types: ['labeled'] })
|
|
expect(build).toMatchObject({
|
|
if: "github.event_name == 'workflow_dispatch' || github.event.label.name == 'python-release-dry-run'",
|
|
uses: './.github/workflows/build-exe-for-python-sdk.yml',
|
|
with: {
|
|
targets: 'node24-linux-x64,node24-linux-arm64,node24-macos-arm64',
|
|
release: true,
|
|
},
|
|
})
|
|
expect(pythonCompat.strategy).toMatchObject({ matrix: { python: ['3.10', '3.14'] } })
|
|
expect(JSON.stringify(pythonCompat.steps)).toContain('deepseek-harness-sdk==${{ steps.compatibility-version.outputs.version }}')
|
|
const validateSteps = JSON.stringify(validate.steps)
|
|
const authorize = validate.steps.filter(isRecord).find(step => step.name === 'Authorize publication request')
|
|
if (!isRecord(authorize) || typeof authorize.run !== 'string') {
|
|
throw new TypeError('Python release validation must authorize publication requests')
|
|
}
|
|
expect(validateSteps).toContain('PUBLIC_PYPI_RELEASE_ENABLED')
|
|
expect(authorize).toMatchObject({
|
|
env: {
|
|
PYPI_PUBLISHER_REPOSITORY: '${{ vars.PYPI_PUBLISHER_REPOSITORY }}',
|
|
REPOSITORY: '${{ github.repository }}',
|
|
},
|
|
})
|
|
expect(authorize.run).toContain('[ "$REPOSITORY" = "$PYPI_PUBLISHER_REPOSITORY" ]')
|
|
expect(validateSteps).toContain('100000000')
|
|
expect(publishRuntime).toMatchObject({
|
|
if: "github.event_name == 'workflow_dispatch' && inputs.publish",
|
|
needs: 'validate',
|
|
environment: 'pypi-runtime',
|
|
permissions: { contents: 'read', 'id-token': 'write' },
|
|
})
|
|
expect(publishSdk).toMatchObject({
|
|
if: "github.event_name == 'workflow_dispatch' && inputs.publish",
|
|
needs: ['validate', 'publish-runtime'],
|
|
environment: 'pypi',
|
|
permissions: { contents: 'read', 'id-token': 'write' },
|
|
})
|
|
const runtimeSteps = publishRuntime.steps.filter(isRecord)
|
|
const sdkSteps = publishSdk.steps.filter(isRecord)
|
|
const runtimePublish = runtimeSteps.find(step => step.name === 'Publish runtime wheels')
|
|
const sdkPublish = sdkSteps.find(step => step.name === 'Publish SDK wheel')
|
|
const runtimeHashes = runtimeSteps.find(step => step.name === 'Verify release artifact hashes')
|
|
const sdkHashes = sdkSteps.find(step => step.name === 'Verify release artifact hashes')
|
|
expect([...runtimeSteps, ...sdkSteps].some(
|
|
step => typeof step.uses === 'string' && step.uses.startsWith('actions/checkout@'),
|
|
)).toBe(false)
|
|
expect([...runtimeSteps, ...sdkSteps].filter(
|
|
step => step.uses === 'pypa/gh-action-pypi-publish@release/v1',
|
|
)).toHaveLength(2)
|
|
expect(runtimePublish).toMatchObject({
|
|
with: { 'packages-dir': 'dist/runtime/', attestations: false },
|
|
})
|
|
expect(sdkPublish).toMatchObject({
|
|
with: { 'packages-dir': 'dist/sdk/', attestations: false },
|
|
})
|
|
expect(runtimeHashes).toMatchObject({ run: 'cd dist && sha256sum -c SHA256SUMS' })
|
|
expect(sdkHashes).toMatchObject({ run: 'cd dist && sha256sum -c SHA256SUMS' })
|
|
})
|
|
|
|
it('exposes the native wheel builder to the release caller with normalized versions', () => {
|
|
const workflow = loadWorkflow('.github/workflows/build-exe-for-python-sdk.yml')
|
|
const call = workflowEvent(workflow, 'workflow_call')
|
|
const plan = workflowJob(workflow, 'plan')
|
|
const build = workflowJob(workflow, 'build')
|
|
if (!isRecord(call.inputs) || !Array.isArray(plan.steps) || !Array.isArray(build.steps)) {
|
|
throw new TypeError('Python wheel builder must define workflow_call inputs and plan steps')
|
|
}
|
|
|
|
const buildSteps: unknown[] = build.steps
|
|
const manylinuxAddon = buildSteps.find(step => isRecord(step) && step.name === 'Rebuild Linux node-pty against manylinux 2.28')
|
|
const macosCheck = buildSteps.find(step => isRecord(step) && step.name === 'Check macOS deployment target')
|
|
const manylinuxSmoke = buildSteps.find(step => isRecord(step) && step.name === 'Run wheel in a manylinux 2.28 container')
|
|
expect(call.inputs).toHaveProperty('targets')
|
|
expect(call.inputs).toMatchObject({
|
|
ci: { type: 'boolean', default: false },
|
|
release: { type: 'boolean', default: false },
|
|
})
|
|
expect(workflow.concurrency).toMatchObject({
|
|
group: 'build-single-exe-${{ github.workflow }}-${{ github.ref }}',
|
|
})
|
|
expect(plan.if).toContain('inputs.ci')
|
|
expect(plan.if).toContain('inputs.release')
|
|
expect(JSON.stringify(plan.steps)).toContain('pep440_version')
|
|
expect(JSON.stringify(workflow)).toContain('macosx_14_0_arm64')
|
|
expect(manylinuxAddon).toMatchObject({ if: "runner.os == 'Linux'" })
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('manylinux_2_28_x86_64')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('manylinux_2_28_aarch64')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('$HOME/setup-pnpm:$HOME/setup-pnpm:ro')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('node-pty-glibc-versions.txt')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('le 2.28')
|
|
expect(macosCheck).toMatchObject({ if: "runner.os == 'macOS'" })
|
|
expect(JSON.stringify(macosCheck)).toContain('scripts/check-macos-deployment-target.py')
|
|
expect(JSON.stringify(macosCheck)).toContain('$EXE-spawn-helper')
|
|
expect(manylinuxSmoke).toMatchObject({ if: "runner.os == 'Linux'" })
|
|
expect(JSON.stringify(manylinuxSmoke)).toContain('-e DSH_TELEMETRY_DISABLED')
|
|
})
|
|
|
|
it('uses the shared macOS deployment-target check in GitLab', () => {
|
|
const workflow = loadWorkflow('.gitlab-ci.yml')
|
|
const runtimeWheel = workflow['.runtime-wheel']
|
|
if (!isRecord(runtimeWheel) || !Array.isArray(runtimeWheel.script)) {
|
|
throw new TypeError('GitLab CI must define the runtime wheel script')
|
|
}
|
|
const runtimeScript: unknown[] = runtimeWheel.script
|
|
const macosCheck = runtimeScript.find(
|
|
step => typeof step === 'string' && step.includes('PLATFORM" = macos-arm64'),
|
|
)
|
|
if (typeof macosCheck !== 'string') {
|
|
throw new TypeError('GitLab CI must check the macOS deployment target')
|
|
}
|
|
|
|
expect(macosCheck).toContain('scripts/check-macos-deployment-target.py')
|
|
expect(macosCheck).toContain('"$EXE" "$EXE-spawn-helper"')
|
|
})
|
|
})
|
|
|
|
describe('Issue lifecycle workflow', () => {
|
|
it('uses explicit review handoff events without rerunning when a draft becomes ready', () => {
|
|
const lifecycle = loadWorkflow('.github/workflows/issue-lifecycle.yml')
|
|
const lifecyclePullRequest = workflowEvent(lifecycle, 'pull_request')
|
|
const lifecycleReview = workflowEvent(lifecycle, 'pull_request_review')
|
|
const lifecycleJob = workflowJob(lifecycle, 'lifecycle')
|
|
const policy = loadWorkflow('.github/workflows/issue-policy.yml')
|
|
const policyPullRequest = workflowEvent(policy, 'pull_request')
|
|
|
|
expect(lifecyclePullRequest.types).not.toContain('ready_for_review')
|
|
expect(lifecyclePullRequest.types).toContain('review_requested')
|
|
expect(lifecycleReview.types).toEqual(['submitted'])
|
|
expect(lifecycleJob.if).toBe(
|
|
"${{ github.event_name != 'pull_request_review' || (github.event.action == 'submitted' && github.event.review.state == 'changes_requested') }}",
|
|
)
|
|
expect(policyPullRequest.types).toContain('ready_for_review')
|
|
})
|
|
})
|
|
|
|
describe('Git hooks', () => {
|
|
it('leaves frozen Agent Note sidecars to the archive verifier', () => {
|
|
const lefthook = loadWorkflow('lefthook.yml')
|
|
|
|
for (const hookName of ['pre-commit', 'pre-merge-commit']) {
|
|
const hook = lefthook[hookName]
|
|
if (!isRecord(hook) || !Array.isArray(hook.jobs)) {
|
|
throw new TypeError(`lefthook must define ${hookName} jobs`)
|
|
}
|
|
const pairing: unknown = hook.jobs.find(
|
|
(job: unknown) => isRecord(job) && job.name === 'translation pairing (staged records)',
|
|
)
|
|
|
|
expect(pairing).toMatchObject({ exclude: ['.agents/notes/archived/**'] })
|
|
}
|
|
})
|
|
})
|
|
|
|
function loadWorkflow(path: string): Record<string, unknown> {
|
|
const workflow: unknown = yaml.load(readFileSync(resolve(root, path), 'utf8'))
|
|
if (!isRecord(workflow)) throw new TypeError(`${path} must define a workflow`)
|
|
return workflow
|
|
}
|
|
|
|
function workflowEvent(workflow: Record<string, unknown>, event: string): Record<string, unknown> {
|
|
if (!isRecord(workflow.on) || !isRecord(workflow.on[event])) {
|
|
throw new TypeError(`workflow must define the ${event} event`)
|
|
}
|
|
return workflow.on[event]
|
|
}
|
|
|
|
function workflowJob(workflow: Record<string, unknown>, job: string): Record<string, unknown> {
|
|
if (!isRecord(workflow.jobs) || !isRecord(workflow.jobs[job])) {
|
|
throw new TypeError(`workflow must define the ${job} job`)
|
|
}
|
|
return workflow.jobs[job]
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
}
|