mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-08-29 04:26:38 +00:00
Tianyi Cui
8101a0d097
Run the GitHub Windows runtime leg under the runner’s native PowerShell instead of inheriting the POSIX Bash body. POSIX and Windows now own explicit output resolution, virtual-environment setup, environment scrubbing, and keyless/live black-box commands, while portable build commands continue to use each runner’s default shell. Put the pinned uv installation on the GitLab Windows job PATH before either the smoke or release builder invokes it. Reject a runtime executable whose basename does not match the selected platform manifest, and reject Intel macOS at platform selection instead of reporting a misleading missing artifact. Add a complete PowerShell path to the published Python tutorial and record the three-phase shutdown-time bound in the Windows runtime decision. Workflow, Python, and bilingual documentation tests pin the resulting behavior.
648 lines
34 KiB
TypeScript
648 lines
34 KiB
TypeScript
import { readFileSync } from 'node:fs'
|
|
import { resolve } from 'node:path'
|
|
import * as yaml from 'js-yaml'
|
|
import { describe, expect, it } from 'vitest'
|
|
|
|
const root = resolve(import.meta.dirname, '..')
|
|
const runnerPrivatePnpmDestination = '${{ runner.temp }}/setup-pnpm'
|
|
const nativeWindowsPnpmDestination = '${{ runner.temp }}/setup-pnpm-js'
|
|
|
|
describe('CI workflow', () => {
|
|
it('isolates every pnpm action setup destination per runner', () => {
|
|
const files = ['.github/workflows/ci.yml', '.github/workflows/ci-master.yml']
|
|
const setups: Array<{ jobName: string; step: unknown }> = []
|
|
for (const file of files) {
|
|
const workflow: unknown = yaml.load(readFileSync(resolve(root, file), 'utf8'))
|
|
if (!isRecord(workflow) || !isRecord(workflow.jobs)) throw new TypeError(`${file} must define jobs`)
|
|
for (const [jobName, job] of Object.entries(workflow.jobs)) {
|
|
if (!isRecord(job) || !Array.isArray(job.steps)) continue
|
|
for (const step of job.steps) {
|
|
if (!isRecord(step) || typeof step.uses !== 'string' || !step.uses.startsWith('pnpm/action-setup@')) continue
|
|
setups.push({ jobName, step })
|
|
}
|
|
}
|
|
}
|
|
|
|
expect(setups.length).toBeGreaterThan(0)
|
|
for (const { jobName, step } of setups) {
|
|
expect(step, `${jobName} must not share pnpm/action-setup's default destination`).toMatchObject({
|
|
with: {
|
|
dest: jobName === 'windows-native'
|
|
? nativeWindowsPnpmDestination
|
|
: runnerPrivatePnpmDestination,
|
|
},
|
|
})
|
|
if (jobName === 'windows-native') expect(step).not.toMatchObject({ with: { standalone: true } })
|
|
}
|
|
})
|
|
|
|
it('keeps a required Wine Windows job, a non-blocking native Windows job with failover, and a master-only standby', () => {
|
|
const workflow = loadWorkflow('.github/workflows/ci.yml')
|
|
const masterWorkflow = loadWorkflow('.github/workflows/ci-master.yml')
|
|
if (!isRecord(workflow.jobs)
|
|
|| !isRecord(workflow.jobs.windows)
|
|
|| !isRecord(workflow.jobs['windows-native'])
|
|
|| !isRecord(workflow.jobs['node-24'])
|
|
|| !isRecord(workflow.jobs['node-24-coverage'])
|
|
|| !isRecord(workflow.jobs['node-24-consumers'])
|
|
|| !isRecord(workflow.jobs['all-checks-passed'])
|
|
|| !isRecord(masterWorkflow.jobs)
|
|
|| !isRecord(masterWorkflow.jobs['wine-apt-cache'])
|
|
|| !isRecord(masterWorkflow.jobs['serial-windows'])) {
|
|
throw new TypeError('CI workflow must define windows, windows-native, node-24, node-24-coverage, node-24-consumers, and all-checks-passed; ci-master must define wine-apt-cache and serial-windows')
|
|
}
|
|
|
|
const windows = workflow.jobs.windows
|
|
const windowsNative = workflow.jobs['windows-native']
|
|
const wineAptCache = masterWorkflow.jobs['wine-apt-cache']
|
|
const serialWindows = masterWorkflow.jobs['serial-windows']
|
|
const node24 = workflow.jobs['node-24']
|
|
const node24Coverage = workflow.jobs['node-24-coverage']
|
|
const node24Consumers = workflow.jobs['node-24-consumers']
|
|
const aggregate = workflow.jobs['all-checks-passed']
|
|
if (!Array.isArray(windows.steps) || !Array.isArray(aggregate.needs)) {
|
|
throw new TypeError('Windows job must define steps and the aggregate must define needs')
|
|
}
|
|
const commandSteps = windows.steps.filter((step): step is Record<string, unknown> & { run: string } => (
|
|
isRecord(step) && typeof step.run === 'string'
|
|
))
|
|
|
|
// Required PR job: Wine on ubuntu-latest, runs wine-windows-gates.sh.
|
|
expect(windows['runs-on']).toBe('ubuntu-latest')
|
|
expect(windows.name).toBe('windows node 24 / wine blocking')
|
|
expect(windows.if).toBe("github.event_name == 'pull_request'")
|
|
expect(commandSteps.some(step => step.run.includes('wine-windows-gates.sh'))).toBe(true)
|
|
|
|
// windows-native: non-blocking native job with failover, runs windows-complete.
|
|
// Its pool is resolved by the Windows-specific switch.
|
|
expect(typeof windowsNative['runs-on']).toBe('string')
|
|
expect(windowsNative['runs-on']).toContain('DSH_CI_FAILOVER_WINDOWS')
|
|
expect(windowsNative['runs-on']).not.toContain('DSH_CI_FAILOVER_LINUX')
|
|
expect(windowsNative['runs-on']).toContain('self-hosted')
|
|
expect(windowsNative['runs-on']).toContain('dsh-win-ci')
|
|
expect(windowsNative['runs-on']).toContain('dsh-windows-2025-16core')
|
|
expect(windowsNative.name).toBe('windows node 24 / native complete')
|
|
expect(windowsNative.if).toBe("github.event_name == 'pull_request'")
|
|
expect(windowsNative.env).toMatchObject({
|
|
DSH_COVERAGE_TEST_TIMEOUT_MS: '30000',
|
|
})
|
|
const nativeSteps = windowsNative.steps as unknown[]
|
|
const nativeCommandSteps = nativeSteps.filter((step): step is Record<string, unknown> & { run: string } => (
|
|
isRecord(step) && typeof step.run === 'string'
|
|
))
|
|
expect(nativeCommandSteps.map(step => step.run)).toContain('pnpm run check:ci:windows-complete')
|
|
|
|
// wine-apt-cache: master-only, seeds the Wine apt cache, lives in ci-master.
|
|
expect(wineAptCache.if).toBe("github.event_name == 'push' && github.ref == 'refs/heads/master'")
|
|
expect(wineAptCache['runs-on']).toBe('ubuntu-latest')
|
|
|
|
// serial-windows: master-only standby, self-hosted, non-blocking, lives in ci-master.
|
|
expect(serialWindows.if).toBe("github.event_name == 'push' && github.ref == 'refs/heads/master'")
|
|
expect(serialWindows['runs-on']).toEqual(['self-hosted', 'dsh-win-ci', 'windows'])
|
|
expect(serialWindows.name).toBe('serial / windows (self-hosted standby)')
|
|
|
|
// Aggregate: Wine `windows` required, native `windows-native` excluded.
|
|
expect(aggregate.needs).toContain('windows')
|
|
expect(aggregate.needs).not.toContain('windows-native')
|
|
expect(aggregate.needs).not.toContain('serial-windows')
|
|
|
|
// Linux failover is a separate switch: the three required Linux workers
|
|
// and the verdict job resolve their pool through DSH_CI_FAILOVER_LINUX,
|
|
// never the Windows switch.
|
|
for (const [jobName, job] of [['node-24', node24], ['node-24-coverage', node24Coverage], ['node-24-consumers', node24Consumers]] as const) {
|
|
expect(typeof job['runs-on']).toBe('string')
|
|
expect(job['runs-on'], `${jobName} runs-on must use the Linux failover switch`).toContain('DSH_CI_FAILOVER_LINUX')
|
|
expect(job['runs-on'], `${jobName} runs-on must not use the Windows failover switch`).not.toContain('DSH_CI_FAILOVER_WINDOWS')
|
|
expect(job['runs-on']).toContain('vm-backup')
|
|
}
|
|
expect(aggregate['runs-on']).toContain('DSH_CI_FAILOVER_LINUX')
|
|
expect(aggregate['runs-on']).not.toContain('DSH_CI_FAILOVER_WINDOWS')
|
|
expect(aggregate['runs-on']).toContain('vm-backup')
|
|
})
|
|
|
|
it('gives the Wine Host TypeScript compile the repository heap budget', () => {
|
|
const wineGates = readFileSync(resolve(root, 'scripts/wine-windows-gates.sh'), 'utf8')
|
|
|
|
expect(wineGates).toContain(
|
|
'wine_node "$scratch/logs/host-tsc.log" --max-old-space-size=4096 "$tsc_js" -b tsconfig.host.json --pretty false',
|
|
)
|
|
})
|
|
|
|
it('exempts push from cancellation in ci-master, so one master merge does not cancel the running drill', () => {
|
|
const workflow = loadWorkflow('.github/workflows/ci-master.yml')
|
|
const prWorkflow = loadWorkflow('.github/workflows/ci.yml')
|
|
if (!isRecord(workflow.jobs) || !isRecord(workflow.concurrency)) {
|
|
throw new TypeError('ci-master workflow must define jobs and a workflow-level concurrency block')
|
|
}
|
|
if (!isRecord(prWorkflow.jobs)) {
|
|
throw new TypeError('ci workflow must define jobs')
|
|
}
|
|
|
|
// Cancellation applies to the whole superseded RUN, so this has to be
|
|
// decided at workflow level and gated on the event: a job-level group
|
|
// cannot exempt its job from its run being cancelled. Only push is exempt —
|
|
// a drill takes longer than the interval between master merges. The negated
|
|
// form is load-bearing: `== 'pull_request'` would also stop cancelling
|
|
// workflow_dispatch, and a re-dispatched runner benchmark holds up to 12
|
|
// larger runners for 15 minutes in this same group on master.
|
|
expect(workflow.concurrency['cancel-in-progress']).toBe("${{ github.event_name != 'push' }}")
|
|
|
|
// The PR-only ci.yml still cancels a superseded run on a new push, so a
|
|
// fresh head does not stack a second full 9-job run behind a stale one.
|
|
// Unlike ci-master it has no push carve-out: every PR event supersedes.
|
|
expect(prWorkflow.concurrency).toMatchObject({
|
|
'cancel-in-progress': true,
|
|
})
|
|
|
|
// The exact event sets are what keep master-only jobs out of the PR check
|
|
// panel: ci-master triggers only on push(master) + workflow_dispatch and
|
|
// never on pull_request; ci.yml is exactly pull_request-only. Assert the
|
|
// full sets so losing the wrong event, or gaining an extra one, fails.
|
|
if (!isRecord(workflow.on) || !isRecord(prWorkflow.on)) {
|
|
throw new TypeError('both CI workflows must define on')
|
|
}
|
|
expect(Object.keys(workflow.on).sort()).toEqual(['push', 'workflow_dispatch'])
|
|
expect(Object.keys(prWorkflow.on)).toEqual(['pull_request'])
|
|
|
|
// Neither drill may carry a job-level group: it would not exempt the job
|
|
// from run-scoped cancellation.
|
|
for (const name of ['serial-linux-selfhosted', 'serial-windows']) {
|
|
const job = workflow.jobs[name]
|
|
if (!isRecord(job)) throw new TypeError(`${name} must be defined`)
|
|
expect(job.concurrency).toBeUndefined()
|
|
// Both stay master-push-only; that is what makes the push carve-out safe.
|
|
expect(job.if).toBe("github.event_name == 'push' && github.ref == 'refs/heads/master'")
|
|
}
|
|
|
|
// What bounds the cost of exempting push: a master push may only carry the
|
|
// cache seeder and the two drills. Any job reachable on push would start
|
|
// accumulating uncancelled runs, so the set is pinned here.
|
|
const NOT_PUSH_REACHABLE = new Set([
|
|
"github.event_name == 'workflow_dispatch' && inputs.suite == 'larger-runner-benchmark'",
|
|
"github.event_name == 'workflow_dispatch' && inputs.suite == 'consolidated-runner-benchmark'",
|
|
])
|
|
const pushReachable = Object.entries(workflow.jobs)
|
|
.filter(([, job]) => {
|
|
if (!isRecord(job)) return false
|
|
if (job.if === undefined) return true // unconditional: runs on every event
|
|
if (job.if === false) return false // `if: false` parses as a boolean
|
|
if (typeof job.if !== 'string') return true // unrecognized shape: surface it
|
|
return !NOT_PUSH_REACHABLE.has(job.if.trim())
|
|
})
|
|
.map(([name]) => name)
|
|
.sort()
|
|
expect(pushReachable).toEqual(['serial-linux-selfhosted', 'serial-windows', 'wine-apt-cache'])
|
|
|
|
// Why workflow_dispatch must keep cancelling: each benchmark fans out to a
|
|
// dozen larger runners at once, in this same group on master. If it stopped
|
|
// cancelling, a re-dispatch would queue ahead of a drill instead of
|
|
// replacing the stale measurement.
|
|
for (const name of ['larger-runner-benchmark', 'consolidated-runner-benchmark']) {
|
|
const job = workflow.jobs[name]
|
|
if (!isRecord(job) || !isRecord(job.strategy)) {
|
|
throw new TypeError(`${name} must define a matrix strategy`)
|
|
}
|
|
expect(job.strategy['max-parallel']).toBe(12)
|
|
expect(job['timeout-minutes']).toBe(15)
|
|
}
|
|
})
|
|
|
|
it('keeps supported LSP source under native Windows coverage', () => {
|
|
const config = readFileSync(resolve(root, 'vitest.config.ts'), 'utf8')
|
|
|
|
expect(config).not.toContain('packages/lsp/lsp-stdio/src/connection.ts')
|
|
expect(config).not.toContain('packages/lsp/lsp-stdio/src/index.ts')
|
|
expect(config).not.toContain('packages/lsp/lsp-stdio/src/instance.ts')
|
|
})
|
|
|
|
it('requires release-shaped Python runtime validation on every published target', () => {
|
|
const workflow = loadWorkflow('.github/workflows/ci.yml')
|
|
const pythonRuntime = workflowJob(workflow, 'python-runtime')
|
|
const aggregate = workflowJob(workflow, 'all-checks-passed')
|
|
if (!Array.isArray(aggregate.needs)) {
|
|
throw new TypeError('CI aggregate must define required job dependencies')
|
|
}
|
|
|
|
expect(pythonRuntime).toMatchObject({
|
|
if: "github.event_name == 'pull_request'",
|
|
name: 'python runtime / release-shaped matrix',
|
|
uses: './.github/workflows/build-exe-for-python-sdk.yml',
|
|
with: {
|
|
targets: 'node24-linux-x64,node24-linux-arm64,node24-macos-arm64,node24-win-x64',
|
|
ci: true,
|
|
},
|
|
secrets: {
|
|
DEEPSEEK_API_KEY_EXTERNAL: '${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }}',
|
|
},
|
|
})
|
|
expect(aggregate.needs).toContain('python-runtime')
|
|
})
|
|
|
|
it('keeps every Vitest project process-isolated on native Windows', () => {
|
|
const config = readFileSync(resolve(root, 'vitest.config.ts'), 'utf8')
|
|
|
|
expect(config).not.toContain("pool: process.platform === 'win32' ? 'threads' : 'forks'")
|
|
expect(config.match(/pool: 'forks'/g)).toHaveLength(2)
|
|
})
|
|
})
|
|
|
|
describe('DeepSeek e2e workflow', () => {
|
|
it('prepares bubblewrap from the pinned payload without a package transaction', () => {
|
|
const workflow = loadWorkflow('.github/workflows/e2e.yml')
|
|
const e2e = workflowJob(workflow, 'e2e')
|
|
if (!Array.isArray(e2e.steps)) throw new TypeError('DeepSeek e2e workflow must define steps')
|
|
|
|
const steps = e2e.steps.filter(isRecord)
|
|
expect(steps.find(step => step.name === 'Prepare bubblewrap (unrestrict userns)')).toMatchObject({
|
|
run: 'bash scripts/prepare-ci-bubblewrap.sh',
|
|
})
|
|
expect(JSON.stringify(steps)).not.toContain('apt-get')
|
|
})
|
|
|
|
it('bounds profile subprocess fan-out to the tested e2e default', () => {
|
|
const workflow = loadWorkflow('.github/workflows/e2e.yml')
|
|
const e2e = workflowJob(workflow, 'e2e')
|
|
if (!Array.isArray(e2e.steps)) throw new TypeError('DeepSeek e2e workflow must define steps')
|
|
|
|
const step = e2e.steps.filter(isRecord).find(candidate => candidate.name === 'E2E tests (real DeepSeek API)')
|
|
expect(step).toMatchObject({ env: { DSH_E2E_MAX_WORKERS: 4 } })
|
|
})
|
|
})
|
|
|
|
describe('E2B e2e workflow', () => {
|
|
it('is manual-only and fails loud before running the focused live suite', () => {
|
|
const workflow = loadWorkflow('.github/workflows/e2b-e2e.yml')
|
|
expect(workflow.on).toEqual({ workflow_dispatch: null })
|
|
if (!isRecord(workflow.jobs) || !isRecord(workflow.jobs.e2b) || !Array.isArray(workflow.jobs.e2b.steps)) {
|
|
throw new TypeError('E2B e2e workflow must define the e2b job steps')
|
|
}
|
|
|
|
const steps = workflow.jobs.e2b.steps.filter(isRecord)
|
|
const preflight = steps.find(step => step.name === 'Preflight (require E2B API key)')
|
|
const e2b = steps.find(step => step.name === 'E2B tests (live sandbox)')
|
|
|
|
expect(preflight).toMatchObject({
|
|
env: { E2B_API_KEY: '${{ secrets.E2B_API_KEY_EXTERNAL }}' },
|
|
})
|
|
expect(preflight?.run).toContain('E2B_API_KEY_EXTERNAL repository secret')
|
|
expect(e2b).toMatchObject({
|
|
env: {
|
|
E2B_API_KEY: '${{ secrets.E2B_API_KEY_EXTERNAL }}',
|
|
DSH_E2E_MAX_WORKERS: '1',
|
|
DSH_EXAMPLE_MODE: 'lib',
|
|
},
|
|
})
|
|
expect(e2b?.run).toContain('packages/e2b/e2b/tests/composition.e2e.ts')
|
|
})
|
|
})
|
|
|
|
describe('Python release workflows', () => {
|
|
it('keeps complete wheel validation separate from protected public publication', () => {
|
|
const workflow = loadWorkflow('.github/workflows/python-release.yml')
|
|
const dispatch = workflowEvent(workflow, 'workflow_dispatch')
|
|
const build = workflowJob(workflow, 'build')
|
|
const pythonCompat = workflowJob(workflow, 'python-compat')
|
|
const validate = workflowJob(workflow, 'validate')
|
|
const publishRuntime = workflowJob(workflow, 'publish-runtime')
|
|
const publishSdk = workflowJob(workflow, 'publish-sdk')
|
|
if (!isRecord(dispatch.inputs)
|
|
|| !isRecord(dispatch.inputs.publish)
|
|
|| !Array.isArray(pythonCompat.steps)
|
|
|| !Array.isArray(validate.steps)
|
|
|| !Array.isArray(publishRuntime.steps)
|
|
|| !Array.isArray(publishSdk.steps)) {
|
|
throw new TypeError('Python release workflow must define publish input and release steps')
|
|
}
|
|
|
|
expect(dispatch.inputs.publish).toMatchObject({ type: 'boolean', default: false })
|
|
if (!isRecord(workflow.on)) throw new TypeError('python-release workflow must define on')
|
|
expect(Object.keys(workflow.on)).toEqual(['workflow_dispatch'])
|
|
expect(build).toMatchObject({
|
|
uses: './.github/workflows/build-exe-for-python-sdk.yml',
|
|
with: {
|
|
targets: 'node24-linux-x64,node24-linux-arm64,node24-macos-arm64,node24-win-x64',
|
|
release: true,
|
|
},
|
|
})
|
|
expect(pythonCompat.strategy).toMatchObject({ matrix: { python: ['3.10', '3.14'] } })
|
|
const pythonCompatSteps = JSON.stringify(pythonCompat.steps)
|
|
expect(pythonCompatSteps).toContain('dist/deepseek_harness_sdk-$VERSION-py3-none-any.whl')
|
|
expect(pythonCompatSteps).toContain('dist/deepseek_harness_runtime_bin-$VERSION-py3-none-manylinux_2_28_x86_64.whl')
|
|
expect(pythonCompatSteps).not.toContain('--find-links')
|
|
const validateSteps = JSON.stringify(validate.steps)
|
|
const authorize = validate.steps.filter(isRecord).find(step => step.name === 'Authorize publication request')
|
|
if (!isRecord(authorize) || typeof authorize.run !== 'string') {
|
|
throw new TypeError('Python release validation must authorize publication requests')
|
|
}
|
|
expect(validateSteps).toContain('PUBLIC_PYPI_RELEASE_ENABLED')
|
|
expect(authorize).toMatchObject({
|
|
env: {
|
|
PYPI_PUBLISHER_REPOSITORY: '${{ vars.PYPI_PUBLISHER_REPOSITORY }}',
|
|
REPOSITORY: '${{ github.repository }}',
|
|
},
|
|
})
|
|
expect(authorize.run).toContain('[ "$REPOSITORY" = "$PYPI_PUBLISHER_REPOSITORY" ]')
|
|
expect(validateSteps).toContain('100000000')
|
|
expect(publishRuntime).toMatchObject({
|
|
if: "github.event_name == 'workflow_dispatch' && inputs.publish",
|
|
needs: 'validate',
|
|
environment: 'pypi-runtime',
|
|
permissions: { contents: 'read', 'id-token': 'write' },
|
|
})
|
|
expect(publishSdk).toMatchObject({
|
|
if: "github.event_name == 'workflow_dispatch' && inputs.publish",
|
|
needs: ['validate', 'publish-runtime'],
|
|
environment: 'pypi',
|
|
permissions: { contents: 'read', 'id-token': 'write' },
|
|
})
|
|
const runtimeSteps = publishRuntime.steps.filter(isRecord)
|
|
const sdkSteps = publishSdk.steps.filter(isRecord)
|
|
const runtimePublish = runtimeSteps.find(step => step.name === 'Publish runtime wheels')
|
|
const sdkPublish = sdkSteps.find(step => step.name === 'Publish SDK wheel')
|
|
const runtimeHashes = runtimeSteps.find(step => step.name === 'Verify release artifact hashes')
|
|
const sdkHashes = sdkSteps.find(step => step.name === 'Verify release artifact hashes')
|
|
expect([...runtimeSteps, ...sdkSteps].some(
|
|
step => typeof step.uses === 'string' && step.uses.startsWith('actions/checkout@'),
|
|
)).toBe(false)
|
|
expect([...runtimeSteps, ...sdkSteps].filter(
|
|
step => step.uses === 'pypa/gh-action-pypi-publish@release/v1',
|
|
)).toHaveLength(2)
|
|
expect(runtimePublish).toMatchObject({
|
|
with: { 'packages-dir': 'dist/runtime/', attestations: false },
|
|
})
|
|
expect(sdkPublish).toMatchObject({
|
|
with: { 'packages-dir': 'dist/sdk/', attestations: false },
|
|
})
|
|
expect(runtimeHashes).toMatchObject({ run: 'cd dist && sha256sum -c SHA256SUMS' })
|
|
expect(sdkHashes).toMatchObject({ run: 'cd dist && sha256sum -c SHA256SUMS' })
|
|
})
|
|
|
|
it('exposes the native wheel builder to the release caller with normalized versions', () => {
|
|
const workflow = loadWorkflow('.github/workflows/build-exe-for-python-sdk.yml')
|
|
const call = workflowEvent(workflow, 'workflow_call')
|
|
const plan = workflowJob(workflow, 'plan')
|
|
const build = workflowJob(workflow, 'build')
|
|
if (!isRecord(call.inputs) || !isRecord(call.secrets) || !Array.isArray(plan.steps) || !Array.isArray(build.steps)) {
|
|
throw new TypeError('Python wheel builder must define workflow_call inputs and plan steps')
|
|
}
|
|
|
|
const buildSteps: unknown[] = build.steps
|
|
const manylinuxAddon = buildSteps.find(step => isRecord(step) && step.name === 'Rebuild Linux node-pty against manylinux 2.28')
|
|
const macosCheck = buildSteps.find(step => isRecord(step) && step.name === 'Check macOS deployment target')
|
|
const manylinuxSmoke = buildSteps.find(step => isRecord(step) && step.name === 'Run wheel in a manylinux 2.28 container')
|
|
const cleanVenvPosix = buildSteps.find(step => isRecord(step) && step.name === 'Install local SDK and runtime wheels into a clean venv (POSIX)')
|
|
const cleanVenvWindows = buildSteps.find(step => isRecord(step) && step.name === 'Install local SDK and runtime wheels into a clean venv (Windows)')
|
|
const installedKeylessPosix = buildSteps.find(step => isRecord(step) && step.name === 'Run installed-wheel keyless black-box tests (POSIX)')
|
|
const installedKeylessWindows = buildSteps.find(step => isRecord(step) && step.name === 'Run installed-wheel keyless black-box tests (Windows)')
|
|
const realApiPreflightPosix = buildSteps.find(step => isRecord(step) && step.name === 'Preflight installed-wheel real API test (POSIX)')
|
|
const realApiPreflightWindows = buildSteps.find(step => isRecord(step) && step.name === 'Preflight installed-wheel real API test (Windows)')
|
|
const installedRealApiPosix = buildSteps.find(step => isRecord(step) && step.name === 'Run installed-wheel real API black-box test (POSIX)')
|
|
const installedRealApiWindows = buildSteps.find(step => isRecord(step) && step.name === 'Run installed-wheel real API black-box test (Windows)')
|
|
if (!isRecord(cleanVenvPosix) || !isRecord(cleanVenvWindows)
|
|
|| !isRecord(installedKeylessPosix) || !isRecord(installedKeylessWindows)
|
|
|| !isRecord(realApiPreflightPosix) || !isRecord(realApiPreflightWindows)
|
|
|| !isRecord(installedRealApiPosix) || !isRecord(installedRealApiWindows)) {
|
|
throw new TypeError('Python wheel builder must define native POSIX and Windows installed-wheel steps')
|
|
}
|
|
expect(call.inputs).toHaveProperty('targets')
|
|
expect(call.inputs).toMatchObject({
|
|
ci: { type: 'boolean', default: false },
|
|
release: { type: 'boolean', default: false },
|
|
})
|
|
expect(call.secrets).toMatchObject({
|
|
DEEPSEEK_API_KEY_EXTERNAL: { required: false },
|
|
})
|
|
expect(workflow.concurrency).toMatchObject({
|
|
group: 'build-single-exe-${{ github.workflow }}-${{ github.ref }}',
|
|
})
|
|
expect(build.defaults).toBeUndefined()
|
|
expect(plan.if).toContain('inputs.ci')
|
|
expect(plan.if).toContain('inputs.release')
|
|
expect(JSON.stringify(plan.steps)).toContain('pep440_version')
|
|
const workflowJson = JSON.stringify(workflow)
|
|
expect(workflowJson).toContain('macosx_14_0_arm64')
|
|
expect(workflowJson).toContain('win_amd64')
|
|
expect(workflowJson).toContain('node24-win-x64')
|
|
expect(workflowJson).toContain('windows-2025')
|
|
expect(workflowJson).toContain('dist-python/$SDK_WHEEL')
|
|
expect(workflowJson).toContain('dist-python/$RUNTIME_WHEEL')
|
|
expect(workflowJson).toContain('/work/dist-python/$SDK_WHEEL')
|
|
expect(workflowJson).toContain('/work/dist-python/$RUNTIME_WHEEL')
|
|
expect(workflowJson).not.toContain('--find-links dist-python')
|
|
expect(workflowJson).not.toContain('--find-links /work/dist-python')
|
|
expect(workflowJson).not.toContain('cygpath')
|
|
expect(manylinuxAddon).toMatchObject({ if: "runner.os == 'Linux'" })
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('manylinux_2_28_x86_64')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('manylinux_2_28_aarch64')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('npm_config_build_from_source=true pnpm run install')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('pnpm_setup_root')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('$pnpm_setup_root:$pnpm_setup_root:ro')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('node-pty-glibc-versions.txt')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('le 2.28')
|
|
expect(macosCheck).toMatchObject({ if: "runner.os == 'macOS'" })
|
|
expect(JSON.stringify(macosCheck)).toContain('scripts/check-macos-deployment-target.py')
|
|
expect(JSON.stringify(macosCheck)).toContain('$EXE-spawn-helper')
|
|
expect(JSON.stringify(installedKeylessPosix)).toContain('--scenario all')
|
|
expect(JSON.stringify(installedKeylessPosix)).toContain('env -u PYTHONPATH')
|
|
expect(JSON.stringify(installedKeylessWindows)).toContain('--scenario all --installed-wheel')
|
|
expect(installedKeylessWindows).toMatchObject({ if: "runner.os == 'Windows'", shell: 'pwsh' })
|
|
expect(cleanVenvWindows).toMatchObject({ if: "runner.os == 'Windows'", shell: 'pwsh' })
|
|
expect(JSON.stringify(cleanVenvWindows)).toContain('Scripts\\\\python.exe')
|
|
expect(realApiPreflightPosix).toMatchObject({
|
|
env: { DEEPSEEK_API_KEY: '${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }}' },
|
|
})
|
|
expect(String(realApiPreflightPosix.if)).toContain('inputs.ci')
|
|
expect(String(realApiPreflightPosix.if)).toContain('head.repo.fork')
|
|
expect(String(realApiPreflightPosix.if)).toContain('dependabot[bot]')
|
|
expect(realApiPreflightWindows).toMatchObject({ shell: 'pwsh' })
|
|
expect(installedRealApiPosix).toMatchObject({
|
|
env: {
|
|
DEEPSEEK_API_KEY: '${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }}',
|
|
DEEPSEEK_BASE_URL: 'https://api.deepseek.com',
|
|
},
|
|
})
|
|
expect(JSON.stringify(installedRealApiPosix)).toContain('--scenario sdk-live')
|
|
expect(JSON.stringify(installedRealApiPosix)).toContain('-u DSH_RUNTIME_MODE')
|
|
expect(installedRealApiWindows).toMatchObject({ shell: 'pwsh' })
|
|
expect(JSON.stringify(installedRealApiWindows)).toContain('--scenario sdk-live --installed-wheel')
|
|
expect(manylinuxSmoke).toMatchObject({ if: "runner.os == 'Linux'" })
|
|
expect(JSON.stringify(manylinuxSmoke)).toContain('-e DSH_TELEMETRY_DISABLED')
|
|
})
|
|
|
|
it('uses the shared macOS deployment-target check in GitLab', () => {
|
|
const workflow = loadWorkflow('.gitlab-ci.yml')
|
|
const runtimeWheel = workflow['.runtime-wheel']
|
|
if (!isRecord(runtimeWheel) || !Array.isArray(runtimeWheel.script)) {
|
|
throw new TypeError('GitLab CI must define the runtime wheel script')
|
|
}
|
|
const runtimeScript: unknown[] = runtimeWheel.script
|
|
const macosCheck = runtimeScript.find(
|
|
step => typeof step === 'string' && step.includes('PLATFORM" = macos-arm64'),
|
|
)
|
|
if (typeof macosCheck !== 'string') {
|
|
throw new TypeError('GitLab CI must check the macOS deployment target')
|
|
}
|
|
|
|
expect(macosCheck).toContain('scripts/check-macos-deployment-target.py')
|
|
expect(macosCheck).toContain('"$EXE" "$EXE-spawn-helper"')
|
|
})
|
|
|
|
it('builds and black-box tests the Windows x64 wheel in GitLab', () => {
|
|
const workflow = loadWorkflow('.gitlab-ci.yml')
|
|
const windows = workflow['runtime-windows-x64']
|
|
const publish = workflow['publish-python']
|
|
if (!isRecord(windows) || !Array.isArray(windows.before_script) || !Array.isArray(windows.script)
|
|
|| !isRecord(publish) || !Array.isArray(publish.needs)) {
|
|
throw new TypeError('GitLab CI must define the Windows runtime and aggregate publication jobs')
|
|
}
|
|
|
|
expect(windows.tags).toEqual(['windows-x64'])
|
|
expect(windows.variables).toMatchObject({ PKG_TARGET: 'node24-win-x64', PLATFORM: 'win-x64' })
|
|
expect(JSON.stringify(windows.before_script)).toContain('.ci-python\\\\Scripts')
|
|
expect(JSON.stringify(windows.before_script)).toContain('[IO.Path]::PathSeparator')
|
|
expect(JSON.stringify(windows.script)).toContain('win_amd64.whl')
|
|
expect(JSON.stringify(windows.script)).toContain('--scenario all --installed-wheel')
|
|
expect(publish.needs).toContainEqual({ job: 'runtime-windows-x64', artifacts: true })
|
|
})
|
|
})
|
|
|
|
describe('Issue lifecycle workflow', () => {
|
|
it('runs the lifecycle job on every PR/review event but gates token and board steps', () => {
|
|
const lifecycle = loadWorkflow('.github/workflows/issue-lifecycle.yml')
|
|
const policy = loadWorkflow('.github/workflows/issue-policy.yml')
|
|
const lifecycleJob = workflowJob(lifecycle, 'lifecycle')
|
|
if (!Array.isArray(lifecycleJob.steps)) throw new TypeError('Issue lifecycle job must define steps')
|
|
|
|
// The job has no job-level `if`, so it is listed on every pull_request /
|
|
// pull_request_review event and reports success instead of a gray skip. The
|
|
// write-capable steps are gated at step level so approved/commented reviews
|
|
// never mint a Project/Issue App token nor touch the board.
|
|
expect(lifecycle.on).toHaveProperty('pull_request')
|
|
expect(lifecycle.on).toHaveProperty('pull_request_review')
|
|
expect(lifecycleJob.if).toBeUndefined()
|
|
// Keep the subscription-type gates: issue-lifecycle does not re-subscribe
|
|
// ready_for_review (issue-policy owns that) and only reacts to submitted
|
|
// review events.
|
|
const lifecyclePullRequest = workflowEvent(lifecycle, 'pull_request')
|
|
const lifecycleReview = workflowEvent(lifecycle, 'pull_request_review')
|
|
expect(lifecyclePullRequest.types).not.toContain('ready_for_review')
|
|
expect(lifecyclePullRequest.types).toContain('review_requested')
|
|
expect(lifecycleReview.types).toEqual(['submitted'])
|
|
const gated = "${{ github.event_name != 'pull_request_review' || github.event.review.state == 'changes_requested' }}"
|
|
const steps = lifecycleJob.steps.filter(isRecord)
|
|
const tokenStep = steps.find(s => s.name === 'Create project token')
|
|
const handleStep = steps.find(s => s.name === 'Handle repository event')
|
|
expect(tokenStep).toMatchObject({ if: gated })
|
|
expect(handleStep).toMatchObject({ if: gated })
|
|
|
|
// issue-policy owns PR validation; it is read-only and a real gate.
|
|
const policyPullRequest = workflowEvent(policy, 'pull_request')
|
|
expect(policyPullRequest.types).toContain('ready_for_review')
|
|
})
|
|
})
|
|
|
|
describe('npm release workflows', () => {
|
|
it('keeps publication dispatch-only and pack in the PR workflow', () => {
|
|
// pack stays in the PR/master release workflows so a PR proves the set packs.
|
|
for (const file of ['release.yml', 'release-vendor.yml']) {
|
|
const workflow = loadWorkflow(`.github/workflows/${file}`)
|
|
if (!isRecord(workflow.jobs)) throw new TypeError(`${file} must define jobs`)
|
|
expect(Object.keys(workflow.jobs).sort()).toEqual(['pack'])
|
|
}
|
|
|
|
// publication is workflow_dispatch-only (never a PR check) and keeps the
|
|
// npm-publish environment plus the shared dist-tag group.
|
|
for (const file of ['release-publish.yml', 'release-vendor-publish.yml']) {
|
|
const workflow = loadWorkflow(`.github/workflows/${file}`)
|
|
if (!isRecord(workflow.on) || !isRecord(workflow.jobs)) throw new TypeError(`${file} must define on and jobs`)
|
|
expect(Object.keys(workflow.on)).toEqual(['workflow_dispatch'])
|
|
const publish = workflow.jobs.publish
|
|
if (!isRecord(publish)) throw new TypeError(`${file} must define a publish job`)
|
|
expect(publish.environment).toBe('npm-publish')
|
|
expect(publish.concurrency).toMatchObject({ group: 'Release-publish' })
|
|
}
|
|
})
|
|
})
|
|
|
|
describe('Documentation site publication', () => {
|
|
it('keeps Pages deployment dispatch-only from a dsh-v* tag', () => {
|
|
const workflow = loadWorkflow('.github/workflows/docs-pages.yml')
|
|
const build = workflowJob(workflow, 'build')
|
|
const deploy = workflowJob(workflow, 'deploy')
|
|
if (!isRecord(workflow.on) || !isRecord(workflow.env) || !Array.isArray(build.steps)) {
|
|
throw new TypeError('Documentation deployment must define on, env, and build steps')
|
|
}
|
|
|
|
// The site presents a released snapshot: a merge must never publish it, and
|
|
// publication must never appear as a PR check.
|
|
expect(Object.keys(workflow.on)).toEqual(['workflow_dispatch'])
|
|
|
|
// RELEASE_PUBLISH makes release:verify reject every ref that is not a dsh-v*
|
|
// tag naming this tree's version, so the site and the npm sequence share one
|
|
// definition of a released version.
|
|
const steps = build.steps.filter(isRecord)
|
|
const verify = steps.find(step => step.name === 'Verify release version')
|
|
const checkout = steps.find(
|
|
step => typeof step.uses === 'string' && step.uses.startsWith('actions/checkout@'),
|
|
)
|
|
expect(verify).toMatchObject({
|
|
env: { RELEASE_PUBLISH: 'true' },
|
|
run: 'pnpm run release:verify --family dsh',
|
|
})
|
|
// Complete history: the release scripts read tags.
|
|
expect(checkout).toMatchObject({ with: { 'fetch-depth': 0 } })
|
|
|
|
// Projected source links stay on the public repository's master. That
|
|
// repository advances only to each release commit, so its master never
|
|
// carries unreleased work, while it retains only the most recent tags:
|
|
// following the dispatched tag would leave every source link on a deploy
|
|
// from an older tag unresolvable.
|
|
expect(workflow.env.DOCS_REPOSITORY_REF).toBe('master')
|
|
|
|
// The environment owns the deployment tag policy and the required reviewers.
|
|
expect(deploy.environment).toMatchObject({ name: 'github-pages' })
|
|
})
|
|
})
|
|
|
|
describe('Git hooks', () => {
|
|
it('leaves frozen Agent Note sidecars to the archive verifier', () => {
|
|
const lefthook = loadWorkflow('lefthook.yml')
|
|
|
|
for (const hookName of ['pre-commit', 'pre-merge-commit']) {
|
|
const hook = lefthook[hookName]
|
|
if (!isRecord(hook) || !Array.isArray(hook.jobs)) {
|
|
throw new TypeError(`lefthook must define ${hookName} jobs`)
|
|
}
|
|
const pairing: unknown = hook.jobs.find(
|
|
(job: unknown) => isRecord(job) && job.name === 'translation pairing (staged records)',
|
|
)
|
|
|
|
expect(pairing).toMatchObject({ exclude: ['.agents/notes/archived/**'] })
|
|
}
|
|
})
|
|
})
|
|
|
|
function loadWorkflow(path: string): Record<string, unknown> {
|
|
const workflow: unknown = yaml.load(readFileSync(resolve(root, path), 'utf8'))
|
|
if (!isRecord(workflow)) throw new TypeError(`${path} must define a workflow`)
|
|
return workflow
|
|
}
|
|
|
|
function workflowEvent(workflow: Record<string, unknown>, event: string): Record<string, unknown> {
|
|
if (!isRecord(workflow.on) || !isRecord(workflow.on[event])) {
|
|
throw new TypeError(`workflow must define the ${event} event`)
|
|
}
|
|
return workflow.on[event]
|
|
}
|
|
|
|
function workflowJob(workflow: Record<string, unknown>, job: string): Record<string, unknown> {
|
|
if (!isRecord(workflow.jobs) || !isRecord(workflow.jobs[job])) {
|
|
throw new TypeError(`workflow must define the ${job} job`)
|
|
}
|
|
return workflow.jobs[job]
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
}
|