mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-09-09 04:02:35 +00:00
Tianyi Cui
2395f112d8
Add a benchmark lane (`vitest.bench.config.ts`, `pnpm run test:bench`, gate mode `ci-bench`) and a required `node 24 / benchmarks` CI job that runs it alone. Benchmarks synthesize their input in-process from fixed parameters and fail on documented budgets: - `open-generation.bench.ts`: a 200-turn released-v0 log with 500 text and 125 reasoning deltas per reply (127,400 events, ~2.8 MB) encoded through the frozen v0 codec; the migrating first `open()` must finish within 2,000 ms in a child process capped at 128 MB of old space, and a fresh process must open the published current generation within 500 ms. - `conversation-fold.bench.client.ts`: 200 replies whose compact streams hold 2,000 text + 500 reasoning deltas each, folded through every Chat Definition by the real assembler; the fold must finish within 150 ms and stay within 3x the fold of the same window with 100 deltas per reply. On this commit both gates fail: the migration exhausts the 128 MB heap (4.8 s and 696 MB peak RSS without the cap; the pre-stack decode of the same bytes took 34 ms and 168 MB) and the fold scales 11x with the delta count. The stacked fixes bring both paths to O(records).
866 lines
47 KiB
TypeScript
866 lines
47 KiB
TypeScript
import { readFileSync } from 'node:fs'
|
|
import { resolve } from 'node:path'
|
|
import * as yaml from 'js-yaml'
|
|
import { describe, expect, it } from 'vitest'
|
|
|
|
const root = resolve(import.meta.dirname, '..')
|
|
const runnerPrivatePnpmDestination = /^\$\{\{ runner\.temp \}\}\/setup-pnpm-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}$/
|
|
const nativeWindowsPnpmDestination = '${{ runner.temp }}/setup-pnpm-js-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}'
|
|
|
|
describe('CI workflow', () => {
|
|
it('isolates every pnpm action setup destination per runner', () => {
|
|
const files = ['.github/workflows/ci.yml', '.github/workflows/ci-master.yml']
|
|
const setups: Array<{ jobName: string; step: unknown }> = []
|
|
for (const file of files) {
|
|
const workflow: unknown = yaml.load(readFileSync(resolve(root, file), 'utf8'))
|
|
if (!isRecord(workflow) || !isRecord(workflow.jobs)) throw new TypeError(`${file} must define jobs`)
|
|
for (const [jobName, job] of Object.entries(workflow.jobs)) {
|
|
if (!isRecord(job) || !Array.isArray(job.steps)) continue
|
|
for (const step of job.steps) {
|
|
if (!isRecord(step) || typeof step.uses !== 'string' || !step.uses.startsWith('pnpm/action-setup@')) continue
|
|
setups.push({ jobName, step })
|
|
}
|
|
}
|
|
}
|
|
|
|
expect(setups.length).toBeGreaterThan(0)
|
|
for (const { jobName, step } of setups) {
|
|
const stepDest = (step as { with?: { dest?: unknown } }).with?.dest
|
|
if (jobName.startsWith('windows-')) {
|
|
expect(stepDest, `${jobName} must use the native Windows pnpm destination`).toBe(nativeWindowsPnpmDestination)
|
|
expect(step).not.toMatchObject({ with: { standalone: true } })
|
|
} else {
|
|
expect(typeof stepDest, `${jobName} must use a runner-and-run-private pnpm destination`).toBe('string')
|
|
expect(stepDest as string).toMatch(runnerPrivatePnpmDestination)
|
|
}
|
|
}
|
|
})
|
|
|
|
it('isolates the python SDK exe pnpm setup destination per job', () => {
|
|
const workflow: unknown = yaml.load(readFileSync(resolve(root, '.github/workflows/build-exe-for-python-sdk.yml'), 'utf8'))
|
|
if (!isRecord(workflow) || !isRecord(workflow.jobs)) throw new TypeError('build-exe-for-python-sdk.yml must define jobs')
|
|
const setups: Array<{ step: unknown }> = []
|
|
for (const job of Object.values(workflow.jobs)) {
|
|
if (!isRecord(job) || !Array.isArray(job.steps)) continue
|
|
for (const step of job.steps) {
|
|
if (!isRecord(step) || typeof step.uses !== 'string' || !step.uses.startsWith('pnpm/action-setup@')) continue
|
|
setups.push({ step })
|
|
}
|
|
}
|
|
expect(setups.length).toBeGreaterThan(0)
|
|
for (const { step } of setups) {
|
|
expect(step).toMatchObject({
|
|
with: { dest: nativeWindowsPnpmDestination },
|
|
})
|
|
}
|
|
})
|
|
|
|
it('keeps required Wine and split native Windows jobs with failover, plus a master-only standby', () => {
|
|
const workflow = loadWorkflow('.github/workflows/ci.yml')
|
|
const masterWorkflow = loadWorkflow('.github/workflows/ci-master.yml')
|
|
if (!isRecord(workflow.jobs)
|
|
|| !isRecord(workflow.jobs.windows)
|
|
|| !isRecord(workflow.jobs['windows-build'])
|
|
|| !isRecord(workflow.jobs['windows-coverage'])
|
|
|| !isRecord(workflow.jobs['windows-native-tests'])
|
|
|| !isRecord(workflow.jobs['windows-observational'])
|
|
|| !isRecord(workflow.jobs['node-24'])
|
|
|| !isRecord(workflow.jobs['node-24-coverage'])
|
|
|| !isRecord(workflow.jobs['node-24-bench'])
|
|
|| !isRecord(workflow.jobs['node-24-consumers'])
|
|
|| !isRecord(workflow.jobs['node-compat'])
|
|
|| !isRecord(workflow.jobs['all-checks-passed'])
|
|
|| !isRecord(masterWorkflow.jobs)
|
|
|| !isRecord(masterWorkflow.jobs['wine-apt-cache'])
|
|
|| !isRecord(masterWorkflow.jobs['serial-windows'])) {
|
|
throw new TypeError('CI workflow must define windows, windows-build, windows-coverage, windows-native-tests, windows-observational, node-24, node-24-coverage, node-24-bench, node-24-consumers, node-compat, and all-checks-passed; ci-master must define wine-apt-cache and serial-windows')
|
|
}
|
|
|
|
const windows = workflow.jobs.windows
|
|
const windowsBuild = workflow.jobs['windows-build']
|
|
const windowsCoverage = workflow.jobs['windows-coverage']
|
|
const windowsNativeTests = workflow.jobs['windows-native-tests']
|
|
const windowsObservational = workflow.jobs['windows-observational']
|
|
const wineAptCache = masterWorkflow.jobs['wine-apt-cache']
|
|
const serialWindows = masterWorkflow.jobs['serial-windows']
|
|
const node24 = workflow.jobs['node-24']
|
|
const node24Coverage = workflow.jobs['node-24-coverage']
|
|
const node24Bench = workflow.jobs['node-24-bench']
|
|
const node24Consumers = workflow.jobs['node-24-consumers']
|
|
const nodeCompat = workflow.jobs['node-compat']
|
|
const aggregate = workflow.jobs['all-checks-passed']
|
|
if (!Array.isArray(windows.steps) || !Array.isArray(aggregate.needs)) {
|
|
throw new TypeError('Windows job must define steps and the aggregate must define needs')
|
|
}
|
|
const commandSteps = windows.steps.filter((step): step is Record<string, unknown> & { run: string } => (
|
|
isRecord(step) && typeof step.run === 'string'
|
|
))
|
|
|
|
// Required PR job: Wine on ubuntu-latest, runs wine-windows-gates.sh.
|
|
expect(windows['runs-on']).toBe('ubuntu-latest')
|
|
expect(windows.name).toBe('windows node 24 / wine blocking')
|
|
expect(windows.if).toBe("github.event_name == 'pull_request'")
|
|
expect(commandSteps.some(step => step.run.includes('wine-windows-gates.sh'))).toBe(true)
|
|
|
|
// The split native jobs all resolve their pool through the Windows switch.
|
|
for (const [jobName, job] of [['windows-build', windowsBuild], ['windows-coverage', windowsCoverage], ['windows-native-tests', windowsNativeTests], ['windows-observational', windowsObservational]] as const) {
|
|
expect(typeof job['runs-on']).toBe('string')
|
|
expect(job['runs-on'], `${jobName} runs-on must use the Windows failover switch`).toContain('DSH_CI_FAILOVER_WINDOWS')
|
|
expect(job['runs-on'], `${jobName} runs-on must not use the Linux failover switch`).not.toContain('DSH_CI_FAILOVER_LINUX')
|
|
expect(job['runs-on']).toContain('self-hosted')
|
|
expect(job['runs-on']).toContain('dsh-win-ci')
|
|
expect(job['runs-on']).toContain('dsh-windows-2025-16core')
|
|
expect(job.if).toBe("github.event_name == 'pull_request'")
|
|
}
|
|
|
|
// windows-build runs the blocking build/site pair.
|
|
expect(windowsBuild.name).toBe('windows node 24 / build')
|
|
const buildSteps = windowsBuild.steps as unknown[]
|
|
const buildCommands = buildSteps.filter((step): step is Record<string, unknown> & { run: string } => (
|
|
isRecord(step) && typeof step.run === 'string'
|
|
))
|
|
expect(buildCommands.map(step => step.run)).toContain('pnpm run check:ci:windows-blocking')
|
|
|
|
// The four native Windows installs branch on the workspace filesystem:
|
|
// clone (ReFS block clone) only on ReFS, plain install elsewhere. This
|
|
// keeps the TS6231 store-path leak (see the Windows ReFS store note) out
|
|
// of the self-hosted pool without forcing clone onto hosted NTFS, which
|
|
// rejects copy-on-write. The branch must stay, or a hosted fallback would
|
|
// fail installs with ERR_PNPM_LINKING_FAILED.
|
|
for (const [jobName, job] of [['windows-build', windowsBuild], ['windows-coverage', windowsCoverage], ['windows-native-tests', windowsNativeTests], ['windows-observational', windowsObservational]] as const) {
|
|
const steps = job.steps as unknown[]
|
|
const install = steps.find((step): step is Record<string, unknown> & { run: string } => (
|
|
isRecord(step) && step.name === 'Install (immutable)' && typeof step.run === 'string'
|
|
))
|
|
expect(install, `${jobName} must define the filesystem-branched install`).toBeDefined()
|
|
expect(install!.run).toContain("$fs -eq 'ReFS'")
|
|
expect(install!.run).toContain('--package-import-method=clone')
|
|
expect(install!.run).toContain('corepack pnpm install')
|
|
// The else branch must keep the plain hosted install as a distinct line
|
|
// (not the corepack clone line, which contains the same substring);
|
|
// dropping it or making both branches clone would force clone onto
|
|
// NTFS, which rejects copy-on-write (ERR_PNPM_LINKING_FAILED). The
|
|
// YAML folded block keeps the first statement on line 1 and folds the
|
|
// rest with leading two-space indents.
|
|
const installLines = install!.run.split('\n').map(line => line.trim())
|
|
expect(installLines).toContain('} else {')
|
|
expect(installLines.some(line => line === 'pnpm install --frozen-lockfile'), `${jobName} else branch must keep the plain hosted install`).toBe(true)
|
|
// The ReFS branch must not use the interpolated empty-flag form, which
|
|
// passes a stray "" positional argument to pnpm.
|
|
expect(install!.run).not.toContain('$cloneFlag')
|
|
}
|
|
|
|
// windows-coverage uses the lower 4-partition profile.
|
|
expect(windowsCoverage.name).toBe('windows node 24 / coverage')
|
|
expect(windowsCoverage.env).toMatchObject({ DSH_COVERAGE_PARTITIONS: '4' })
|
|
const coverageSteps = windowsCoverage.steps as unknown[]
|
|
const coverageCommands = coverageSteps.filter((step): step is Record<string, unknown> & { run: string } => (
|
|
isRecord(step) && typeof step.run === 'string'
|
|
))
|
|
expect(coverageCommands.map(step => step.run)).toContain('pnpm run check:ci:coverage')
|
|
// Windows coverage runs zero-build like the Linux lane: workspace imports
|
|
// resolve to src through the tsconfig paths map, and the lib-consuming
|
|
// suites (webworker-packer image-loadable, webworker-runtime
|
|
// transform-corpus, client ui-trajectory client-bundle) self-skip on
|
|
// unbuilt checkouts. The regex catches a regression spelled as
|
|
// 'corepack pnpm run build' or folded into a multi-line run block, which
|
|
// an exact string match would miss.
|
|
expect(coverageCommands.every(step => !/\bpnpm\s+run\s+build(?:\s|$)/.test(step.run))).toBe(true)
|
|
|
|
// windows-native-tests runs the Windows-specific specs.
|
|
expect(windowsNativeTests.name).toBe('windows node 24 / native tests')
|
|
const nativeTestSteps = windowsNativeTests.steps as unknown[]
|
|
const nativeTestCommands = nativeTestSteps.filter((step): step is Record<string, unknown> & { run: string } => (
|
|
isRecord(step) && typeof step.run === 'string'
|
|
))
|
|
const nativeTestCommand = nativeTestCommands.map(step => step.run).join('\n')
|
|
expect(nativeTestCommand).toContain('--no-file-parallelism')
|
|
expect(nativeTestCommand).toContain('--testTimeout 90000')
|
|
expect(nativeTestCommand).toContain('tool-pwsh/tests/loader.spec.ts')
|
|
expect(nativeTestCommand).toContain('workflow-worker-thread.spec.ts')
|
|
|
|
// windows-observational is non-blocking.
|
|
expect(windowsObservational.name).toBe('windows node 24 / observational')
|
|
expect(windowsObservational['continue-on-error']).toBe(true)
|
|
|
|
// wine-apt-cache: master-only, seeds the Wine apt cache, lives in ci-master.
|
|
expect(wineAptCache.if).toBe("github.event_name == 'push' && github.ref == 'refs/heads/master'")
|
|
expect(wineAptCache['runs-on']).toBe('ubuntu-latest')
|
|
|
|
// serial-windows: master-only standby, self-hosted, non-blocking, lives in ci-master.
|
|
expect(serialWindows.if).toBe("github.event_name == 'push' && github.ref == 'refs/heads/master'")
|
|
expect(serialWindows['runs-on']).toEqual(['self-hosted', 'dsh-win-ci', 'windows'])
|
|
expect(serialWindows.name).toBe('serial / windows (self-hosted standby)')
|
|
// Its store must share the ReFS workspace volume for clone; the install
|
|
// must carry the same filesystem branch as the PR jobs.
|
|
const serialSteps = serialWindows.steps as unknown[]
|
|
const serialStore = serialSteps.find((step): step is Record<string, unknown> & { run: string } => (
|
|
isRecord(step) && step.name === 'Configure persistent pnpm store' && typeof step.run === 'string'
|
|
))
|
|
expect(serialStore).toBeDefined()
|
|
expect(serialStore!.run).toContain('F:\\.pnpm-store')
|
|
const serialInstall = serialSteps.find((step): step is Record<string, unknown> & { run: string } => (
|
|
isRecord(step) && step.name === 'Install (immutable)' && typeof step.run === 'string'
|
|
))
|
|
expect(serialInstall).toBeDefined()
|
|
expect(serialInstall!.run).toContain("$fs -eq 'ReFS'")
|
|
expect(serialInstall!.run).toContain('--package-import-method=clone')
|
|
expect(serialInstall!.run).toContain('corepack pnpm install')
|
|
// Distinct else-branch line, as for the PR jobs: the corepack clone line
|
|
// contains the plain-install substring too.
|
|
expect(serialInstall!.run.split('\n').map(line => line.trim())).toContain('} else {')
|
|
expect(serialInstall!.run.split('\n').map(line => line.trim())).toContain('pnpm install --frozen-lockfile')
|
|
expect(serialInstall!.run).not.toContain('$cloneFlag')
|
|
// The unsharded reference runs the whole coverage inventory at the same
|
|
// per-test budget the PR coverage lane grants; the default 5000ms times
|
|
// out load-sensitive store scans (e.g. gen-third-party-notices).
|
|
const serialGate = serialSteps.find((step): step is Record<string, unknown> & { env?: Record<string, unknown> } => (
|
|
isRecord(step) && step.name === 'Run complete unsharded Windows gate inventory serially'
|
|
))
|
|
expect(serialGate).toBeDefined()
|
|
expect(serialGate!.env).toMatchObject({ DSH_COVERAGE_TEST_TIMEOUT_MS: '90000' })
|
|
|
|
// Aggregate: Wine and the required split native jobs are needed;
|
|
// windows-coverage is temporarily non-blocking while Windows ACP
|
|
// half-close tests are stabilized; observational stays out too.
|
|
expect(aggregate.needs).toContain('windows')
|
|
expect(aggregate.needs).toContain('windows-build')
|
|
// The benchmark lane is a required verdict input and runs alone so its
|
|
// wall-clock budgets never share a runner with a concurrent aggregate.
|
|
expect(aggregate.needs).toContain('node-24-bench')
|
|
expect(node24Bench.name).toBe('node 24 / benchmarks')
|
|
expect(node24Bench.env).toBeUndefined()
|
|
expect(aggregate.needs).not.toContain('windows-coverage')
|
|
expect(aggregate.needs).toContain('windows-native-tests')
|
|
expect(aggregate.needs).not.toContain('windows-observational')
|
|
expect(aggregate.needs).not.toContain('serial-windows')
|
|
|
|
// Linux failover is a separate switch: the four required Linux workers
|
|
// and the verdict job resolve their pool through DSH_CI_FAILOVER_LINUX,
|
|
// never the Windows switch.
|
|
for (const [jobName, job] of [['node-24', node24], ['node-24-coverage', node24Coverage], ['node-24-bench', node24Bench], ['node-24-consumers', node24Consumers]] as const) {
|
|
expect(typeof job['runs-on']).toBe('string')
|
|
expect(job['runs-on'], `${jobName} runs-on must use the Linux failover switch`).toContain('DSH_CI_FAILOVER_LINUX')
|
|
expect(job['runs-on'], `${jobName} runs-on must not use the Windows failover switch`).not.toContain('DSH_CI_FAILOVER_WINDOWS')
|
|
expect(job['runs-on']).toContain('vm-backup')
|
|
}
|
|
expect(aggregate['runs-on']).toContain('DSH_CI_FAILOVER_LINUX')
|
|
expect(aggregate['runs-on']).not.toContain('DSH_CI_FAILOVER_WINDOWS')
|
|
expect(aggregate['runs-on']).toContain('vm-backup')
|
|
|
|
// The run-gates aggregate lanes stop at the first blocking gate failure so
|
|
// a red aggregate does not keep burning runner time on the remaining
|
|
// gates. Removing the flag silently reverts to running every independent
|
|
// gate to completion.
|
|
for (const [jobName, job] of [['node-24', node24], ['node-24-coverage', node24Coverage], ['node-24-consumers', node24Consumers], ['node-compat', nodeCompat]] as const) {
|
|
expect(job.env, `${jobName} must enable fail-fast`).toMatchObject({ DSH_GATE_FAIL_FAST: '1' })
|
|
}
|
|
|
|
// The native Windows lanes with run-gates aggregates fail fast for the
|
|
// same reason: a failing gate aborts the sibling gate instead of waiting
|
|
// out the multi-minute instrumented coverage run.
|
|
expect(windowsBuild.env, 'windows-build must enable fail-fast').toMatchObject({ DSH_GATE_FAIL_FAST: '1' })
|
|
expect(windowsCoverage.env, 'windows-coverage must enable fail-fast').toMatchObject({ DSH_GATE_FAIL_FAST: '1' })
|
|
|
|
// The observational lane stays complete: it is continue-on-error by design
|
|
// and exists to collect as much Windows-native evidence per run as
|
|
// possible, so the first failure must not truncate the rest.
|
|
expect(windowsObservational.env).toBeDefined()
|
|
expect(windowsObservational.env).not.toMatchObject({ DSH_GATE_FAIL_FAST: '1' })
|
|
})
|
|
|
|
it('gives the Wine Host TypeScript compile the repository heap budget', () => {
|
|
const wineGates = readFileSync(resolve(root, 'scripts/wine-windows-gates.sh'), 'utf8')
|
|
|
|
expect(wineGates).toContain(
|
|
'wine_node "$scratch/logs/host-tsc.log" --max-old-space-size=4096 "$tsc_js" -b tsconfig.host.json --pretty false',
|
|
)
|
|
})
|
|
|
|
it('exempts push from cancellation in ci-master, so one master merge does not cancel the running drill', () => {
|
|
const workflow = loadWorkflow('.github/workflows/ci-master.yml')
|
|
const prWorkflow = loadWorkflow('.github/workflows/ci.yml')
|
|
if (!isRecord(workflow.jobs) || !isRecord(workflow.concurrency)) {
|
|
throw new TypeError('ci-master workflow must define jobs and a workflow-level concurrency block')
|
|
}
|
|
if (!isRecord(prWorkflow.jobs)) {
|
|
throw new TypeError('ci workflow must define jobs')
|
|
}
|
|
|
|
// Cancellation applies to the whole superseded RUN, so this has to be
|
|
// decided at workflow level and gated on the event: a job-level group
|
|
// cannot exempt its job from its run being cancelled. Only push is exempt —
|
|
// a drill takes longer than the interval between master merges. The negated
|
|
// form is load-bearing: `== 'pull_request'` would also stop cancelling
|
|
// workflow_dispatch, and a re-dispatched runner benchmark holds up to 12
|
|
// larger runners for 15 minutes in this same group on master.
|
|
expect(workflow.concurrency['cancel-in-progress']).toBe("${{ github.event_name != 'push' }}")
|
|
|
|
// The PR-only ci.yml still cancels a superseded run on a new push, so a
|
|
// fresh head does not stack a second full 9-job run behind a stale one.
|
|
// Unlike ci-master it has no push carve-out: every PR event supersedes.
|
|
expect(prWorkflow.concurrency).toMatchObject({
|
|
'cancel-in-progress': true,
|
|
})
|
|
|
|
// The exact event sets are what keep master-only jobs out of the PR check
|
|
// panel: ci-master triggers only on push(master) + workflow_dispatch and
|
|
// never on pull_request; ci.yml is exactly pull_request-only. Assert the
|
|
// full sets so losing the wrong event, or gaining an extra one, fails.
|
|
if (!isRecord(workflow.on) || !isRecord(prWorkflow.on)) {
|
|
throw new TypeError('both CI workflows must define on')
|
|
}
|
|
expect(Object.keys(workflow.on).sort()).toEqual(['push', 'workflow_dispatch'])
|
|
expect(Object.keys(prWorkflow.on)).toEqual(['pull_request'])
|
|
|
|
// Neither drill may carry a job-level group: it would not exempt the job
|
|
// from run-scoped cancellation.
|
|
for (const name of ['serial-linux-selfhosted', 'serial-windows']) {
|
|
const job = workflow.jobs[name]
|
|
if (!isRecord(job)) throw new TypeError(`${name} must be defined`)
|
|
expect(job.concurrency).toBeUndefined()
|
|
// Both stay master-push-only; that is what makes the push carve-out safe.
|
|
expect(job.if).toBe("github.event_name == 'push' && github.ref == 'refs/heads/master'")
|
|
}
|
|
|
|
// What bounds the cost of exempting push: a master push may only carry the
|
|
// cache seeder and the two drills. Any job reachable on push would start
|
|
// accumulating uncancelled runs, so the set is pinned here.
|
|
const NOT_PUSH_REACHABLE = new Set([
|
|
"github.event_name == 'workflow_dispatch' && inputs.suite == 'larger-runner-benchmark'",
|
|
"github.event_name == 'workflow_dispatch' && inputs.suite == 'consolidated-runner-benchmark'",
|
|
])
|
|
const pushReachable = Object.entries(workflow.jobs)
|
|
.filter(([, job]) => {
|
|
if (!isRecord(job)) return false
|
|
if (job.if === undefined) return true // unconditional: runs on every event
|
|
if (job.if === false) return false // `if: false` parses as a boolean
|
|
if (typeof job.if !== 'string') return true // unrecognized shape: surface it
|
|
return !NOT_PUSH_REACHABLE.has(job.if.trim())
|
|
})
|
|
.map(([name]) => name)
|
|
.sort()
|
|
expect(pushReachable).toEqual(['serial-linux-selfhosted', 'serial-windows', 'wine-apt-cache'])
|
|
|
|
// Why workflow_dispatch must keep cancelling: each benchmark fans out to a
|
|
// dozen larger runners at once, in this same group on master. If it stopped
|
|
// cancelling, a re-dispatch would queue ahead of a drill instead of
|
|
// replacing the stale measurement.
|
|
for (const name of ['larger-runner-benchmark', 'consolidated-runner-benchmark']) {
|
|
const job = workflow.jobs[name]
|
|
if (!isRecord(job) || !isRecord(job.strategy)) {
|
|
throw new TypeError(`${name} must define a matrix strategy`)
|
|
}
|
|
expect(job.strategy['max-parallel']).toBe(12)
|
|
expect(job['timeout-minutes']).toBe(15)
|
|
}
|
|
})
|
|
|
|
it('keeps supported LSP source under native Windows coverage', () => {
|
|
const config = readFileSync(resolve(root, 'vitest.config.ts'), 'utf8')
|
|
|
|
expect(config).not.toContain('packages/lsp/lsp-stdio/src/connection.ts')
|
|
expect(config).not.toContain('packages/lsp/lsp-stdio/src/index.ts')
|
|
expect(config).not.toContain('packages/lsp/lsp-stdio/src/instance.ts')
|
|
})
|
|
|
|
it('requires release-shaped Python runtime validation on every published target', () => {
|
|
const workflow = loadWorkflow('.github/workflows/ci.yml')
|
|
const pythonRuntime = workflowJob(workflow, 'python-runtime')
|
|
const aggregate = workflowJob(workflow, 'all-checks-passed')
|
|
if (!Array.isArray(aggregate.needs)) {
|
|
throw new TypeError('CI aggregate must define required job dependencies')
|
|
}
|
|
|
|
expect(pythonRuntime).toMatchObject({
|
|
if: "github.event_name == 'pull_request'",
|
|
name: 'python runtime / release-shaped matrix',
|
|
uses: './.github/workflows/build-exe-for-python-sdk.yml',
|
|
with: {
|
|
targets: 'node24-linux-x64,node24-linux-arm64,node24-macos-arm64,node24-macos-x64,node24-win-x64',
|
|
ci: true,
|
|
},
|
|
secrets: {
|
|
DEEPSEEK_API_KEY_EXTERNAL: '${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }}',
|
|
},
|
|
})
|
|
expect(aggregate.needs).toContain('python-runtime')
|
|
})
|
|
|
|
it('keeps every Vitest project process-isolated on native Windows', () => {
|
|
const config = readFileSync(resolve(root, 'vitest.config.ts'), 'utf8')
|
|
|
|
expect(config).not.toContain("pool: process.platform === 'win32' ? 'threads' : 'forks'")
|
|
expect(config.match(/pool: 'forks'/g)).toHaveLength(2)
|
|
})
|
|
})
|
|
|
|
describe('DeepSeek e2e workflow', () => {
|
|
it('prepares bubblewrap from the pinned payload without a package transaction', () => {
|
|
const workflow = loadWorkflow('.github/workflows/e2e.yml')
|
|
const e2e = workflowJob(workflow, 'e2e')
|
|
if (!Array.isArray(e2e.steps)) throw new TypeError('DeepSeek e2e workflow must define steps')
|
|
|
|
const steps = e2e.steps.filter(isRecord)
|
|
expect(steps.find(step => step.name === 'Prepare bubblewrap (unrestrict userns)')).toMatchObject({
|
|
run: 'bash scripts/prepare-ci-bubblewrap.sh',
|
|
})
|
|
expect(JSON.stringify(steps)).not.toContain('apt-get')
|
|
})
|
|
|
|
it('bounds profile subprocess fan-out to the tested e2e default', () => {
|
|
const workflow = loadWorkflow('.github/workflows/e2e.yml')
|
|
const e2e = workflowJob(workflow, 'e2e')
|
|
if (!Array.isArray(e2e.steps)) throw new TypeError('DeepSeek e2e workflow must define steps')
|
|
|
|
const step = e2e.steps.filter(isRecord).find(candidate => candidate.name === 'E2E tests (real DeepSeek API)')
|
|
expect(step).toMatchObject({ env: { DSH_E2E_MAX_WORKERS: 4 } })
|
|
})
|
|
})
|
|
|
|
describe('E2B e2e workflow', () => {
|
|
it('is manual-only and fails loud before running the focused live suite', () => {
|
|
const workflow = loadWorkflow('.github/workflows/e2b-e2e.yml')
|
|
expect(workflow.on).toEqual({ workflow_dispatch: null })
|
|
if (!isRecord(workflow.jobs) || !isRecord(workflow.jobs.e2b) || !Array.isArray(workflow.jobs.e2b.steps)) {
|
|
throw new TypeError('E2B e2e workflow must define the e2b job steps')
|
|
}
|
|
|
|
const steps = workflow.jobs.e2b.steps.filter(isRecord)
|
|
const preflight = steps.find(step => step.name === 'Preflight (require E2B API key)')
|
|
const e2b = steps.find(step => step.name === 'E2B tests (live sandbox)')
|
|
|
|
expect(preflight).toMatchObject({
|
|
env: { E2B_API_KEY: '${{ secrets.E2B_API_KEY_EXTERNAL }}' },
|
|
})
|
|
expect(preflight?.run).toContain('E2B_API_KEY_EXTERNAL repository secret')
|
|
expect(e2b).toMatchObject({
|
|
env: {
|
|
E2B_API_KEY: '${{ secrets.E2B_API_KEY_EXTERNAL }}',
|
|
DSH_E2E_MAX_WORKERS: '1',
|
|
DSH_EXAMPLE_MODE: 'lib',
|
|
},
|
|
})
|
|
expect(e2b?.run).toContain('packages/e2b/e2b/tests/composition.e2e.ts')
|
|
})
|
|
})
|
|
|
|
describe('Python release workflows', () => {
|
|
it('keeps complete wheel validation separate from protected public publication', () => {
|
|
const workflow = loadWorkflow('.github/workflows/python-release.yml')
|
|
const dispatch = workflowEvent(workflow, 'workflow_dispatch')
|
|
const build = workflowJob(workflow, 'build')
|
|
const pythonCompat = workflowJob(workflow, 'python-compat')
|
|
const validate = workflowJob(workflow, 'validate')
|
|
const publishRuntime = workflowJob(workflow, 'publish-runtime')
|
|
const publishSdk = workflowJob(workflow, 'publish-sdk')
|
|
if (!isRecord(dispatch.inputs)
|
|
|| !isRecord(dispatch.inputs.publish)
|
|
|| !Array.isArray(pythonCompat.steps)
|
|
|| !Array.isArray(validate.steps)
|
|
|| !Array.isArray(publishRuntime.steps)
|
|
|| !Array.isArray(publishSdk.steps)) {
|
|
throw new TypeError('Python release workflow must define publish input and release steps')
|
|
}
|
|
|
|
expect(dispatch.inputs.publish).toMatchObject({ type: 'boolean', default: false })
|
|
if (!isRecord(workflow.on)) throw new TypeError('python-release workflow must define on')
|
|
expect(Object.keys(workflow.on)).toEqual(['workflow_dispatch'])
|
|
expect(build).toMatchObject({
|
|
uses: './.github/workflows/build-exe-for-python-sdk.yml',
|
|
with: {
|
|
targets: 'node24-linux-x64,node24-linux-arm64,node24-macos-arm64,node24-macos-x64,node24-win-x64',
|
|
release: true,
|
|
},
|
|
})
|
|
expect(pythonCompat.strategy).toMatchObject({ matrix: { python: ['3.10', '3.14'] } })
|
|
const pythonCompatSteps = JSON.stringify(pythonCompat.steps)
|
|
expect(pythonCompatSteps).toContain('dist/deepseek_harness_sdk-$VERSION-py3-none-any.whl')
|
|
expect(pythonCompatSteps).toContain('dist/deepseek_harness_runtime_bin-$VERSION-py3-none-manylinux_2_28_x86_64.whl')
|
|
expect(pythonCompatSteps).not.toContain('--find-links')
|
|
const validateSteps = JSON.stringify(validate.steps)
|
|
const authorize = validate.steps.filter(isRecord).find(step => step.name === 'Authorize publication request')
|
|
if (!isRecord(authorize) || typeof authorize.run !== 'string') {
|
|
throw new TypeError('Python release validation must authorize publication requests')
|
|
}
|
|
expect(validateSteps).toContain('PUBLIC_PYPI_RELEASE_ENABLED')
|
|
expect(authorize).toMatchObject({
|
|
env: {
|
|
PYPI_PUBLISHER_REPOSITORY: '${{ vars.PYPI_PUBLISHER_REPOSITORY }}',
|
|
REPOSITORY: '${{ github.repository }}',
|
|
},
|
|
})
|
|
expect(authorize.run).toContain('[ "$REPOSITORY" = "$PYPI_PUBLISHER_REPOSITORY" ]')
|
|
expect(validateSteps).toContain('100000000')
|
|
expect(publishRuntime).toMatchObject({
|
|
if: "github.event_name == 'workflow_dispatch' && inputs.publish",
|
|
needs: 'validate',
|
|
environment: 'pypi-runtime',
|
|
permissions: { contents: 'read', 'id-token': 'write' },
|
|
})
|
|
expect(publishSdk).toMatchObject({
|
|
if: "github.event_name == 'workflow_dispatch' && inputs.publish",
|
|
needs: ['validate', 'publish-runtime'],
|
|
environment: 'pypi',
|
|
permissions: { contents: 'read', 'id-token': 'write' },
|
|
})
|
|
const runtimeSteps = publishRuntime.steps.filter(isRecord)
|
|
const sdkSteps = publishSdk.steps.filter(isRecord)
|
|
const runtimePublish = runtimeSteps.find(step => step.name === 'Publish runtime wheels')
|
|
const sdkPublish = sdkSteps.find(step => step.name === 'Publish SDK wheel')
|
|
const runtimeHashes = runtimeSteps.find(step => step.name === 'Verify release artifact hashes')
|
|
const sdkHashes = sdkSteps.find(step => step.name === 'Verify release artifact hashes')
|
|
expect([...runtimeSteps, ...sdkSteps].some(
|
|
step => typeof step.uses === 'string' && step.uses.startsWith('actions/checkout@'),
|
|
)).toBe(false)
|
|
expect([...runtimeSteps, ...sdkSteps].filter(
|
|
step => step.uses === 'pypa/gh-action-pypi-publish@release/v1',
|
|
)).toHaveLength(2)
|
|
expect(runtimePublish).toMatchObject({
|
|
with: { 'packages-dir': 'dist/runtime/', attestations: false },
|
|
})
|
|
expect(sdkPublish).toMatchObject({
|
|
with: { 'packages-dir': 'dist/sdk/', attestations: false },
|
|
})
|
|
expect(runtimeHashes).toMatchObject({ run: 'cd dist && sha256sum -c SHA256SUMS' })
|
|
expect(sdkHashes).toMatchObject({ run: 'cd dist && sha256sum -c SHA256SUMS' })
|
|
})
|
|
|
|
it('exposes the native wheel builder to the release caller with normalized versions', () => {
|
|
const workflow = loadWorkflow('.github/workflows/build-exe-for-python-sdk.yml')
|
|
expect(Object.keys(workflow.on as Record<string, unknown>).sort()).toEqual(['workflow_call', 'workflow_dispatch'])
|
|
const call = workflowEvent(workflow, 'workflow_call')
|
|
const plan = workflowJob(workflow, 'plan')
|
|
const build = workflowJob(workflow, 'build')
|
|
if (!isRecord(call.inputs) || !isRecord(call.secrets) || !Array.isArray(plan.steps) || !Array.isArray(build.steps)) {
|
|
throw new TypeError('Python wheel builder must define workflow_call inputs and plan steps')
|
|
}
|
|
|
|
const buildSteps: unknown[] = build.steps
|
|
const manylinuxAddon = buildSteps.find(step => isRecord(step) && step.name === 'Rebuild Linux node-pty against manylinux 2.28')
|
|
const macosCheck = buildSteps.find(step => isRecord(step) && step.name === 'Check macOS payload architecture and deployment target')
|
|
const manylinuxSmoke = buildSteps.find(step => isRecord(step) && step.name === 'Run wheel in a manylinux 2.28 container')
|
|
const cleanVenvPosix = buildSteps.find(step => isRecord(step) && step.name === 'Install local SDK and runtime wheels into a clean venv (POSIX)')
|
|
const cleanVenvWindows = buildSteps.find(step => isRecord(step) && step.name === 'Install local SDK and runtime wheels into a clean venv (Windows)')
|
|
const installedKeylessPosix = buildSteps.find(step => isRecord(step) && step.name === 'Run installed-wheel keyless black-box tests (POSIX)')
|
|
const installedKeylessWindows = buildSteps.find(step => isRecord(step) && step.name === 'Run installed-wheel keyless black-box tests (Windows)')
|
|
const realApiPreflightPosix = buildSteps.find(step => isRecord(step) && step.name === 'Preflight installed-wheel real API test (POSIX)')
|
|
const realApiPreflightWindows = buildSteps.find(step => isRecord(step) && step.name === 'Preflight installed-wheel real API test (Windows)')
|
|
const installedRealApiPosix = buildSteps.find(step => isRecord(step) && step.name === 'Run installed-wheel real API black-box test (POSIX)')
|
|
const installedRealApiWindows = buildSteps.find(step => isRecord(step) && step.name === 'Run installed-wheel real API black-box test (Windows)')
|
|
if (!isRecord(macosCheck) || typeof macosCheck.run !== 'string'
|
|
|| !isRecord(cleanVenvPosix) || !isRecord(cleanVenvWindows)
|
|
|| !isRecord(installedKeylessPosix) || !isRecord(installedKeylessWindows)
|
|
|| !isRecord(realApiPreflightPosix) || !isRecord(realApiPreflightWindows)
|
|
|| !isRecord(installedRealApiPosix) || !isRecord(installedRealApiWindows)) {
|
|
throw new TypeError('Python wheel builder must define native POSIX and Windows installed-wheel steps')
|
|
}
|
|
expect(call.inputs).toHaveProperty('targets')
|
|
expect(call.inputs).toMatchObject({
|
|
ci: { type: 'boolean', default: false },
|
|
release: { type: 'boolean', default: false },
|
|
})
|
|
expect(call.secrets).toMatchObject({
|
|
DEEPSEEK_API_KEY_EXTERNAL: { required: false },
|
|
})
|
|
expect(workflow.concurrency).toMatchObject({
|
|
group: 'build-single-exe-${{ github.workflow }}-${{ github.ref }}',
|
|
})
|
|
expect(build.defaults).toBeUndefined()
|
|
expect(plan.if).toContain('inputs.ci')
|
|
expect(plan.if).toContain('inputs.release')
|
|
expect(JSON.stringify(plan.steps)).toContain('pep440_version')
|
|
const workflowJson = JSON.stringify(workflow)
|
|
expect(workflowJson).toContain('macosx_14_0_arm64')
|
|
expect(workflowJson).toContain('macosx_14_0_x86_64')
|
|
expect(workflowJson).toContain('node24-macos-x64')
|
|
expect(workflowJson).toContain('macos-15-intel')
|
|
expect(workflowJson).toContain('win_amd64')
|
|
expect(workflowJson).toContain('node24-win-x64')
|
|
expect(workflowJson).toContain('windows-2025')
|
|
expect(workflowJson).toContain('dist-python/$SDK_WHEEL')
|
|
expect(workflowJson).toContain('dist-python/$RUNTIME_WHEEL')
|
|
expect(workflowJson).toContain('/work/dist-python/$SDK_WHEEL')
|
|
expect(workflowJson).toContain('/work/dist-python/$RUNTIME_WHEEL')
|
|
expect(workflowJson).not.toContain('--find-links dist-python')
|
|
expect(workflowJson).not.toContain('--find-links /work/dist-python')
|
|
expect(workflowJson).not.toContain('cygpath')
|
|
expect(manylinuxAddon).toMatchObject({ if: "runner.os == 'Linux'" })
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('manylinux_2_28_x86_64')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('manylinux_2_28_aarch64')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('npm_config_build_from_source=true pnpm run install')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('pnpm_setup_root')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('$pnpm_setup_root:$pnpm_setup_root:ro')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('node-pty-glibc-versions.txt')
|
|
expect(JSON.stringify(manylinuxAddon)).toContain('le 2.28')
|
|
expect(macosCheck).toMatchObject({ if: "runner.os == 'macOS'" })
|
|
expect(macosCheck.run).toContain('scripts/check-macos-deployment-target.py')
|
|
expect(macosCheck.run).toContain('lipo "$payload" -verify_arch')
|
|
expect(macosCheck.run).toContain('$EXE-rg')
|
|
expect(macosCheck.run).toContain('$EXE-spawn-helper')
|
|
expect(JSON.stringify(installedKeylessPosix)).toContain('--scenario all')
|
|
expect(JSON.stringify(installedKeylessPosix)).toContain('env -u PYTHONPATH')
|
|
expect(JSON.stringify(installedKeylessWindows)).toContain('--scenario all --installed-wheel')
|
|
expect(installedKeylessWindows).toMatchObject({ if: "runner.os == 'Windows'", shell: 'pwsh' })
|
|
expect(cleanVenvWindows).toMatchObject({ if: "runner.os == 'Windows'", shell: 'pwsh' })
|
|
expect(JSON.stringify(cleanVenvWindows)).toContain('Scripts\\\\python.exe')
|
|
expect(realApiPreflightPosix).toMatchObject({
|
|
env: { DEEPSEEK_API_KEY: '${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }}' },
|
|
})
|
|
expect(String(realApiPreflightPosix.if)).toContain('inputs.ci')
|
|
expect(String(realApiPreflightPosix.if)).toContain('head.repo.fork')
|
|
expect(String(realApiPreflightPosix.if)).toContain('dependabot[bot]')
|
|
expect(realApiPreflightWindows).toMatchObject({ shell: 'pwsh' })
|
|
expect(installedRealApiPosix).toMatchObject({
|
|
env: {
|
|
DEEPSEEK_API_KEY: '${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }}',
|
|
DEEPSEEK_BASE_URL: 'https://api.deepseek.com',
|
|
},
|
|
})
|
|
expect(JSON.stringify(installedRealApiPosix)).toContain('--scenario sdk-live')
|
|
expect(JSON.stringify(installedRealApiPosix)).toContain('-u DSH_RUNTIME_MODE')
|
|
expect(installedRealApiWindows).toMatchObject({ shell: 'pwsh' })
|
|
expect(JSON.stringify(installedRealApiWindows)).toContain('--scenario sdk-live --installed-wheel')
|
|
expect(manylinuxSmoke).toMatchObject({ if: "runner.os == 'Linux'" })
|
|
expect(JSON.stringify(manylinuxSmoke)).toContain('-e DSH_TELEMETRY_DISABLED')
|
|
})
|
|
|
|
it('uses the shared macOS deployment-target check in GitLab', () => {
|
|
const workflow = loadWorkflow('.gitlab-ci.yml')
|
|
const runtimeWheel = workflow['.runtime-wheel']
|
|
if (!isRecord(runtimeWheel) || !Array.isArray(runtimeWheel.script)) {
|
|
throw new TypeError('GitLab CI must define the runtime wheel script')
|
|
}
|
|
const runtimeScript: unknown[] = runtimeWheel.script
|
|
const macosCheck = runtimeScript.find(
|
|
step => typeof step === 'string' && step.includes('${PLATFORM#macos-}'),
|
|
)
|
|
if (typeof macosCheck !== 'string') {
|
|
throw new TypeError('GitLab CI must check the macOS deployment target')
|
|
}
|
|
|
|
expect(macosCheck).toContain('scripts/check-macos-deployment-target.py')
|
|
expect(macosCheck).toContain('lipo "$payload" -verify_arch')
|
|
expect(macosCheck).toContain('"$EXE" "$EXE-rg" "$EXE-spawn-helper"')
|
|
})
|
|
|
|
it('builds the macOS x64 wheel on the matching GitLab runner', () => {
|
|
const workflow = loadWorkflow('.gitlab-ci.yml')
|
|
const macosX64 = workflow['runtime-macos-x64']
|
|
const publish = workflow['publish-python']
|
|
if (!isRecord(macosX64) || !isRecord(publish) || !Array.isArray(publish.needs)) {
|
|
throw new TypeError('GitLab CI must define the macOS x64 runtime and publication jobs')
|
|
}
|
|
|
|
expect(macosX64.tags).toEqual(['macos-x64'])
|
|
expect(macosX64.variables).toMatchObject({ PKG_TARGET: 'node24-macos-x64', PLATFORM: 'macos-x64' })
|
|
expect(publish.needs).toContainEqual({ job: 'runtime-macos-x64', artifacts: true })
|
|
expect(JSON.stringify(publish.script)).toContain('macosx_14_0_x86_64.whl')
|
|
})
|
|
|
|
it('builds and black-box tests the Windows x64 wheel in GitLab', () => {
|
|
const workflow = loadWorkflow('.gitlab-ci.yml')
|
|
const windows = workflow['runtime-windows-x64']
|
|
const publish = workflow['publish-python']
|
|
if (!isRecord(windows) || !Array.isArray(windows.before_script) || !Array.isArray(windows.script)
|
|
|| !isRecord(publish) || !Array.isArray(publish.needs)) {
|
|
throw new TypeError('GitLab CI must define the Windows runtime and aggregate publication jobs')
|
|
}
|
|
|
|
expect(windows.tags).toEqual(['windows-x64'])
|
|
expect(windows.variables).toMatchObject({ PKG_TARGET: 'node24-win-x64', PLATFORM: 'win-x64' })
|
|
expect(JSON.stringify(windows.before_script)).toContain('.ci-python\\\\Scripts')
|
|
expect(JSON.stringify(windows.before_script)).toContain('[IO.Path]::PathSeparator')
|
|
expect(JSON.stringify(windows.script)).toContain('win_amd64.whl')
|
|
expect(JSON.stringify(windows.script)).toContain('--scenario all --installed-wheel')
|
|
expect(publish.needs).toContainEqual({ job: 'runtime-windows-x64', artifacts: true })
|
|
})
|
|
})
|
|
|
|
describe('Issue lifecycle workflow', () => {
|
|
it('runs the lifecycle job on every PR/review event but gates token and board steps', () => {
|
|
const lifecycle = loadWorkflow('.github/workflows/issue-lifecycle.yml')
|
|
const policy = loadWorkflow('.github/workflows/issue-policy.yml')
|
|
const lifecycleJob = workflowJob(lifecycle, 'lifecycle')
|
|
if (!Array.isArray(lifecycleJob.steps)) throw new TypeError('Issue lifecycle job must define steps')
|
|
|
|
// The job has no job-level `if`, so it is listed on every pull_request /
|
|
// pull_request_review event and reports success instead of a gray skip. The
|
|
// write-capable steps are gated at step level so approved/commented reviews
|
|
// never mint a Project/Issue App token nor touch the board.
|
|
expect(lifecycle.on).toHaveProperty('pull_request')
|
|
expect(lifecycle.on).toHaveProperty('pull_request_review')
|
|
expect(lifecycleJob.if).toBeUndefined()
|
|
// Keep the subscription-type gates: issue-lifecycle does not re-subscribe
|
|
// ready_for_review (issue-policy owns that) and only reacts to submitted
|
|
// review events.
|
|
const lifecyclePullRequest = workflowEvent(lifecycle, 'pull_request')
|
|
const lifecycleReview = workflowEvent(lifecycle, 'pull_request_review')
|
|
expect(lifecyclePullRequest.types).toContain('opened')
|
|
expect(lifecyclePullRequest.types).not.toContain('ready_for_review')
|
|
expect(lifecyclePullRequest.types).toContain('review_requested')
|
|
expect(lifecycleReview.types).toEqual(['submitted'])
|
|
const gated = "${{ github.event_name != 'pull_request_review' || github.event.review.state == 'changes_requested' }}"
|
|
const steps = lifecycleJob.steps.filter(isRecord)
|
|
const tokenStep = steps.find(s => s.name === 'Create project token')
|
|
const handleStep = steps.find(s => s.name === 'Handle repository event')
|
|
expect(tokenStep).toMatchObject({ if: gated })
|
|
expect(handleStep).toMatchObject({ if: gated })
|
|
|
|
// issue-policy owns PR validation; it is read-only and a real gate.
|
|
const policyPullRequest = workflowEvent(policy, 'pull_request')
|
|
expect(policyPullRequest.types).toContain('ready_for_review')
|
|
})
|
|
|
|
it('uses a read-only Project token only for human pull request policy metadata', () => {
|
|
const policy = loadWorkflow('.github/workflows/issue-policy.yml')
|
|
const policyJob = workflowJob(policy, 'policy')
|
|
if (!Array.isArray(policyJob.steps)) throw new TypeError('Issue policy job must define steps')
|
|
const steps = policyJob.steps.filter(isRecord)
|
|
const tokenStep = steps.find(step => step.name === 'Create Project read token')
|
|
const validateStep = steps.find(step => step.name === 'Validate pull request')
|
|
const humanPullRequest =
|
|
"${{ github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.type != 'App' }}"
|
|
|
|
expect(tokenStep).toMatchObject({
|
|
id: 'app-token',
|
|
if: humanPullRequest,
|
|
uses: 'actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1',
|
|
with: {
|
|
'client-id': '${{ vars.DSH_ISSUE_APP_CLIENT_ID }}',
|
|
'private-key': '${{ secrets.DSH_ISSUE_APP_PRIVATE_KEY }}',
|
|
owner: 'deepseek-harness',
|
|
repositories: 'deepseek-harness',
|
|
'permission-issues': 'read',
|
|
'permission-organization-projects': 'read',
|
|
},
|
|
})
|
|
expect(validateStep).toMatchObject({
|
|
if: humanPullRequest,
|
|
env: {
|
|
GITHUB_TOKEN: '${{ github.token }}',
|
|
PROJECT_TOKEN: '${{ steps.app-token.outputs.token }}',
|
|
},
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('npm release workflows', () => {
|
|
it('keeps publication dispatch-only and pack in the PR workflow', () => {
|
|
// pack stays in the PR/master release workflows so a PR proves the set packs.
|
|
for (const file of ['release.yml', 'release-vendor.yml']) {
|
|
const workflow = loadWorkflow(`.github/workflows/${file}`)
|
|
if (!isRecord(workflow.jobs)) throw new TypeError(`${file} must define jobs`)
|
|
expect(Object.keys(workflow.jobs).sort()).toEqual(file === 'release.yml' ? ['dependencies', 'pack'] : ['pack'])
|
|
}
|
|
|
|
// publication is workflow_dispatch-only (never a PR check) and keeps the
|
|
// npm-publish environment plus the shared dist-tag group.
|
|
for (const file of ['release-publish.yml', 'release-vendor-publish.yml']) {
|
|
const workflow = loadWorkflow(`.github/workflows/${file}`)
|
|
if (!isRecord(workflow.on) || !isRecord(workflow.jobs)) throw new TypeError(`${file} must define on and jobs`)
|
|
expect(Object.keys(workflow.on)).toEqual(['workflow_dispatch'])
|
|
const publish = workflow.jobs.publish
|
|
if (!isRecord(publish)) throw new TypeError(`${file} must define a publish job`)
|
|
expect(publish.environment).toBe('npm-publish')
|
|
expect(publish.concurrency).toMatchObject({ group: 'Release-publish' })
|
|
}
|
|
})
|
|
|
|
it('runs dependency policy and npm layout checks in the DSH release workflow', () => {
|
|
const workflow = loadWorkflow('.github/workflows/release.yml')
|
|
const dependencies = workflowJob(workflow, 'dependencies')
|
|
if (!isRecord(workflow.on) || !Array.isArray(dependencies.steps)) {
|
|
throw new TypeError('DSH release workflow must define triggers and dependency steps')
|
|
}
|
|
const commands = dependencies.steps.flatMap(step =>
|
|
isRecord(step) && typeof step.run === 'string' ? [step.run] : [])
|
|
|
|
expect(Object.keys(workflow.on).sort()).toEqual(['pull_request', 'push', 'workflow_dispatch'])
|
|
expect(commands).toContain('pnpm run verify-package-dependencies')
|
|
expect(commands).toContain('pnpm run verify-npm-install-layout')
|
|
})
|
|
})
|
|
|
|
describe('Documentation site publication', () => {
|
|
it('keeps Pages deployment dispatch-only from a dsh-v* tag', () => {
|
|
const workflow = loadWorkflow('.github/workflows/docs-pages.yml')
|
|
const build = workflowJob(workflow, 'build')
|
|
const deploy = workflowJob(workflow, 'deploy')
|
|
if (!isRecord(workflow.on) || !isRecord(workflow.env) || !Array.isArray(build.steps)) {
|
|
throw new TypeError('Documentation deployment must define on, env, and build steps')
|
|
}
|
|
|
|
// The site presents a released snapshot: a merge must never publish it, and
|
|
// publication must never appear as a PR check.
|
|
expect(Object.keys(workflow.on)).toEqual(['workflow_dispatch'])
|
|
|
|
// RELEASE_PUBLISH makes release:verify reject every ref that is not a dsh-v*
|
|
// tag naming this tree's version, so the site and the npm sequence share one
|
|
// definition of a released version.
|
|
const steps = build.steps.filter(isRecord)
|
|
const verify = steps.find(step => step.name === 'Verify release version')
|
|
const checkout = steps.find(
|
|
step => typeof step.uses === 'string' && step.uses.startsWith('actions/checkout@'),
|
|
)
|
|
expect(verify).toMatchObject({
|
|
env: { RELEASE_PUBLISH: 'true' },
|
|
run: 'pnpm run release:verify --family dsh',
|
|
})
|
|
// Complete history: the release scripts read tags.
|
|
expect(checkout).toMatchObject({ with: { 'fetch-depth': 0 } })
|
|
|
|
// Projected source links stay on the public repository's master. That
|
|
// repository advances only to each release commit, so its master never
|
|
// carries unreleased work, while it retains only the most recent tags:
|
|
// following the dispatched tag would leave every source link on a deploy
|
|
// from an older tag unresolvable.
|
|
expect(workflow.env.DOCS_REPOSITORY_REF).toBe('master')
|
|
|
|
// The environment owns the deployment tag policy and the required reviewers.
|
|
expect(deploy.environment).toMatchObject({ name: 'github-pages' })
|
|
})
|
|
})
|
|
|
|
describe('Git hooks', () => {
|
|
it('leaves frozen Agent Note sidecars to the archive verifier', () => {
|
|
const lefthook = loadWorkflow('lefthook.yml')
|
|
|
|
for (const hookName of ['pre-commit', 'pre-merge-commit']) {
|
|
const hook = lefthook[hookName]
|
|
if (!isRecord(hook) || !Array.isArray(hook.jobs)) {
|
|
throw new TypeError(`lefthook must define ${hookName} jobs`)
|
|
}
|
|
const pairing: unknown = hook.jobs.find(
|
|
(job: unknown) => isRecord(job) && job.name === 'translation pairing (staged records)',
|
|
)
|
|
|
|
expect(pairing).toMatchObject({ exclude: ['.agents/notes/archived/**'] })
|
|
}
|
|
})
|
|
})
|
|
|
|
function loadWorkflow(path: string): Record<string, unknown> {
|
|
const workflow: unknown = yaml.load(readFileSync(resolve(root, path), 'utf8'))
|
|
if (!isRecord(workflow)) throw new TypeError(`${path} must define a workflow`)
|
|
return workflow
|
|
}
|
|
|
|
function workflowEvent(workflow: Record<string, unknown>, event: string): Record<string, unknown> {
|
|
if (!isRecord(workflow.on) || !isRecord(workflow.on[event])) {
|
|
throw new TypeError(`workflow must define the ${event} event`)
|
|
}
|
|
return workflow.on[event]
|
|
}
|
|
|
|
function workflowJob(workflow: Record<string, unknown>, job: string): Record<string, unknown> {
|
|
if (!isRecord(workflow.jobs) || !isRecord(workflow.jobs[job])) {
|
|
throw new TypeError(`workflow must define the ${job} job`)
|
|
}
|
|
return workflow.jobs[job]
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
}
|