mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-08-29 04:26:38 +00:00
perf(snapshot): parallelize replay scenarios
This commit is contained in:
@@ -3,6 +3,8 @@
|
|||||||
* compares normalized stdout; comparable session fixtures are both replay input and expected
|
* compares normalized stdout; comparable session fixtures are both replay input and expected
|
||||||
* output. Record mode refreshes reproducible model scenarios from the live API, while refresh
|
* output. Record mode refreshes reproducible model scenarios from the live API, while refresh
|
||||||
* mode replays committed scripts and rewrites derived artifacts without a key.
|
* mode replays committed scripts and rewrites derived artifacts without a key.
|
||||||
|
* Replay scenarios run concurrently because each subprocess owns unique temp cwd and persistence
|
||||||
|
* roots and only reads committed fixtures. Record and refresh scenarios stay serial while writing.
|
||||||
*
|
*
|
||||||
* Exactly one scenario per header-composition class pins the system prompt and tool schemas in
|
* Exactly one scenario per header-composition class pins the system prompt and tool schemas in
|
||||||
* dedicated sidecars. Every live header is checked against that pin, so session-dependent
|
* dedicated sidecars. Every live header is checked against that pin, so session-dependent
|
||||||
@@ -461,7 +463,7 @@ export function stabilizeRefreshLog(fresh: string, existing: string, replacement
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Register the suite: one `describe` per scenario (the golden/log compares and
|
* Register the suite: one test per scenario (the golden/log compares and
|
||||||
* the header-uniformity guard) plus the fixture guard block (no orphan
|
* the header-uniformity guard) plus the fixture guard block (no orphan
|
||||||
* scenario dirs, required files present, exactly one pin per header class,
|
* scenario dirs, required files present, exactly one pin per header class,
|
||||||
* pinning fixtures well-formed, every JSONL prompt-scrubbed, non-pinning
|
* pinning fixtures well-formed, every JSONL prompt-scrubbed, non-pinning
|
||||||
@@ -477,6 +479,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
|||||||
const RECORDING = mode === 'record'
|
const RECORDING = mode === 'record'
|
||||||
const REFRESHING = mode === 'refresh'
|
const REFRESHING = mode === 'refresh'
|
||||||
const childMode: 'replay' | 'record' = RECORDING ? 'record' : 'replay'
|
const childMode: 'replay' | 'record' = RECORDING ? 'record' : 'replay'
|
||||||
|
const scenarioSuite = mode === 'replay' ? describe.concurrent : describe
|
||||||
|
|
||||||
/** The class a scenario's header composition belongs to (see {@link Scenario.headerClass}). */
|
/** The class a scenario's header composition belongs to (see {@link Scenario.headerClass}). */
|
||||||
const classOf = (scenario: Scenario): string => scenario.headerClass ?? 'default'
|
const classOf = (scenario: Scenario): string => scenario.headerClass ?? 'default'
|
||||||
@@ -496,11 +499,11 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const scenario of scenarios) {
|
scenarioSuite('snapshot scenarios', () => {
|
||||||
describe(`snapshot: ${scenario.name}`, () => {
|
for (const scenario of scenarios) {
|
||||||
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
|
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
|
||||||
// (sidecar-driven errors/cancel) are never re-recorded.
|
// (sidecar-driven errors/cancel) are never re-recorded.
|
||||||
it.skipIf(RECORDING && !scenario.recorded)('matches the goldens', async () => {
|
it.skipIf(RECORDING && !scenario.recorded)(`snapshot: ${scenario.name} matches the goldens`, async ({ expect }) => {
|
||||||
const dir = join(snapshotsDir, scenario.name)
|
const dir = join(snapshotsDir, scenario.name)
|
||||||
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
||||||
const overrideFile = join(dir, 'replay.override.json')
|
const overrideFile = join(dir, 'replay.override.json')
|
||||||
@@ -658,8 +661,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
})
|
}
|
||||||
}
|
})
|
||||||
|
|
||||||
describe('snapshot fixtures', () => {
|
describe('snapshot fixtures', () => {
|
||||||
it('every scenario directory is registered (no orphans)', async () => {
|
it('every scenario directory is registered (no orphans)', async () => {
|
||||||
|
|||||||
@@ -1,6 +1,25 @@
|
|||||||
|
import { availableParallelism } from 'node:os'
|
||||||
import tsconfigPaths from 'vite-tsconfig-paths'
|
import tsconfigPaths from 'vite-tsconfig-paths'
|
||||||
import { defineConfig } from 'vitest/config'
|
import { defineConfig } from 'vitest/config'
|
||||||
|
|
||||||
|
const DEFAULT_SNAPSHOT_MAX_CONCURRENCY = 5
|
||||||
|
|
||||||
|
function positiveIntFromEnv(name: string, fallback: number): number {
|
||||||
|
const raw = process.env[name]
|
||||||
|
if (raw === undefined || raw === '') return fallback
|
||||||
|
|
||||||
|
const value = Number(raw)
|
||||||
|
if (!Number.isInteger(value) || value < 1) {
|
||||||
|
throw new Error(`${name} must be a positive integer, got ${JSON.stringify(raw)}`)
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
const snapshotMaxConcurrency = positiveIntFromEnv(
|
||||||
|
'DSH_SNAPSHOT_MAX_CONCURRENCY',
|
||||||
|
Math.min(DEFAULT_SNAPSHOT_MAX_CONCURRENCY, availableParallelism()),
|
||||||
|
)
|
||||||
|
|
||||||
// Replay is the keyless default: boot the real ACP subprocess from recorded model scripts and diff
|
// Replay is the keyless default: boot the real ACP subprocess from recorded model scripts and diff
|
||||||
// normalized transcript plus persisted-log goldens. `record` calls the real API and updates fixtures
|
// normalized transcript plus persisted-log goldens. `record` calls the real API and updates fixtures
|
||||||
// and goldens; `refresh` replays committed scripts and updates only current goldens. Replay/refresh
|
// and goldens; `refresh` replays committed scripts and updates only current goldens. Replay/refresh
|
||||||
@@ -21,10 +40,12 @@ export default defineConfig({
|
|||||||
plugins: [tsconfigPaths({ projects: ['./tsconfig.json'] })],
|
plugins: [tsconfigPaths({ projects: ['./tsconfig.json'] })],
|
||||||
test: {
|
test: {
|
||||||
include: ['examples/*/tests/**/*.snapshot.ts', 'packages/sdk/*/tests/**/*.snapshot.ts'],
|
include: ['examples/*/tests/**/*.snapshot.ts', 'packages/sdk/*/tests/**/*.snapshot.ts'],
|
||||||
// Each test boots a subprocess; give it room, and run files one at a time
|
// Each test boots a subprocess; give it room and keep the worker file singular. Replay tests
|
||||||
// (a record run hits the live API, and replay subprocess boot is heavy).
|
// opt into bounded in-file concurrency, while record/refresh stay serial because they write
|
||||||
|
// fixtures. The environment knob restores serial replay with value 1 on constrained machines.
|
||||||
testTimeout: 120_000,
|
testTimeout: 120_000,
|
||||||
hookTimeout: 30_000,
|
hookTimeout: 30_000,
|
||||||
fileParallelism: false,
|
fileParallelism: false,
|
||||||
|
maxConcurrency: snapshotMaxConcurrency,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|||||||
Reference in New Issue
Block a user