perf(snapshot): parallelize replay scenarios

This commit is contained in:
imccyu
2026-07-15 23:57:42 +08:00
parent 867d248c2e
commit a28d95afb2
2 changed files with 32 additions and 8 deletions
+9 -6
View File
@@ -3,6 +3,8 @@
* compares normalized stdout; comparable session fixtures are both replay input and expected * compares normalized stdout; comparable session fixtures are both replay input and expected
* output. Record mode refreshes reproducible model scenarios from the live API, while refresh * output. Record mode refreshes reproducible model scenarios from the live API, while refresh
* mode replays committed scripts and rewrites derived artifacts without a key. * mode replays committed scripts and rewrites derived artifacts without a key.
* Replay scenarios run concurrently because each subprocess owns unique temp cwd and persistence
* roots and only reads committed fixtures. Record and refresh scenarios stay serial while writing.
* *
* Exactly one scenario per header-composition class pins the system prompt and tool schemas in * Exactly one scenario per header-composition class pins the system prompt and tool schemas in
* dedicated sidecars. Every live header is checked against that pin, so session-dependent * dedicated sidecars. Every live header is checked against that pin, so session-dependent
@@ -461,7 +463,7 @@ export function stabilizeRefreshLog(fresh: string, existing: string, replacement
} }
/** /**
* Register the suite: one `describe` per scenario (the golden/log compares and * Register the suite: one test per scenario (the golden/log compares and
* the header-uniformity guard) plus the fixture guard block (no orphan * the header-uniformity guard) plus the fixture guard block (no orphan
* scenario dirs, required files present, exactly one pin per header class, * scenario dirs, required files present, exactly one pin per header class,
* pinning fixtures well-formed, every JSONL prompt-scrubbed, non-pinning * pinning fixtures well-formed, every JSONL prompt-scrubbed, non-pinning
@@ -477,6 +479,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
const RECORDING = mode === 'record' const RECORDING = mode === 'record'
const REFRESHING = mode === 'refresh' const REFRESHING = mode === 'refresh'
const childMode: 'replay' | 'record' = RECORDING ? 'record' : 'replay' const childMode: 'replay' | 'record' = RECORDING ? 'record' : 'replay'
const scenarioSuite = mode === 'replay' ? describe.concurrent : describe
/** The class a scenario's header composition belongs to (see {@link Scenario.headerClass}). */ /** The class a scenario's header composition belongs to (see {@link Scenario.headerClass}). */
const classOf = (scenario: Scenario): string => scenario.headerClass ?? 'default' const classOf = (scenario: Scenario): string => scenario.headerClass ?? 'default'
@@ -496,11 +499,11 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
} }
} }
for (const scenario of scenarios) { scenarioSuite('snapshot scenarios', () => {
describe(`snapshot: ${scenario.name}`, () => { for (const scenario of scenarios) {
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones // In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
// (sidecar-driven errors/cancel) are never re-recorded. // (sidecar-driven errors/cancel) are never re-recorded.
it.skipIf(RECORDING && !scenario.recorded)('matches the goldens', async () => { it.skipIf(RECORDING && !scenario.recorded)(`snapshot: ${scenario.name} matches the goldens`, async ({ expect }) => {
const dir = join(snapshotsDir, scenario.name) const dir = join(snapshotsDir, scenario.name)
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
const overrideFile = join(dir, 'replay.override.json') const overrideFile = join(dir, 'replay.override.json')
@@ -658,8 +661,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
} }
} }
}) })
}) }
} })
describe('snapshot fixtures', () => { describe('snapshot fixtures', () => {
it('every scenario directory is registered (no orphans)', async () => { it('every scenario directory is registered (no orphans)', async () => {
+23 -2
View File
@@ -1,6 +1,25 @@
import { availableParallelism } from 'node:os'
import tsconfigPaths from 'vite-tsconfig-paths' import tsconfigPaths from 'vite-tsconfig-paths'
import { defineConfig } from 'vitest/config' import { defineConfig } from 'vitest/config'
const DEFAULT_SNAPSHOT_MAX_CONCURRENCY = 5
function positiveIntFromEnv(name: string, fallback: number): number {
const raw = process.env[name]
if (raw === undefined || raw === '') return fallback
const value = Number(raw)
if (!Number.isInteger(value) || value < 1) {
throw new Error(`${name} must be a positive integer, got ${JSON.stringify(raw)}`)
}
return value
}
const snapshotMaxConcurrency = positiveIntFromEnv(
'DSH_SNAPSHOT_MAX_CONCURRENCY',
Math.min(DEFAULT_SNAPSHOT_MAX_CONCURRENCY, availableParallelism()),
)
// Replay is the keyless default: boot the real ACP subprocess from recorded model scripts and diff // Replay is the keyless default: boot the real ACP subprocess from recorded model scripts and diff
// normalized transcript plus persisted-log goldens. `record` calls the real API and updates fixtures // normalized transcript plus persisted-log goldens. `record` calls the real API and updates fixtures
// and goldens; `refresh` replays committed scripts and updates only current goldens. Replay/refresh // and goldens; `refresh` replays committed scripts and updates only current goldens. Replay/refresh
@@ -21,10 +40,12 @@ export default defineConfig({
plugins: [tsconfigPaths({ projects: ['./tsconfig.json'] })], plugins: [tsconfigPaths({ projects: ['./tsconfig.json'] })],
test: { test: {
include: ['examples/*/tests/**/*.snapshot.ts', 'packages/sdk/*/tests/**/*.snapshot.ts'], include: ['examples/*/tests/**/*.snapshot.ts', 'packages/sdk/*/tests/**/*.snapshot.ts'],
// Each test boots a subprocess; give it room, and run files one at a time // Each test boots a subprocess; give it room and keep the worker file singular. Replay tests
// (a record run hits the live API, and replay subprocess boot is heavy). // opt into bounded in-file concurrency, while record/refresh stay serial because they write
// fixtures. The environment knob restores serial replay with value 1 on constrained machines.
testTimeout: 120_000, testTimeout: 120_000,
hookTimeout: 30_000, hookTimeout: 30_000,
fileParallelism: false, fileParallelism: false,
maxConcurrency: snapshotMaxConcurrency,
}, },
}) })