From 10b31043ab02ce68d7ca0a2db2c59c6206598d8d Mon Sep 17 00:00:00 2001 From: Chinesezjc Date: Wed, 26 Aug 2026 17:40:28 +0800 Subject: [PATCH] feat(ci): assign coverage partitions by recorded file duration Replace Vitest's hash-based --shard with a coordinator-side longest-processing-time assignment. The coordinator collects the instrumented inventory from a vitest list run (dropping the exempt heavy suites that list does not exclude), reads per-file durations from the Vitest results cache, and seeds heavy subprocess-bound suites into different partitions. A weight-aware test fails when assignment ignores recorded weights, verified by injecting a file-count-only rule. Windows coverage measured partition spread of 442s (275-717s) under hash sharding; a simulation with the same file durations and the new assignment balances partitions to within 21s, cutting the critical partition to roughly half. --- ...8-18-in-job-partitioned-coverage.i18n.yaml | 4 +- .../2026-08-18-in-job-partitioned-coverage.md | 4 +- ...26-08-18-in-job-partitioned-coverage.zh.md | 4 +- .github/workflows/ci.yml | 22 + .gitignore | 1 + scripts/coverage-partitions.spec.ts | 254 ++++++++++- scripts/coverage-partitions.ts | 412 +++++++++++++++++- 7 files changed, 684 insertions(+), 17 deletions(-) diff --git a/.agents/notes/implemented/process/2026-08-18-in-job-partitioned-coverage.i18n.yaml b/.agents/notes/implemented/process/2026-08-18-in-job-partitioned-coverage.i18n.yaml index 70a774f23b..619841a551 100644 --- a/.agents/notes/implemented/process/2026-08-18-in-job-partitioned-coverage.i18n.yaml +++ b/.agents/notes/implemented/process/2026-08-18-in-job-partitioned-coverage.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/process/2026-08-18-in-job-partitioned-coverage.md -2026-08-18-in-job-partitioned-coverage.md: 9b8e6308ec746ee9a31465e57f58457130178b85 -2026-08-18-in-job-partitioned-coverage.zh.md: cf4d50996d9ecd4cb14ee762df11ade4aa63dc31 +2026-08-18-in-job-partitioned-coverage.md: 33824f0aa6f3541df8ca2e0cc8c417b40b5d7933 +2026-08-18-in-job-partitioned-coverage.zh.md: c32738462f7e33d9377d314a9ded5f82cebe9db3 diff --git a/.agents/notes/implemented/process/2026-08-18-in-job-partitioned-coverage.md b/.agents/notes/implemented/process/2026-08-18-in-job-partitioned-coverage.md index 9b8e6308ec..33824f0aa6 100644 --- a/.agents/notes/implemented/process/2026-08-18-in-job-partitioned-coverage.md +++ b/.agents/notes/implemented/process/2026-08-18-in-job-partitioned-coverage.md @@ -14,7 +14,7 @@ The optimization must retain every test and the merged per-file 100% thresholds. The ordinary `pnpm run test:coverage` command remains one Vitest invocation. Linux coverage CI fixes `DSH_COVERAGE_PARTITIONS=4`; native Windows now also fixes it at 4 to reduce process-creation pressure under high self-hosted concurrency. No elapsed-time trigger changes either count while a run is in progress. The [coverage-exempt heavy suite](2026-07-31-coverage-exempt-heavy-suites.md) remains a separate uninstrumented gate beside the instrumented work. -When partitioning is enabled, `scripts/run-gates.ts` selects `pnpm run test:coverage:partitioned` for the instrumented gate. `scripts/coverage-partitions.ts` starts the configured Vitest children concurrently, each with one worker and one `--shard=/` option. Partition mode suppresses thresholds and coverage reporters in each child, gives every child a separate report directory, and writes one blob report per process. +When partitioning is enabled, `scripts/run-gates.ts` selects `pnpm run test:coverage:partitioned` for the instrumented gate. `scripts/coverage-partitions.ts` starts the configured Vitest children concurrently, each with one worker. The coordinator collects the instrumented inventory from a `vitest list --filesOnly` run (caller filters narrow it; exempt heavy suites are removed because list does not apply their exclusion), reads recorded per-file durations from a coordinator-persisted gitignored file (restored and saved through the GitHub cache on the windows-coverage job, because checkout removes it and Vitest's own cache never survives CI), and assigns files to partitions by longest-processing-time by way of a min-heap, so the heavy subprocess-bound suites spread across children instead of piling into whichever shard a path hash lands them in. Each partition receives a temporary Vitest config whose include is its file list per project (command-line files exceeded the Windows CreateProcess limit; the mutually exclusive thread-safe and process-bound projects keep only their own files so nothing runs twice), an empty partition is rejected before any child starts, the heaviest partition starts first so its verdict lands earliest (fail-fast), and the duration history is restored and saved through the GitHub cache with per-run keys (cache entries are immutable). Partition mode suppresses thresholds and coverage reporters in each child, gives every child a separate report directory, and writes one blob report per process. The coordinator waits for every child, validates that the blob directory contains exactly the expected files, and then runs one `vitest --merge-reports ... --coverage` command. Only that merged command applies the repository's per-file statement, branch, function, and line thresholds, so a partition is never judged against an intentionally partial inventory. @@ -28,7 +28,7 @@ A normal failed test still emits a blob through `--coverage.reportOnFailure`, al ## Verification -`scripts/coverage-partitions.spec.ts` pins argument construction, package-script separator removal, one-worker partitions, the single merged threshold command, failed-test merging, failure diagnostics before complete-blob validation, waiting for sibling partitions after a spawn failure, and link-safe cleanup. `scripts/run-gates.spec.ts` pins opt-in selection, invalid-count rejection, both native Windows coverage gates' complete-build dependency, the complete Windows inventory with its blocking split, and unbuffered streamed output. React fake-timer cases that can move between partitions advance timers inside `act()`; geometry-dependent portal tests stub their element rectangles so a different shard schedule cannot turn deferred updates or jsdom coordinates into coverage-only failures. +`scripts/coverage-partitions.spec.ts` pins argument construction, package-script separator removal, one-worker partitions, weighted longest-processing-time assignment (including a case that fails when assignment ignores recorded weights), the single merged threshold command, failed-test merging, failure diagnostics before complete-blob validation, waiting for sibling partitions after a spawn failure, and link-safe cleanup. `scripts/run-gates.spec.ts` pins opt-in selection, invalid-count rejection, both native Windows coverage gates' complete-build dependency, the complete Windows inventory with its blocking split, and unbuffered streamed output. React fake-timer cases that can move between partitions advance timers inside `act()`; geometry-dependent portal tests stub their element rectangles so a different shard schedule cannot turn deferred updates or jsdom coordinates into coverage-only failures. Completed native Windows comparisons measured two partitions near 405 seconds and sixteen partitions at 112.66–122.01 seconds under the earlier gate ordering; those values compare partition latency, not the current peak. The current post-build phase runs four instrumented partition processes beside two exempt workers, for six coverage execution units. Sixteen partitions would raise that phase to eighteen before any still-running production-site work or system overhead. Four partitions keep separate-process isolation and match Linux, at the cost of a longer single-job coverage wall time; the trade-off is accepted to reduce vitest worker startup failures under high self-hosted concurrency. Two Linux samples measured the conservative two-partition configuration at 276.68 and 282.27 seconds; that configuration was stable but halved the ordinary path's four instrumented workers. Four partitions restore that fan-out, for six total coverage execution units on the 16-core hosted runner and at most 36 across the failover VM's six runner instances. These values come from completed runs or fixed capacity bounds; an unfinished run crossing an arbitrary elapsed-time mark is not evidence for increasing concurrency. diff --git a/.agents/notes/implemented/process/2026-08-18-in-job-partitioned-coverage.zh.md b/.agents/notes/implemented/process/2026-08-18-in-job-partitioned-coverage.zh.md index cf4d50996d..c32738462f 100644 --- a/.agents/notes/implemented/process/2026-08-18-in-job-partitioned-coverage.zh.md +++ b/.agents/notes/implemented/process/2026-08-18-in-job-partitioned-coverage.zh.md @@ -14,7 +14,7 @@ Status: implemented 普通的 `pnpm run test:coverage` 命令仍只启动一次 Vitest。Linux 覆盖率 CI 将 `DSH_COVERAGE_PARTITIONS` 固定为 4；原生 Windows 现在也固定为 4，以降低自托管高并发下的进程创建压力。运行期间不会由任何耗时触发器改变这两个数量。[覆盖率豁免重型套件](2026-07-31-coverage-exempt-heavy-suites.zh.md)仍作为独立的无插桩门禁与插桩工作并排运行。 -启用分区后，`scripts/run-gates.ts` 为插桩门禁选择 `pnpm run test:coverage:partitioned`。`scripts/coverage-partitions.ts` 按配置数量并发启动 Vitest 子进程，每个进程只用 1 个 worker，并各自接收一个 `--shard=/` 选项。分区模式会在各子进程中关闭阈值与覆盖率报告器，为每个子进程分配独立报告目录，并让每个进程写出 1 份 blob 报告。 +启用分区后，`scripts/run-gates.ts` 为插桩门禁选择 `pnpm run test:coverage:partitioned`。`scripts/coverage-partitions.ts` 按配置数量并发启动 Vitest 子进程，每个进程只用 1 个 worker。协调器通过 `vitest list --filesOnly` 收集插桩清单（调用方过滤器会先收窄清单；exempt 重型套件需在此剔除，因为 list 不应用其排除），从协调器持久化的 gitignore 文件读取逐文件耗时（windows-coverage job 通过 GitHub cache 恢复并保存该文件，因为 checkout 会删除它且 Vitest 自身缓存无法在 CI 存活），并借最小堆按最长处理时间把文件分配到各分区，使重量级子进程密集型套件分散到不同子进程，而不是全部落入路径 hash 恰好命中的那一个分片。每个分区获得一个临时 Vitest 配置，其 include 按 project 拆分（命令行传文件会超过 Windows CreateProcess 上限；互斥的 thread-safe 与 process-bound project 只保留各自的文件，避免任何文件跑两次）；空分区会在任何子进程启动前被拒绝，最重的分区最先启动使其结论最早落地（fail-fast），耗时历史通过 GitHub cache 以每 run 唯一键恢复与保存（cache 条目不可变）。分区模式会在各子进程中关闭阈值与覆盖率报告器，为每个子进程分配独立报告目录，并让每个进程写出 1 份 blob 报告。 协调器等待全部子进程结束，验证 blob 目录只包含预期文件，然后执行一次 `vitest --merge-reports ... --coverage`。只有这条合并命令应用仓库的逐文件语句、分支、函数与行阈值，因此系统不会拿有意不完整的测试清单单独判定任一分区。 @@ -28,7 +28,7 @@ Status: implemented ## 验证 -`scripts/coverage-partitions.spec.ts` 固定了参数构造、包脚本分隔符移除、单 worker 分区、唯一一次合并阈值命令、失败测试合并、完整 blob 校验前的失败诊断、spawn 失败后等待兄弟分区，以及链接安全清理。`scripts/run-gates.spec.ts` 固定了显式启用、非法数量拒绝、两道原生 Windows 覆盖率门禁对完整构建的依赖、完整 Windows 清单及其阻断性划分，以及不缓冲的流式输出。可能在分区间移动的 React fake-timer 用例会在 `act()` 内推进计时器；依赖几何位置的 portal 测试会固定元素矩形，使不同分片调度不会把延迟更新或 jsdom 坐标变成只在覆盖率运行中出现的失败。 +`scripts/coverage-partitions.spec.ts` 固定了参数构造、包脚本分隔符移除、单 worker 分区、加权最长处理时间分配（含一个在分配忽略记录权重时必然失败的用例）、唯一一次合并阈值命令、失败测试合并、完整 blob 校验前的失败诊断、spawn 失败后等待兄弟分区，以及链接安全清理。`scripts/run-gates.spec.ts` 固定了显式启用、非法数量拒绝、两道原生 Windows 覆盖率门禁对完整构建的依赖、完整 Windows 清单及其阻断性划分，以及不缓冲的流式输出。可能在分区间移动的 React fake-timer 用例会在 `act()` 内推进计时器；依赖几何位置的 portal 测试会固定元素矩形，使不同分片调度不会把延迟更新或 jsdom 坐标变成只在覆盖率运行中出现的失败。 已完成的原生 Windows 对比中，双分区耗时约 405 秒，16 分区耗时 112.66–122.01 秒；这些数据来自先前的门禁顺序，只用于比较分区延迟，不代表当前峰值。当前的构建后阶段会让 4 个插桩分区进程与 2 个豁免 worker 并行，共形成 6 个覆盖率执行单元。若改为 16 个分区，则在尚未结束的生产网站工作或系统开销计入之前，该阶段就会达到 18 个执行单元。4 个分区保留独立进程隔离并与 Linux 对齐，代价是单 job 覆盖率墙钟更长；这是为了降低自托管高并发下 vitest worker 启动失败而接受的取舍。两个 Linux 样本中，保守的双分区配置耗时 276.68 秒和 282.27 秒；该配置运行稳定，却把普通路径原有的 4 个插桩 worker 减半。4 个分区恢复这份并发，使 16 核托管 runner 上的覆盖率执行单元总数为 6，故障切换虚拟机的 6 个 runner 实例最多合计 36 个执行单元。这些数值来自完整运行或固定容量上限；运行尚未结束时跨过任意耗时刻度，不构成增加并发的证据。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20fa5fc99c..a0c8e681b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -459,6 +459,19 @@ jobs: - uses: actions/checkout@v6 with: persist-credentials: false + # Checkout removes the gitignored duration history, so restore it from + # the GitHub cache before coverage and save the updated one afterwards. + # The coordinator then weights partitions by measured durations on the + # second run instead of degrading to a file-count split. GitHub cache + # entries are immutable, so the save key is unique per run and the + # restore matches the newest entry through the stable prefix. + - name: Restore coverage duration history + uses: actions/cache/restore@v4 + with: + path: .coverage-times.json + key: coverage-times-${{ github.run_id }} + restore-keys: | + coverage-times- - name: Enable Developer Mode (symlink support) shell: pwsh run: >- @@ -483,6 +496,15 @@ jobs: - name: Run Windows coverage shell: pwsh run: pnpm run check:ci:coverage + - name: Save coverage duration history + # Coverage flakes must not prevent the cache from building; the + # measured durations remain useful even when a partition failed. The + # per-run key keeps every save a fresh immutable cache entry. + if: always() + uses: actions/cache/save@v4 + with: + path: .coverage-times.json + key: coverage-times-${{ github.run_id }} windows-native-tests: if: github.event_name == 'pull_request' diff --git a/.gitignore b/.gitignore index d33bbc8a4e..e2a11e6bc7 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ examples/*/*.jsonl .sessions/ examples/*/.sessions/ coverage/ +.coverage-times.json .doc-typecheck-*/ .node-next-types-*/ .oxlint-contract-*/ diff --git a/scripts/coverage-partitions.spec.ts b/scripts/coverage-partitions.spec.ts index 264a2686c4..6c6e31baa2 100644 --- a/scripts/coverage-partitions.spec.ts +++ b/scripts/coverage-partitions.spec.ts @@ -1,4 +1,4 @@ -import { access, mkdir, mkdtemp, symlink, writeFile } from 'node:fs/promises' +import { access, mkdir, mkdtemp, readFile, symlink, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' @@ -7,11 +7,17 @@ import { COVERAGE_PARTITIONS_ENV, COVERAGE_TEST_TIMEOUT_ENV, CoveragePartitionCoordinator, + assignWeightedPartitions, + collectPartitionDurations, coverageTestTimeoutArgs, forwardedCoverageArgs, parseCoveragePartitionCount, + parseListOutput, + readFileDurations, + writeFileDurations, type CoverageCommand, type CoverageCommandResult, + type CoveragePartitionCoordinatorOptions, } from './coverage-partitions.ts' const passed: CoverageCommandResult = { exitCode: 0, signalCode: null } @@ -28,6 +34,37 @@ async function temporaryRoot(): Promise { return await mkdtemp(join(tmpdir(), 'dsh-coverage-partitions-')) } +/** Write a Vitest results cache under a temporary root. */ +async function writeVitestCache(root: string, entries: Array<[string, { duration: number }]>): Promise { + const cacheDir = join(root, 'node_modules/.vite/vitest/cache-hash') + await mkdir(cacheDir, { recursive: true }) + await writeFile(join(cacheDir, 'results.json'), JSON.stringify({ version: '4.1.8', results: entries })) +} + +/** Run the coordinator and capture every partition config's source. */ +async function runCoordinatorReadingConfigs( + root: string, + options: Omit, +): Promise> { + const configContents = new Map() + const runCommand = vi.fn(async (command: CoverageCommand) => { + const configArgument = command.args.find(argument => argument.startsWith('--config=')) + if (configArgument !== undefined) { + configContents.set(command.label, await readFile(join(root, configArgument.slice('--config='.length)), 'utf8')) + } + await writeBlob(command) + return passed + }) + const coordinator = new CoveragePartitionCoordinator({ + root, + pnpmEntrypoint: '/pnpm.cjs', + runCommand, + ...options, + }) + await expect(coordinator.run()).resolves.toBe(0) + return configContents +} + function successfulCommandRecorder(commands: CoverageCommand[]) { return vi.fn(async (command: CoverageCommand) => { commands.push(command) @@ -81,16 +118,162 @@ describe('coverage forwarded arguments', () => { }) }) +describe('weighted partition assignment', () => { + it('seeds the heaviest files into different partitions', () => { + const weights = new Map([ + ['packages/a/tests/heavy-1.spec.ts', 100], + ['packages/a/tests/heavy-2.spec.ts', 90], + ['packages/a/tests/heavy-3.spec.ts', 80], + ['packages/a/tests/light.spec.ts', 1], + ]) + const buckets = assignWeightedPartitions([...weights.keys()], weights, 3) + expect(buckets).toHaveLength(3) + for (const bucket of buckets) { + expect(bucket).not.toHaveLength(0) + expect(bucket.filter(file => file.includes('heavy'))).toHaveLength(1) + } + }) + + it('balances total weight across partitions', () => { + const files = Array.from({ length: 20 }, (_, index) => `packages/a/tests/file-${index}.spec.ts`) + const weights = new Map(files.map((file, index) => [file, (index % 7) + 1])) + const buckets = assignWeightedPartitions(files, weights, 4) + const sums = buckets.map(bucket => bucket.reduce((sum, file) => sum + (weights.get(file) ?? 0), 0)) + const spread = Math.max(...sums) - Math.min(...sums) + expect(spread).toBeLessThanOrEqual(7) + }) + + it('steers assignment by weight, not by file count', () => { + // Weight-aware LPT balances three buckets to 1500 each. A file-count-only + // rule pairs the heaviest file with the fourth (1700), so the assertion + // only passes when recorded weights steer the assignment. + const weights = new Map([ + ['a.spec.ts', 1000], + ['b.spec.ts', 900], + ['c.spec.ts', 800], + ['d.spec.ts', 700], + ['e.spec.ts', 600], + ['f.spec.ts', 500], + ]) + const buckets = assignWeightedPartitions([...weights.keys()], weights, 3) + const sums = buckets.map(bucket => bucket.reduce((sum, file) => sum + (weights.get(file) ?? 0), 0)) + expect(Math.max(...sums)).toBeLessThanOrEqual(1550) + }) + + it('leaves trailing empty buckets when files are scarce', () => { + const files = ['a.spec.ts', 'b.spec.ts'] + const buckets = assignWeightedPartitions(files, new Map(), 4) + expect(buckets.map(bucket => bucket.length).sort()).toEqual([0, 0, 1, 1]) + }) + + it('returns one empty bucket per partition for an empty inventory', () => { + expect(assignWeightedPartitions([], new Map(), 3)).toEqual([[], [], []]) + }) + + it('assigns unknown-weight files evenly', () => { + const files = ['a.spec.ts', 'b.spec.ts', 'c.spec.ts', 'd.spec.ts'] + const buckets = assignWeightedPartitions(files, new Map(), 2) + expect(buckets.map(bucket => bucket.length).sort()).toEqual([2, 2]) + }) +}) + +describe('coverage file inventory', () => { + it('parses vitest list --filesOnly output, keeps project ownership, and drops exempt suites', async () => { + const root = await temporaryRoot() + const exemptDir = join(root, 'packages/experimental/webworker-runtime/tests') + await mkdir(exemptDir, { recursive: true }) + await writeFile(join(exemptDir, 'transform-corpus.spec.ts'), '') + const output = [ + '[thread-safe] packages/a/tests/a.spec.ts', + '[process-bound] packages/b/tests/b.spec.ts', + '[thread-safe] packages/experimental/webworker-runtime/tests/transform-corpus.spec.ts', + 'not a test line', + ].join('\n') + const inventory = parseListOutput(output, root) + expect(inventory.files).toEqual([ + 'packages/a/tests/a.spec.ts', + 'packages/b/tests/b.spec.ts', + ]) + expect(inventory.projectOf.get('packages/a/tests/a.spec.ts')).toBe('thread-safe') + expect(inventory.projectOf.get('packages/b/tests/b.spec.ts')).toBe('process-bound') + }) + + it('averages recorded durations per file from the results cache', async () => { + const root = await temporaryRoot() + await writeVitestCache(root, [ + ['thread-safe:packages/a/tests/x.spec.ts', { duration: 10 }], + ['process-bound:packages/a/tests/x.spec.ts', { duration: 30 }], + ['thread-safe:packages/a/tests/y.spec.ts', { duration: 5 }], + ]) + const durations = readFileDurations(root) + expect(durations.get('packages/a/tests/x.spec.ts')).toBe(20) + expect(durations.get('packages/a/tests/y.spec.ts')).toBe(5) + }) + + it('prefers the persisted duration file over the vitest cache', async () => { + const root = await temporaryRoot() + await writeVitestCache(root, [['thread-safe:packages/a/tests/x.spec.ts', { duration: 100 }]]) + writeFileDurations(root, new Map([['packages/a/tests/x.spec.ts', 42]])) + expect(readFileDurations(root).get('packages/a/tests/x.spec.ts')).toBe(42) + }) + + it('merges new durations into the persisted file', async () => { + const root = await temporaryRoot() + writeFileDurations(root, new Map([['packages/a/tests/x.spec.ts', 42]])) + writeFileDurations(root, new Map([ + ['packages/a/tests/x.spec.ts', 55], + ['packages/a/tests/y.spec.ts', 7], + ])) + const durations = readFileDurations(root) + expect(durations.get('packages/a/tests/x.spec.ts')).toBe(55) + expect(durations.get('packages/a/tests/y.spec.ts')).toBe(7) + }) + + it('extracts per-file durations from partition json reports', async () => { + const root = await temporaryRoot() + const report = join(root, 'partition-1.report.json') + await writeFile(report, JSON.stringify({ + testResults: [ + { name: join(root, 'packages/a/tests/x.spec.ts'), startTime: 1000, endTime: 1500 }, + { name: 'not-a-spec', startTime: 1, endTime: 2 }, + ], + })) + const durations = collectPartitionDurations([report], root) + expect(durations.get('packages/a/tests/x.spec.ts')).toBe(500) + }) +}) + describe('coverage partition coordinator', () => { + const weightedFiles = ['a.spec.ts', 'b.spec.ts', 'c.spec.ts'] + const weightedDurations = new Map([ + ['a.spec.ts', 100], + ['b.spec.ts', 50], + ['c.spec.ts', 10], + ]) + const weightedProjects = new Map([ + ['a.spec.ts', 'thread-safe'], + ['b.spec.ts', 'process-bound'], + ['c.spec.ts', 'process-bound'], + ]) it('runs every single-worker partition before one merged threshold check', async () => { const root = await temporaryRoot() const commands: CoverageCommand[] = [] - const runCommand = successfulCommandRecorder(commands) + const partitionConfigs: string[] = [] + const runCommand = vi.fn(async (command: CoverageCommand) => { + commands.push(command) + const configArgument = command.args.find(argument => argument.startsWith('--config=')) + if (configArgument !== undefined) { + partitionConfigs.push(await readFile(join(root, configArgument.slice('--config='.length)), 'utf8')) + } + await writeBlob(command) + return passed + }) const coordinator = new CoveragePartitionCoordinator({ root, partitions: 3, pnpmEntrypoint: '/pnpm.cjs', vitestArgs: ['--testTimeout=30000'], + files: ['a.spec.ts', 'b.spec.ts', 'c.spec.ts'], runCommand, }) @@ -102,23 +285,35 @@ describe('coverage partition coordinator', () => { 'partition 3/3', 'merged coverage report', ]) - for (const [index, command] of commands.slice(0, 3).entries()) { + for (const command of commands.slice(0, 3)) { expect(command.command).toBe(process.execPath) expect(command.args[0]).toBe('/pnpm.cjs') expect(command.args).toEqual(expect.arrayContaining([ '--coverage', '--coverage.reportOnFailure', '--maxWorkers=1', - `--shard=${index + 1}/3`, '--reporter=default', '--reporter=blob', + '--reporter=json', '--testTimeout=30000', ])) + expect(command.args).not.toContain('--shard=1/3') + expect(command.args.some(argument => argument.startsWith('--config='))).toBe(true) expect(command.env).toEqual({ [COVERAGE_PARTITIONS_ENV]: undefined, [COVERAGE_PARTITION_MODE_ENV]: '1', }) } + // The partition file list travels in a temporary config, not on the + // command line (which exceeds the Windows CreateProcess limit). + expect(partitionConfigs).toHaveLength(3) + const allConfigs = partitionConfigs.join('\n') + expect(allConfigs).toContain('a.spec.ts') + expect(allConfigs).toContain('b.spec.ts') + expect(allConfigs).toContain('c.spec.ts') + for (const source of partitionConfigs) { + expect(source).toContain("from '../../vitest.config.ts'") + } const mergeCommand = commands[3] if (mergeCommand === undefined) throw new Error('coverage merge command was not observed') expect(mergeCommand.args).toContain('--coverage') @@ -129,6 +324,51 @@ describe('coverage partition coordinator', () => { }) }) + it('rejects an empty partition assignment before spawning any command', async () => { + const root = await temporaryRoot() + const runCommand = vi.fn() + const coordinator = new CoveragePartitionCoordinator({ + root, + partitions: 3, + pnpmEntrypoint: '/pnpm.cjs', + // One file for three partitions leaves two buckets empty; an empty + // bucket would make Vitest run the whole suite. + files: ['a.spec.ts'], + runCommand, + }) + + await expect(coordinator.run()).rejects.toThrow('partition 2/3 has no files') + expect(runCommand).not.toHaveBeenCalled() + }) + + it('starts the heaviest partition first for fail-fast', async () => { + const root = await temporaryRoot() + const configContents = await runCoordinatorReadingConfigs(root, { + partitions: 2, + files: weightedFiles, + weights: weightedDurations, + }) + // LPT: p1=[a] sum 100, p2=[b,c] sum 60; sorted heaviest-first makes + // partition 1/2 the heavy one, so its config names a.spec.ts. + expect(configContents.get('partition 1/2')).toContain('a.spec.ts') + expect(configContents.get('partition 1/2')).not.toContain('b.spec.ts') + }) + + it('gives each project only its own files in the partition config', async () => { + const root = await temporaryRoot() + const configContents = await runCoordinatorReadingConfigs(root, { + partitions: 2, + files: weightedFiles, + weights: weightedDurations, + projectOf: weightedProjects, + }) + const allConfigs = [...configContents.values()].join('\n') + // The process-bound project must not receive the thread-safe file and + // vice versa, or plain files would run twice. + expect(allConfigs).toContain("include: project.test.name === 'process-bound' ?") + expect(allConfigs).not.toContain('"a.spec.ts","b.spec.ts","c.spec.ts"') + }) + it('runs a native pnpm entrypoint directly', async () => { const root = await temporaryRoot() const commands: CoverageCommand[] = [] @@ -137,6 +377,7 @@ describe('coverage partition coordinator', () => { root, partitions: 2, pnpmEntrypoint: '/tools/pnpm', + files: ['a.spec.ts', 'b.spec.ts'], runCommand, }) @@ -161,6 +402,7 @@ describe('coverage partition coordinator', () => { root, partitions: 2, pnpmEntrypoint: '/pnpm.cjs', + files: ['a.spec.ts', 'b.spec.ts'], runCommand, }) @@ -182,6 +424,7 @@ describe('coverage partition coordinator', () => { root, partitions: 2, pnpmEntrypoint: '/pnpm.cjs', + files: ['a.spec.ts', 'b.spec.ts'], runCommand, }) @@ -202,6 +445,7 @@ describe('coverage partition coordinator', () => { root, partitions: 2, pnpmEntrypoint: '/pnpm.cjs', + files: ['a.spec.ts', 'b.spec.ts'], runCommand, }) @@ -225,6 +469,7 @@ describe('coverage partition coordinator', () => { root, partitions: 2, pnpmEntrypoint: '/pnpm.cjs', + files: ['a.spec.ts', 'b.spec.ts'], runCommand, }) @@ -248,6 +493,7 @@ describe('coverage partition coordinator', () => { root, partitions: 2, pnpmEntrypoint: '/pnpm.cjs', + files: ['a.spec.ts', 'b.spec.ts'], runCommand, }) diff --git a/scripts/coverage-partitions.ts b/scripts/coverage-partitions.ts index fed0706f5e..86d919115d 100644 --- a/scripts/coverage-partitions.ts +++ b/scripts/coverage-partitions.ts @@ -1,7 +1,9 @@ /** Coordinate single-worker Vitest coverage partitions and one merged report. */ import { spawn } from 'node:child_process' -import { lstat, mkdir, readdir, rm, unlink } from 'node:fs/promises' +import { globSync, readFileSync, writeFileSync } from 'node:fs' +import { lstat, mkdir, readdir, rm, unlink, writeFile } from 'node:fs/promises' import { join, relative, sep } from 'node:path' +import { coverageExemptHeavySuites } from './coverage-exempt.ts' import { pnpmInvocation } from './pnpm-invocation.ts' /** Environment variable selecting the number of instrumented coverage processes. */ @@ -56,6 +58,12 @@ export interface CoveragePartitionCoordinatorOptions { vitestArgs?: string[] /** Child executor, injectable for scheduler tests. */ runCommand?: CoverageCommandRunner + /** Instrumented inventory; collected from the workspace when absent or empty. */ + files?: readonly string[] + /** Recorded durations paired with `files`; read from persistence when absent. */ + weights?: ReadonlyMap + /** Project ownership paired with `files`; collected from `vitest list` when absent. */ + projectOf?: ReadonlyMap } /** Parse an optional coverage partition count. */ @@ -83,6 +91,316 @@ export function forwardedCoverageArgs(args: readonly string[]): string[] { return [...args.slice(args[0] === '--' ? 1 : 0)] } +/** + * Weight assigned to a file with no recorded duration. One millisecond keeps + * the LPT assignment purely duration-driven once history exists, while a + * first run (no cache at all) degrades to an even file-count split. + */ +const UNKNOWN_FILE_WEIGHT = 1 + +/** + * Coordinator-maintained duration history. CI removes `node_modules/.vite` + * on every checkout, so Vitest's own cache never survives there; this + * gitignored file at the repository root carries recorded durations across + * runs on a persistent checkout (self-hosted runners). + */ +const FILE_TIMES_NAME = '.coverage-times.json' + +/** + * The instrumented inventory: every file plus the Vitest project it belongs + * to (`thread-safe` or `process-bound`). Preserving the per-project split + * matters because the projects are mutually exclusive: a file's own project + * must run it exactly once, so a partition config cannot hand the whole + * partition list to every project. + */ +export interface InstrumentedInventory { + files: string[] + /** Project name per file; the pool prefix of the `vitest list` line. */ + projectOf: Map +} + +/** + * Parse `vitest list --filesOnly` output into the instrumented inventory: + * one `[pool] path` line per file, deduplicated, minus the exempt heavy + * suites that `vitest list` itself does not exclude. + */ +export function parseListOutput(output: string, root: string): InstrumentedInventory { + const files = new Set() + const projectOf = new Map() + for (const line of output.split(/\r?\n/)) { + const match = /^\[([^\]]+)\]\s+(\S+\.spec\.(?:ts|tsx))$/.exec(line) + if (match !== null && match[1] !== undefined && match[2] !== undefined) { + files.add(match[2]) + projectOf.set(match[2], match[1]) + } + } + for (const suite of coverageExemptHeavySuites) { + for (const file of globSync(suite.exclude, { cwd: root })) { + // globSync returns platform separators on Windows; the parsed inventory + // and Vitest include patterns both use forward slashes. + const normalized = file.split('\\').join('/') + files.delete(normalized) + projectOf.delete(normalized) + } + } + return { files: [...files].sort(), projectOf } +} + +/** + * Collect the instrumented coverage inventory from a `vitest list --filesOnly` + * run: no test collection and no worker pool, just the file list. Caller + * filters (positional args after `--`) narrow the list before the exempt + * heavy suites are removed here because `vitest list` does not apply the + * `COVERAGE_EXEMPT_ENV` exclusion. + */ +async function collectInstrumentedFiles( + root: string, + pnpmEntrypoint: string, + filters: readonly string[] = [], +): Promise { + const invocation = pnpmInvocation(['exec', 'vitest', 'list', '--filesOnly', ...filters], { npm_execpath: pnpmEntrypoint }) + const output = await runListCommand(invocation.command, invocation.args, root) + return parseListOutput(output, root) +} + +/** Run `vitest list` and return its stdout, or throw with exit code and stderr. */ +function runListCommand(command: string, args: string[], root: string): Promise { + return new Promise((resolveList, rejectList) => { + const child = spawn(command, args, { cwd: root, env: process.env, stdio: ['ignore', 'pipe', 'pipe'] }) + let output = '' + let errorOutput = '' + child.stdout.setEncoding('utf8') + child.stderr.setEncoding('utf8') + child.stdout.on('data', (chunk: string) => { output += chunk }) + child.stderr.on('data', (chunk: string) => { errorOutput += chunk }) + child.once('error', (error: Error) => { rejectList(error) }) + child.once('close', (code) => { + if (code === 0) resolveList(output) + else rejectList(new Error(`vitest list exited with ${String(code ?? 'signal')}${errorOutput === '' ? '' : `: ${errorOutput.trim().slice(0, 300)}`}`)) + }) + }) +} + +/** + * Read recorded per-file durations: the coordinator's persisted file first + * (survives CI checkouts), falling back to the Vitest results cache for local + * development. Cache entries are `[projectName:relativePath, {duration}]`; + * a file appearing several times keeps the average duration. + */ +export function readFileDurations(root: string): Map { + const persisted = readPersistedDurations(root) + if (persisted.size > 0) return persisted + const totals = new Map() + for (const file of globSync('node_modules/.vite/vitest/*/results.json', { cwd: root })) { + let cache: { results?: Array<[string, { duration?: number }]> } + try { + cache = JSON.parse(readFileSync(join(root, file), 'utf8')) as { results?: Array<[string, { duration?: number }]> } + } catch { + continue + } + for (const [key, entry] of cache.results ?? []) { + const separator = key.indexOf(':') + if (separator < 0) continue + const path = key.slice(separator + 1) + const duration = entry.duration + if (typeof duration !== 'number') continue + const total = totals.get(path) + if (total === undefined) totals.set(path, { sum: duration, count: 1 }) + else { + total.sum += duration + total.count++ + } + } + } + return new Map([...totals].map(([path, { sum, count }]) => [path, sum / count])) +} + +/** Read the coordinator's persisted duration map; empty when absent or corrupt. */ +function readPersistedDurations(root: string): Map { + let raw: Record + try { + raw = JSON.parse(readFileSync(join(root, FILE_TIMES_NAME), 'utf8')) as Record + } catch { + return new Map() + } + const durations = new Map() + for (const [file, duration] of Object.entries(raw)) { + if (typeof duration === 'number' && Number.isFinite(duration)) durations.set(file, duration) + } + return durations +} + +/** + * Merge new durations into the persisted file and rewrite it. A fresh run's + * measurements overwrite earlier ones, so the history tracks the latest + * checkout's behavior; entries whose file no longer exists in the current + * inventory are dropped, so deleted or renamed specs never linger with stale + * weights. + */ +export function writeFileDurations( + root: string, + durations: ReadonlyMap, + currentFiles?: readonly string[], +): void { + if (durations.size === 0) return + const merged = new Map(readPersistedDurations(root)) + for (const [file, duration] of durations) merged.set(file, duration) + if (currentFiles !== undefined) { + const present = new Set(currentFiles) + for (const file of [...merged.keys()]) { + if (!present.has(file)) merged.delete(file) + } + } + writeFileSync(join(root, FILE_TIMES_NAME), `${JSON.stringify(Object.fromEntries(merged), null, 1)}\n`, 'utf8') +} + +/** + * Extract per-file durations from Vitest JSON reporter outputs (one per + * partition). Each `testResults` entry names an absolute spec path and carries + * `startTime`/`endTime`; the difference is the file's recorded duration. + */ +export function collectPartitionDurations(reportFiles: readonly string[], root: string): Map { + const durations = new Map() + for (const file of reportFiles) { + let report: { testResults?: Array<{ name?: unknown; startTime?: number; endTime?: number }> } + try { + report = JSON.parse(readFileSync(file, 'utf8')) as { testResults?: Array<{ name?: unknown; startTime?: number; endTime?: number }> } + } catch { + continue + } + for (const result of report.testResults ?? []) { + if (typeof result.name !== 'string' || typeof result.startTime !== 'number' || typeof result.endTime !== 'number') continue + const relativePath = relative(root, result.name).split(sep).join('/') + durations.set(relativePath, Math.max(0, result.endTime - result.startTime)) + } + } + return durations +} + +/** + * Assign files to partitions by longest-processing-time: heavier files are + * seeded first into the currently lightest partition, so recorded durations + * (and the import/environment cost that scales with a partition's file set) + * spread instead of piling into whichever shard the hash lands them in. A + * min-heap over the buckets keeps each placement at O(log partitions). + * @returns one file list per partition, every partition non-empty. + */ +export function assignWeightedPartitions( + files: readonly string[], + weights: ReadonlyMap, + partitions: number, +): string[][] { + if (files.length === 0) return Array.from({ length: partitions }, () => []) + const weighted = files + .map(file => ({ file, weight: weights.get(file) ?? UNKNOWN_FILE_WEIGHT })) + .sort((a, b) => b.weight - a.weight || a.file.localeCompare(b.file)) + const buckets = Array.from({ length: partitions }, () => ({ sum: 0, files: [] as string[] })) + // Min-heap of bucket indices ordered by (sum, file count); equal sums pick + // the leaner bucket so a duration-sparse inventory still balances file count. + const heap = buckets.map((_, index) => index) + for (const { file, weight } of weighted) { + const top = heap[0] + if (top === undefined) throw new Error('coverage partitions: partition heap index is out of bounds.') + const bucket = buckets[top] + if (bucket === undefined) throw new Error('coverage partitions: partition bucket is missing.') + bucket.sum += weight + bucket.files.push(file) + siftDown(buckets, heap, 0) + } + return buckets.map(bucket => bucket.files) +} + +/** Restore the min-heap property after the root bucket grew heavier. */ +function siftDown( + buckets: Array<{ sum: number; files: string[] }>, + heap: number[], + index: number, +): void { + const size = heap.length + for (;;) { + const left = 2 * index + 1 + const right = 2 * index + 2 + let smallest = index + let smallestBucket = bucketAt(buckets, heap, index) + const leftBucket = left < size ? bucketAt(buckets, heap, left) : undefined + const rightBucket = right < size ? bucketAt(buckets, heap, right) : undefined + if (leftBucket !== undefined && smallestBucket !== undefined && bucketLess(leftBucket, smallestBucket)) { + smallest = left + smallestBucket = leftBucket + } + if (rightBucket !== undefined && smallestBucket !== undefined && bucketLess(rightBucket, smallestBucket)) { + smallest = right + } + if (smallest === index) return + const moved = heap[index] + const replacement = heap[smallest] + if (moved === undefined || replacement === undefined) return + heap[index] = replacement + heap[smallest] = moved + index = smallest + } +} + +/** Read the bucket at a heap position, or undefined when the index is absent. */ +function bucketAt( + buckets: Array<{ sum: number; files: string[] }>, + heap: number[], + index: number, +): { sum: number; files: string[] } | undefined { + const bucketIndex = heap[index] + return bucketIndex === undefined ? undefined : buckets[bucketIndex] +} + +/** Order buckets by total weight, then by file count, then by nothing (stable). */ +function bucketLess( + left: { sum: number; files: string[] } | undefined, + right: { sum: number; files: string[] } | undefined, +): boolean { + if (left === undefined || right === undefined) return left !== undefined + if (left.sum !== right.sum) return left.sum < right.sum + return left.files.length < right.files.length +} + +/** Sum of a partition's file weights; unknown weights count as one. */ +function partitionWeight(files: readonly string[], weights: ReadonlyMap): number { + return files.reduce((sum, file) => sum + (weights.get(file) ?? UNKNOWN_FILE_WEIGHT), 0) +} + +/** + * Source of one partition's temporary Vitest config: the workspace config + * with `test.include` narrowed to the partition's file list, per project. + * The config sits under `coverage/.partitioned/`, so the workspace config is + * two directories up, and the partition processes run with cwd at the + * repository root (Vite resolves the relative include patterns against it). + * Each project keeps only the files that belong to it: the projects are + * mutually exclusive, so handing the whole partition list to every project + * would run plain files twice (once per project). + */ +function partitionConfigSource( + files: readonly string[], + projectOf: ReadonlyMap, +): string { + const threadSafe = JSON.stringify(files.filter(file => projectOf.get(file) !== 'process-bound').map(file => file.split('\\').join('/'))) + const processBound = JSON.stringify(files.filter(file => projectOf.get(file) === 'process-bound').map(file => file.split('\\').join('/'))) + return [ + "import base from '../../vitest.config.ts'", + 'export default {', + ' ...base,', + ' test: {', + ' ...base.test,', + ' projects: (base.test.projects ?? []).map(project => ({', + ' ...project,', + ' test: {', + ' ...project.test,', + ' include: project.test.name === \'process-bound\' ? ' + processBound + ' : ' + threadSafe + ',', + ' },', + ' })),', + ' },', + '}', + '', + ].join('\n') +} + /** Run instrumented partitions, validate their blobs, and merge once. */ export class CoveragePartitionCoordinator { private readonly root: string @@ -90,6 +408,9 @@ export class CoveragePartitionCoordinator { private readonly pnpmEntrypoint: string private readonly vitestArgs: string[] private readonly runCommand: CoverageCommandRunner + private readonly files: readonly string[] + private readonly weights: ReadonlyMap | undefined + private projectOf = new Map() private readonly temporaryRoot: string private readonly blobsRoot: string @@ -103,6 +424,9 @@ export class CoveragePartitionCoordinator { this.pnpmEntrypoint = options.pnpmEntrypoint this.vitestArgs = options.vitestArgs ?? [] this.runCommand = options.runCommand ?? runCoverageCommand + this.files = options.files ?? [] + this.weights = options.weights + this.projectOf = new Map(options.projectOf ?? []) this.temporaryRoot = join(this.root, 'coverage', '.partitioned') this.blobsRoot = join(this.temporaryRoot, 'blobs') } @@ -116,10 +440,10 @@ export class CoveragePartitionCoordinator { await mkdir(this.blobsRoot, { recursive: true }) try { - const commands = Array.from( - { length: this.partitions }, - (_, index) => this.partitionCommand(index + 1), - ) + const assignments = await this.assignFiles() + this.assertNonEmptyAssignments(assignments) + const configPaths = await this.writePartitionConfigs(assignments) + const commands = assignments.map((_, index) => this.partitionCommand(index + 1, configPaths[index] ?? '')) const results = await Promise.all(commands.map(async (command) => { console.log(`coverage-partitions: start ${command.label}`) const result = await this.runCommand(command) @@ -131,6 +455,9 @@ export class CoveragePartitionCoordinator { } return result })) + // Persist durations before blob validation: a missing blob aborts the + // run, but the completed partitions' timings are still worth keeping. + this.persistDurations(this.partitions) await this.assertCompleteBlobSet(commands) const mergeCommand = this.mergeCommand() @@ -142,9 +469,78 @@ export class CoveragePartitionCoordinator { } } - private partitionCommand(index: number): CoverageCommand { + /** + * Refuse an empty partition: Vitest treats a config with no matching files + * as "run everything", so an empty bucket would silently execute the whole + * suite once per empty partition. + */ + private assertNonEmptyAssignments(assignments: readonly (readonly string[])[]): void { + const empty = assignments.findIndex(files => files.length === 0) + if (empty >= 0) { + throw new Error( + `coverage partitions: partition ${empty + 1}/${this.partitions} has no files; ` + + 'the instrumented inventory is empty or smaller than the partition count.', + ) + } + } + + /** Persist measured per-file durations so the next run can weight by them. */ + private persistDurations(partitionCount: number): void { + const reportFiles = Array.from( + { length: partitionCount }, + (_, index) => join(this.temporaryRoot, `partition-${index + 1}.report.json`), + ) + const currentFiles = this.projectOf.size > 0 ? [...this.projectOf.keys()] : undefined + writeFileDurations(this.root, collectPartitionDurations(reportFiles, this.root), currentFiles) + } + + /** + * Distribute the instrumented inventory across partitions by recorded + * duration, heaviest partition first so the longest child starts earliest + * (fail-fast: its verdict, success or failure, lands before the light + * children settle). An injected file list skips workspace collection and + * cache reads (scheduler tests); production collection always runs. + */ + private async assignFiles(): Promise { + let files: readonly string[] + let weights: ReadonlyMap + if (this.files.length > 0) { + files = this.files + weights = this.weights ?? new Map() + } else { + // Positional filters live after the `--` separator; options and their + // values (`--testTimeout 5000`) must never be mistaken for filters. + const separator = this.vitestArgs.indexOf('--') + const filters = separator >= 0 ? this.vitestArgs.slice(separator + 1) : [] + const inventory = await collectInstrumentedFiles(this.root, this.pnpmEntrypoint, filters) + files = inventory.files + this.projectOf = inventory.projectOf + weights = readFileDurations(this.root) + } + const buckets = assignWeightedPartitions(files, weights, this.partitions) + buckets.sort((left, right) => partitionWeight(right, weights) - partitionWeight(left, weights)) + return buckets + } + + /** + * Write one temporary Vitest config per partition whose `include` (top level + * and per project) is the partition's file list. Passing files on the + * command line exceeded the Windows CreateProcess limit once a partition + * held a few hundred paths, so each partition instead points Vitest at a + * short `--config` path. + */ + private async writePartitionConfigs(assignments: readonly (readonly string[])[]): Promise { + return await Promise.all(assignments.map(async (files, index) => { + const configPath = join(this.temporaryRoot, `vitest-partition-${index + 1}.config.ts`) + await writeFile(configPath, partitionConfigSource(files, this.projectOf), 'utf8') + return configPath + })) + } + + private partitionCommand(index: number, configPath: string): CoverageCommand { const blobPath = join(this.blobsRoot, `partition-${index}.json`) const reportsDirectory = join(this.temporaryRoot, `coverage-${index}`) + const jsonReportPath = join(this.temporaryRoot, `partition-${index}.report.json`) const invocation = pnpmInvocation([ 'exec', 'vitest', @@ -152,10 +548,12 @@ export class CoveragePartitionCoordinator { '--coverage', '--coverage.reportOnFailure', '--maxWorkers=1', - `--shard=${index}/${this.partitions}`, + `--config=${this.relativePath(configPath)}`, '--reporter=default', '--reporter=blob', + '--reporter=json', `--outputFile.blob=${this.relativePath(blobPath)}`, + `--outputFile.json=${this.relativePath(jsonReportPath)}`, `--coverage.reportsDirectory=${this.relativePath(reportsDirectory)}`, ...this.vitestArgs, ], { npm_execpath: this.pnpmEntrypoint })