From f75aabcb1640f5cab926ded982eda28bb399d1e5 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 17:10:30 +0800 Subject: [PATCH] test(ci): run required benchmarks on standard hosted Linux --- ...04-session-open-performance-gate.i18n.yaml | 4 +- ...026-09-04-session-open-performance-gate.md | 2 +- ...-09-04-session-open-performance-gate.zh.md | 2 +- ...standard-hosted-benchmark-runner.i18n.yaml | 6 +++ ...-09-06-standard-hosted-benchmark-runner.md | 26 +++++++++++ ...-06-standard-hosted-benchmark-runner.zh.md | 26 +++++++++++ .github/workflows/ci.yml | 11 ++--- docs/development.i18n.yaml | 4 +- docs/development.md | 2 +- docs/development.zh.md | 2 +- scripts/ci-workflow.spec.ts | 43 ++++++++++++++++++- 11 files changed, 111 insertions(+), 17 deletions(-) create mode 100644 .agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.i18n.yaml create mode 100644 .agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.md create mode 100644 .agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.zh.md diff --git a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.i18n.yaml b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.i18n.yaml index 395f28c0f0..ab8b3c01b9 100644 --- a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md -2026-09-04-session-open-performance-gate.md: 30e65eb52dea48426cf87cf91053a9133ab42763 -2026-09-04-session-open-performance-gate.zh.md: 3e202c966ea4764e135c79e1d238af707dca1f8d +2026-09-04-session-open-performance-gate.md: b866ea4bea513500ee2477c328c78081cc87c70d +2026-09-04-session-open-performance-gate.zh.md: 42929bdb2ff6686137ef8fb2ec0c0bc6344d0e93 diff --git a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md index 30e65eb52d..b866ea4bea 100644 --- a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md +++ b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md @@ -12,7 +12,7 @@ Measuring only `SessionPersistence.open()` does not stably describe the result f ## Decision -Linux pull requests run a required `node 24 / benchmarks` job that executes `pnpm run check:ci:bench` → `pnpm run test:bench`. The private `@deepseek-ai/dsh-benchmarks` workspace owns benchmark-only dependencies. The command first builds workspace libraries and dedicated workers under `benchmarks/.dsh-build/`, then invokes `vitest.bench.config.ts`. The job runs the benchmark lane alone; Vitest runs one file at a time and only prepares input, starts measurement children, aggregates results, and enforces budgets. Every timed CPU path executes compiled JavaScript under plain Node with `NODE_OPTIONS` removed and no TypeScript loader; bare workspace imports therefore resolve from `benchmarks/node_modules` through package exports to built `lib/` entries. +Linux pull requests run a required `node 24 / benchmarks` job that executes `pnpm run check:ci:bench` → `pnpm run test:bench`. The private `@deepseek-ai/dsh-benchmarks` workspace owns benchmark-only dependencies. The command first builds workspace libraries and dedicated workers under `benchmarks/.dsh-build/`, then invokes `vitest.bench.config.ts`. The [standard hosted runner decision](2026-09-06-standard-hosted-benchmark-runner.md) owns runner selection and the outer job timeout. The job runs the benchmark lane alone; Vitest runs one file at a time and only prepares input, starts measurement children, aggregates results, and enforces budgets. Every timed CPU path executes compiled JavaScript under plain Node with `NODE_OPTIONS` removed and no TypeScript loader; bare workspace imports therefore resolve from `benchmarks/node_modules` through package exports to built `lib/` entries. Required performance gates live under top-level `benchmarks/`, grouped by measured user path rather than package ownership. Host files use `*.bench.ts`, Client-face files use `*.bench.client.ts`, and scenario-specific workers and fixtures stay beside their benchmark without a benchmark suffix. Package-local `.perf.ts` files remain non-gating diagnostics; `scripts/` owns orchestration rather than benchmark cases. diff --git a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.zh.md b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.zh.md index 3e202c966e..42929bdb2f 100644 --- a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.zh.md +++ b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.zh.md @@ -12,7 +12,7 @@ Session format v2 的推出改变了两条成本随模型输出增长的路径 ## 决定 -Linux pull request 运行必需的 `node 24 / benchmarks` job，执行 `pnpm run check:ci:bench` → `pnpm run test:bench`。私有 `@deepseek-ai/dsh-benchmarks` workspace 拥有 benchmark 专属依赖。该命令先构建 workspace library 和 `benchmarks/.dsh-build/` 下的专用 worker，再调用 `vitest.bench.config.ts`。该 job 单独运行 benchmark lane；Vitest 逐文件运行，只负责准备输入、启动测量子进程、汇总结果和执行预算断言。每条被计时的 CPU 路径都以纯 Node 执行编译后的 JavaScript，并移除 `NODE_OPTIONS` 且不加载 TypeScript runtime；workspace 裸导入因此从 `benchmarks/node_modules` 通过 package exports 解析到构建后的 `lib/` 入口。 +Linux pull request 运行必需的 `node 24 / benchmarks` job，执行 `pnpm run check:ci:bench` → `pnpm run test:bench`。私有 `@deepseek-ai/dsh-benchmarks` workspace 拥有 benchmark 专属依赖。该命令先构建 workspace library 和 `benchmarks/.dsh-build/` 下的专用 worker，再调用 `vitest.bench.config.ts`。[标准托管运行器决策](2026-09-06-standard-hosted-benchmark-runner.zh.md)拥有运行器选择及外层 job 超时。该 job 单独运行 benchmark lane；Vitest 逐文件运行，只负责准备输入、启动测量子进程、汇总结果和执行预算断言。每条被计时的 CPU 路径都以纯 Node 执行编译后的 JavaScript，并移除 `NODE_OPTIONS` 且不加载 TypeScript runtime；workspace 裸导入因此从 `benchmarks/node_modules` 通过 package exports 解析到构建后的 `lib/` 入口。 必需性能 gate 位于顶层 `benchmarks/`，按被测用户路径而非 package 归属组织。Host 文件使用 `*.bench.ts`，Client 面文件使用 `*.bench.client.ts`，场景专属 worker 与 fixture 留在对应 benchmark 旁且不带 benchmark 后缀。包内 `.perf.ts` 文件仍是非门禁诊断；`scripts/` 负责编排而不承载 benchmark case。 diff --git a/.agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.i18n.yaml new file mode 100644 index 0000000000..d4269b9c2e --- /dev/null +++ b/.agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.md +2026-09-06-standard-hosted-benchmark-runner.md: 9cad59db21f65e6fef1e18fe58faeb8b071657cd +2026-09-06-standard-hosted-benchmark-runner.zh.md: 3fc79c849d00465053482d43d0aa78ee743578d2 diff --git a/.agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.md b/.agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.md new file mode 100644 index 0000000000..9cad59db21 --- /dev/null +++ b/.agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.md @@ -0,0 +1,26 @@ +# Agent Note: Standard hosted runner for required benchmarks + +Status: implemented + +English | [中文](2026-09-06-standard-hosted-benchmark-runner.zh.md) + +## Problem + +Wall-clock performance checks need an isolated execution lane and a consistent runner class. Routing them through the enterprise Linux failover switch makes their measurements depend on either larger hosted capacity or a shared self-hosted VM, while also consuming capacity needed by parallel correctness checks. + +## Decision + +The required benchmark job in [ci.yml](../../../../.github/workflows/ci.yml) uses the standard GitHub-hosted `ubuntu-24.04` runner independently of Linux failover. It always attempts to restore the pnpm store cache and retains a standalone benchmark lane. The complete job has a 15-minute timeout covering setup, installation, builds, and measurements. This bounds infrastructure execution, not an individual performance assertion. + +The [Session performance decision](2026-09-04-session-open-performance-gate.md) continues to own workloads, timing and memory budgets, worker isolation, and calibration. Runner selection does not relax those budgets or the worker, test, and hook deadlines. Successful raw measurements remain in the Actions log through step-local `DSH_GATE_VERBOSE=1`. The hardware-comparison workflows retain their deliberately different runner sizes. + +## Alternatives considered + +- Enterprise or shared self-hosted routing retains more build capacity but ties the measurement environment to unrelated failover operations. +- Increasing performance thresholds together with the job timeout conflates a bounded CI execution with a regression allowance. Threshold changes require measured calibration and positive and negative controls. + +## Consequences + +A standard runner trades parallel build capacity for a fixed measurement class without removing the required verdict. Cache misses and runner variation can still affect total duration. Each runner change needs an actual hosted benchmark run before its job timeout is treated as validated; local workflow assertions alone cannot establish execution time. + +The owning [workflow tests](../../../../scripts/ci-workflow.spec.ts) pin runner routing, unconditional cache restoration, required status, and the job timeout. Negative controls reject failover routing, a cache condition, and the former 30-minute job bound. diff --git a/.agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.zh.md b/.agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.zh.md new file mode 100644 index 0000000000..3fc79c849d --- /dev/null +++ b/.agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.zh.md @@ -0,0 +1,26 @@ +# Agent Note: 必需 benchmark 使用标准托管运行器 + +Status: implemented + +[English](2026-09-06-standard-hosted-benchmark-runner.md) | 中文 + +## 问题 + +壁钟性能检查需要独立执行的 lane 和一致的运行器类别。通过企业 Linux 故障转移开关路由这些检查，会让测量取决于大型托管运行器或共享自托管虚拟机，同时占用并行正确性检查所需的容量。 + +## 决定 + +[ci.yml](../../../../.github/workflows/ci.yml) 中的必需 benchmark job 使用标准 GitHub 托管 `ubuntu-24.04` 运行器，不受 Linux 故障转移影响。它始终尝试恢复 pnpm 存储缓存，并保留独立的 benchmark lane。整个 job 的超时为 15 分钟，覆盖准备、安装、构建和测量。这限制的是基础设施执行时间，而非单项性能断言。 + +[Session 性能决策](2026-09-04-session-open-performance-gate.zh.md) 继续拥有工作负载、时间和内存预算、worker 隔离及校准。运行器选择不放宽这些预算，也不放宽 worker、测试和钩子的截止时间。步骤级 `DSH_GATE_VERBOSE=1` 使成功运行的原始测量保留在 Actions 日志中。硬件比较工作流保留有意设置的不同运行器规格。 + +## 考虑过的替代方案 + +- 企业或共享自托管路由保留更多构建容量，但使测量环境受无关故障转移操作影响。 +- 同时提高性能阈值和 job 超时，会混淆有界 CI 执行与退化容许量。阈值调整需要实测校准及正反例。 + +## 后果 + +标准运行器以并行构建容量换取固定测量类别，不移除必需判定。缓存未命中和运行器波动仍会影响总耗时。每次更换运行器都需要实际托管 benchmark 运行，才能认定 job 超时经过验证；本地工作流断言无法单独证明执行耗时。 + +所属[工作流测试](../../../../scripts/ci-workflow.spec.ts) 固定运行器路由、无条件缓存恢复、必需状态及 job 超时。反例验证拒绝故障转移路由、缓存条件和原来的 30 分钟 job 上限。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d21df3c02a..c55aa4e58f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -158,15 +158,11 @@ jobs: node-24-bench: if: github.event_name == 'pull_request' - runs-on: >- - ${{ vars.DSH_CI_FAILOVER_LINUX == 'selfhosted' - && github.event.pull_request.user.login != 'dependabot[bot]' - && fromJSON('["self-hosted", "linux", "x64", "vm-backup"]') - || 'dsh-ubuntu-24-04-16core' }} + runs-on: ubuntu-24.04 name: node 24 / benchmarks # Wall-clock budgets need an otherwise idle runner, so this job runs the # benchmark lane alone instead of joining a concurrent gate aggregate. - timeout-minutes: 30 + timeout-minutes: 15 steps: - uses: actions/checkout@v6 with: @@ -189,7 +185,6 @@ jobs: echo "path=$store_path" >> "$GITHUB_OUTPUT" - uses: actions/cache/restore@v4 - if: vars.DSH_CI_FAILOVER_LINUX != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]' with: path: ${{ steps.pnpm-store.outputs.path }} key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} @@ -200,6 +195,8 @@ jobs: run: pnpm install --frozen-lockfile - name: Run performance benchmarks + env: + DSH_GATE_VERBOSE: '1' run: pnpm run check:ci:bench node-24-consumers: diff --git a/docs/development.i18n.yaml b/docs/development.i18n.yaml index 6c7316250a..71b047fa7a 100644 --- a/docs/development.i18n.yaml +++ b/docs/development.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/development.md -development.md: a57c99d606a73cb938f339e080ab0ba05913902a -development.zh.md: 5439fec59cb7c245d655393690bf6c848cdf20fa +development.md: 37028720ff2487a81caecb8b2e2c6e5bc26b2df5 +development.zh.md: 01e3e88605cb4f9ea0b14df4a0099f436c48c5e9 diff --git a/docs/development.md b/docs/development.md index a57c99d606..37028720ff 100644 --- a/docs/development.md +++ b/docs/development.md @@ -120,7 +120,7 @@ Contributors can opt into the comprehensive local gate set with `pnpm run check: ### CI gates -The keyless [CI workflow](../.github/workflows/ci.yml) groups independent gates into broad lanes and runs a smaller compatibility signal across supported Node versions. Artifact consumers wait for one build within their lane. The separate real-API workflow runs `pnpm run test:e2e` with its configured worker bound. See [scripts/run-gates.ts](../scripts/run-gates.ts) and the workflow files for the current gate and job inventory. +The keyless [CI workflow](../.github/workflows/ci.yml) groups independent gates into broad lanes and runs a smaller compatibility signal across supported Node versions. Artifact consumers wait for one build within their lane. Required benchmarks run separately on standard GitHub-hosted Linux; the [benchmark runner decision](../.agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.md) owns routing and the job timeout. The separate real-API workflow runs `pnpm run test:e2e` with its configured worker bound. See [scripts/run-gates.ts](../scripts/run-gates.ts) and the workflow files for the current gate and job inventory. The credential-free dsh dependency-layout and dsh/vendor pack rehearsals use the existing Linux self-hosted pool only when `DSH_CI_FAILOVER_LINUX=selfhosted` and the event is a trusted master push or same-repository, non-fork, non-Dependabot pull request. All other cases, including manual dispatch, use `ubuntu-24.04`; manual publication stays hosted. See the [release rehearsal runner decision](../.agents/notes/implemented/process/2026-09-06-release-rehearsal-selfhosted.md) for persistent-store isolation and fallback limits. diff --git a/docs/development.zh.md b/docs/development.zh.md index 5439fec59c..01e3e88605 100644 --- a/docs/development.zh.md +++ b/docs/development.zh.md @@ -124,7 +124,7 @@ vendor manifest 守卫检查 `vendor/*/src` 下的改动是否连同对应的 `v ### CI 门禁 -keyless [CI 工作流](../.github/workflows/ci.yml) 将独立门禁分组到若干宽粒度 lane，并在受支持的 Node 版本上运行一组较小的兼容性检查。产物消费方在各自 lane 内等待一次 build。单独的真实 API 工作流按其配置的 worker 上限运行 `pnpm run test:e2e`。当前门禁和 job 清单以 [scripts/run-gates.ts](../scripts/run-gates.ts) 和工作流文件为准。 +keyless [CI 工作流](../.github/workflows/ci.yml) 将独立门禁分组到若干宽粒度 lane，并在受支持的 Node 版本上运行一组较小的兼容性检查。产物消费方在各自 lane 内等待一次 build。必需 benchmark 在标准 GitHub 托管 Linux 上独立运行；[benchmark 运行器决策](../.agents/notes/implemented/testing/2026-09-06-standard-hosted-benchmark-runner.zh.md)拥有路由及 job 超时。单独的真实 API 工作流按其配置的 worker 上限运行 `pnpm run test:e2e`。当前门禁和 job 清单以 [scripts/run-gates.ts](../scripts/run-gates.ts) 和工作流文件为准。 不带凭据的 dsh 依赖布局检查与 dsh/vendor 打包演练仅在 `DSH_CI_FAILOVER_LINUX=selfhosted`，且事件为受信任的 master 推送或同仓库、非 fork、非 Dependabot 拉取请求时使用现有 Linux 自托管池。其余情况（包括手动触发）均使用 `ubuntu-24.04`；手动发布仍使用托管运行器。持久化存储隔离与回退限制见[发布演练运行器决策](../.agents/notes/implemented/process/2026-09-06-release-rehearsal-selfhosted.zh.md)。 diff --git a/scripts/ci-workflow.spec.ts b/scripts/ci-workflow.spec.ts index 421c72b0d5..eb06172527 100644 --- a/scripts/ci-workflow.spec.ts +++ b/scripts/ci-workflow.spec.ts @@ -235,10 +235,10 @@ describe('CI workflow', () => { expect(aggregate.needs).not.toContain('windows-observational') expect(aggregate.needs).not.toContain('serial-windows') - // Linux failover is a separate switch: the four required Linux workers + // Linux failover is a separate switch: the three enterprise Linux workers // and the verdict job resolve their pool through DSH_CI_FAILOVER_LINUX, // never the Windows switch. - for (const [jobName, job] of [['node-24', node24], ['node-24-coverage', node24Coverage], ['node-24-bench', node24Bench], ['node-24-consumers', node24Consumers]] as const) { + for (const [jobName, job] of [['node-24', node24], ['node-24-coverage', node24Coverage], ['node-24-consumers', node24Consumers]] as const) { expect(typeof job['runs-on']).toBe('string') expect(job['runs-on'], `${jobName} runs-on must use the Linux failover switch`).toContain('DSH_CI_FAILOVER_LINUX') expect(job['runs-on'], `${jobName} runs-on must not use the Windows failover switch`).not.toContain('DSH_CI_FAILOVER_WINDOWS') @@ -269,6 +269,45 @@ describe('CI workflow', () => { expect(windowsObservational.env).not.toMatchObject({ DSH_GATE_FAIL_FAST: '1' }) }) + it('runs required benchmarks on standard hosted Linux independently of failover', () => { + const workflow = loadWorkflow('.github/workflows/ci.yml') + const benchmark = workflowJob(workflow, 'node-24-bench') + const aggregate = workflowJob(workflow, 'all-checks-passed') + + expect(benchmark['runs-on']).toBe('ubuntu-24.04') + expect(benchmark.if).toBe("github.event_name == 'pull_request'") + expect(benchmark.needs).toBeUndefined() + expect(benchmark['continue-on-error']).toBeUndefined() + expect(benchmark.env).toBeUndefined() + expect(aggregate.needs).toContain('node-24-bench') + }) + + it('always restores the hosted benchmark pnpm cache', () => { + const benchmark = workflowJob(loadWorkflow('.github/workflows/ci.yml'), 'node-24-bench') + if (!Array.isArray(benchmark.steps)) throw new TypeError('benchmark job must define steps') + const caches = benchmark.steps.filter(step => isRecord(step) && step.uses === 'actions/cache/restore@v4') + + expect(caches).toHaveLength(1) + expect(caches[0]).not.toHaveProperty('if') + expect(caches[0]).toMatchObject({ + with: { + path: '${{ steps.pnpm-store.outputs.path }}', + key: "${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}", + }, + }) + }) + + it('bounds the complete benchmark job to fifteen minutes', () => { + const benchmark = workflowJob(loadWorkflow('.github/workflows/ci.yml'), 'node-24-bench') + + expect(benchmark['timeout-minutes']).toBe(15) + expect(benchmark.steps).toContainEqual({ + name: 'Run performance benchmarks', + env: { DSH_GATE_VERBOSE: '1' }, + run: 'pnpm run check:ci:bench', + }) + }) + it('gives the Wine Host TypeScript compile the repository heap budget', () => { const wineGates = readFileSync(resolve(root, 'scripts/wine-windows-gates.sh'), 'utf8')