From fa6bf62a981bd10d0f9ecd234954d4f74bde9bce Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 7 Sep 2026 13:24:04 +0800 Subject: [PATCH 01/22] fix(chat): settle pinned scroll deliveries before layout growth --- ...ed-scroll-delivery-before-layout.i18n.yaml | 6 + ...07-pinned-scroll-delivery-before-layout.md | 23 ++++ ...pinned-scroll-delivery-before-layout.zh.md | 23 ++++ apps/web/tests/chat-scroll-contract.e2e.ts | 1 + packages/client/ui-chat/README.i18n.yaml | 4 +- packages/client/ui-chat/README.md | 2 +- packages/client/ui-chat/README.zh.md | 2 +- .../ui-chat/src/client/chat/ChatView.tsx | 9 +- .../ui-chat/tests/chat-view.client.spec.tsx | 122 ++++++++++++++++++ 9 files changed, 186 insertions(+), 6 deletions(-) create mode 100644 .agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.i18n.yaml create mode 100644 .agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.md create mode 100644 .agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.zh.md diff --git a/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.i18n.yaml new file mode 100644 index 0000000000..ac6b884af9 --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.md +2026-09-07-pinned-scroll-delivery-before-layout.md: 7a00cf653824df13272fcf0cc2baf35b298f170c +2026-09-07-pinned-scroll-delivery-before-layout.zh.md: d78c388e4e7a8f6f2fa6070149e652e0e25cc358 diff --git a/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.md b/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.md new file mode 100644 index 0000000000..7a00cf6538 --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.md @@ -0,0 +1,23 @@ +# Agent Note: Settle pinned scroll deliveries before layout changes + +Status: implemented + +English | [中文](2026-09-07-pinned-scroll-delivery-before-layout.zh.md) + +## Problem + +A delayed scroll sample compares positions from different layouts. While Chat is pinned, a composer or transcript shrink can move the browser floor; subsequent growth can move the browser position again before `scrollend` or the sampling timer. Deferring follow during that interval leaves the observed-top ledger stale and can classify browser layout movement as reader input, disabling follow without a reader gesture. + +## Decision + +[ChatView](../../../../packages/client/ui-chat/src/client/chat/ChatView.tsx) samples pinned scroll deliveries synchronously through the same sample operation that clears pending work. This preserves the existing observed-top comparison for genuine reader movement and releases layout follow before further growth. Pinned samples use scroll metrics, not semantic-row geometry; moving away still disarms follow immediately. Away-reader samples remain coalesced at the existing interval or `scrollend`. + +## Alternatives considered + +**Defer every delivery.** Coalescing reduces geometry work while reading history, but a pinned browser position and its floor must be attributed in the same layout. A longer timeout or retry cannot recover ownership once the stale comparison disarms it. + +**Sample every delivery synchronously.** This restores attribution but also repeats semantic-anchor and reading-line measurements throughout an away-reader scroll burst. Only pinned ownership needs the immediate path. + +## Consequences + +Pinned deliveries incur immediate scroll-metric reads. History reading retains its bounded sampling cadence, and explicit return-to-bottom deliveries clear any pending away sample. [Focused tests](../../../../packages/client/ui-chat/tests/chat-view.client.spec.tsx) cover shrink/regrowth before scrollend, observer growth without row measurements, repinning with a pending sample, timer and scrollend sampling, and unmount cancellation. The [keyless browser scenario](../../../../apps/web/tests/chat-scroll-contract.e2e.ts) covers pinned Send, real scroll-away input, streaming, and tool disclosure across the long transcript. diff --git a/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.zh.md b/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.zh.md new file mode 100644 index 0000000000..d78c388e4e --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.zh.md @@ -0,0 +1,23 @@ +# Agent Note: 在布局变化前处理贴底滚动事件 + +Status: implemented + +[English](2026-09-07-pinned-scroll-delivery-before-layout.md) | 中文 + +## Problem + +延迟的滚动采样会比较来自不同布局的位置。Chat 贴底时，输入框或 transcript（文本记录）收缩可能改变浏览器底部位置；随后的增长又可能在 `scrollend` 或采样定时器触发前改变浏览器位置。在此期间推迟跟随会使已观察顶部位置记录过期，把浏览器布局移动误判为读者输入，在没有读者操作时关闭跟随。 + +## Decision + +[ChatView](../../../../packages/client/ui-chat/src/client/chat/ChatView.tsx) 通过同一个清除待处理工作的采样操作，同步采样贴底滚动事件。该机制保留现有的已观察顶部位置比较来识别真实读者移动，并在后续增长前恢复布局跟随。贴底采样只读取滚动指标，不读取语义行几何；离底移动仍会立即关闭跟随。离底读者的采样仍合并到现有周期或 `scrollend` 时执行。 + +## Alternatives considered + +**延迟所有事件。** 合并采样减少阅读历史时的几何计算，但贴底浏览器位置及其底部必须在同一布局中完成归因。过期比较关闭跟随后，延长超时或重试都无法恢复归属。 + +**同步采样所有事件。** 这能恢复归因，却也会在离底读者连续滚动时重复测量语义锚点和阅读线。只有贴底归属需要立即处理。 + +## Consequences + +贴底事件会立即读取滚动指标。历史阅读保留有界采样节奏，显式回到底部的滚动事件会清除任何待处理的离底采样。[聚焦测试](../../../../packages/client/ui-chat/tests/chat-view.client.spec.tsx) 覆盖 scrollend 前的收缩与增长、无需行测量的观察器增长、存在待处理采样时重新贴底、定时器与 scrollend 采样，以及卸载取消。[无密钥浏览器场景](../../../../apps/web/tests/chat-scroll-contract.e2e.ts) 覆盖长 transcript 中贴底发送、真实离底输入、流式输出与工具详情展开。 diff --git a/apps/web/tests/chat-scroll-contract.e2e.ts b/apps/web/tests/chat-scroll-contract.e2e.ts index 3dae7b93ff..d7ee3ae289 100644 --- a/apps/web/tests/chat-scroll-contract.e2e.ts +++ b/apps/web/tests/chat-scroll-contract.e2e.ts @@ -660,6 +660,7 @@ describe('web e2e: long Chat scroll contract', () => { await liveRow.waitFor({ timeout: 15_000 }) expect(await liveRow.getAttribute('data-state')).toBe('running') await expectBottom(world.page) + expect(await world.page.getByRole('button', { name: 'Back to bottom', exact: true }).count()).toBe(0) await wheelTranscript(world.page, -1_200) await world.page.getByRole('button', { name: 'Back to bottom', exact: true }).waitFor({ timeout: 10_000 }) diff --git a/packages/client/ui-chat/README.i18n.yaml b/packages/client/ui-chat/README.i18n.yaml index a2b157652e..80d26dca50 100644 --- a/packages/client/ui-chat/README.i18n.yaml +++ b/packages/client/ui-chat/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/client/ui-chat/README.md -README.md: 405e5095d75c3e8b752c028bbc187307a9647bb6 -README.zh.md: 7f57bc3480ea0186e89d12e8e443926906677742 +README.md: 34b66da24b35f8cfd4c26da9b759e7991c5cb856 +README.zh.md: a9b7c8fca3e8137950dd02cd5398c11f15344757 diff --git a/packages/client/ui-chat/README.md b/packages/client/ui-chat/README.md index 405e5095d7..34b66da24b 100644 --- a/packages/client/ui-chat/README.md +++ b/packages/client/ui-chat/README.md @@ -46,7 +46,7 @@ Settings → General exposes a persisted `Normal` / `Compact` conversation-displ ## Scroll ownership -Chat restores semantic anchors across history prepend and renderer remounts. While the reader is pinned to the floor, `ResizeObserver` follows the new floor and selects the latest loaded Turn without reading row geometry. Once the reader moves away, flow-height changes preserve the top position and the reading-line geometry selects the active Turn. Turn-rail previews paint above sticky Markdown code-block banners, while the rail frame remains inside the transcript band above the composer. +Chat restores semantic anchors across history prepend and renderer remounts. Pinned scroll deliveries update follow ownership immediately, before subsequent layout changes can invalidate their floor; away-reader anchor sampling remains coalesced until the sampling interval or `scrollend`. While the reader is pinned to the floor, `ResizeObserver` follows the new floor and selects the latest loaded Turn without reading row geometry. Once the reader moves away, flow-height changes preserve the top position and the reading-line geometry selects the active Turn. Turn-rail previews paint above sticky Markdown code-block banners, while the rail frame remains inside the transcript band above the composer. ----- diff --git a/packages/client/ui-chat/README.zh.md b/packages/client/ui-chat/README.zh.md index 7f57bc3480..a9b7c8fca3 100644 --- a/packages/client/ui-chat/README.zh.md +++ b/packages/client/ui-chat/README.zh.md @@ -46,7 +46,7 @@ Chat 会为非空的初始请求、显式消息序列起点、真实 system 字 ## 滚动归属 -Chat 会在历史前插与 renderer 重新挂载时恢复语义锚点。读者跟随底部时，`ResizeObserver` 追随新的底部，并且无需读取行几何就选中最后一个已加载 Turn；读者离开底部后，高度变化会保持顶部位置，再由阅读线几何选择活跃 Turn。轮次导航预览位于 Markdown 代码块粘性头栏上方，而导航外框始终处于 composer 上方的 transcript 区域内。 +Chat 会在历史前插与 renderer 重新挂载时恢复语义锚点。贴底滚动事件会立即更新跟随归属，避免后续布局变化使其底部位置失效；离底读者的锚点采样仍合并到采样周期或 `scrollend` 时执行。读者跟随底部时，`ResizeObserver` 追随新的底部，并且无需读取行几何就选中最后一个已加载 Turn；读者离开底部后，高度变化会保持顶部位置，再由阅读线几何选择活跃 Turn。轮次导航预览位于 Markdown 代码块粘性头栏上方，而导航外框始终处于 composer 上方的 transcript 区域内。 ----- diff --git a/packages/client/ui-chat/src/client/chat/ChatView.tsx b/packages/client/ui-chat/src/client/chat/ChatView.tsx index a8dc57d321..e8b4438266 100644 --- a/packages/client/ui-chat/src/client/chat/ChatView.tsx +++ b/packages/client/ui-chat/src/client/chat/ChatView.tsx @@ -579,8 +579,9 @@ export function ChatView({ scheduleActiveTurn() } - // Raw scroll events only schedule work. Geometry is sampled at most once - // per interval, with scrollend providing the final sample for a short burst. + // Pinned deliveries must settle before layout growth can invalidate their + // floor. Away-reader anchor geometry stays coalesced until the interval or + // scrollend; pinned samples read only scroll metrics unless the reader leaves. useEffect(() => { const local = listRef.current /* v8 ignore next -- ref-null guard: effect runs after the list node commits. */ @@ -597,6 +598,10 @@ export function ChatView({ } const onScroll = (): void => { scrollSamplePendingRef.current = true + if (atBottomRef.current) { + sample() + return + } sampleTimer ??= window.setTimeout(sample, SCROLL_SAMPLE_INTERVAL_MS) } el.addEventListener('scroll', onScroll, { passive: true }) diff --git a/packages/client/ui-chat/tests/chat-view.client.spec.tsx b/packages/client/ui-chat/tests/chat-view.client.spec.tsx index 2c8fe220ea..f6b7ec63a9 100644 --- a/packages/client/ui-chat/tests/chat-view.client.spec.tsx +++ b/packages/client/ui-chat/tests/chat-view.client.spec.tsx @@ -2387,6 +2387,128 @@ describe('ChatView', () => { expect(scroller.scrollTop).toBe(900) }) + it('keeps following when a shrink clamp regrows before scrollend', () => { + const h = makeHarness({ nodes: [user(1, 'q'), assistant(2, 'a')] }) + const view = render() + const scroller = view.container.querySelector('[class*="scroll"]') as HTMLDivElement + const metrics = installScrollMetrics(scroller, 1_000, 300) + scroller.scrollTop = 700 + fireEvent.scroll(scroller) + fireEvent(scroller, new Event('scrollend')) + + metrics.setLayout(800, 700) + fireEvent.scroll(scroller) + metrics.setHeight(962) + act(() => { h.setSession({ running: true }) }) + fireEvent(scroller, new Event('scrollend')) + + expect(scroller.scrollTop).toBe(662) + expect(view.queryByLabelText('回到底部')).toBeNull() + expect(h.chatScroll.read()).toBeNull() + }) + + it('settles pinned deliveries before observer growth without reading row geometry', () => { + let notify: (() => void) | undefined + class ResizeObserverStub { + constructor(callback: ResizeObserverCallback) { + notify = () => { callback([], this as unknown as ResizeObserver) } + } + + observe = vi.fn() + disconnect = vi.fn() + } + vi.stubGlobal('ResizeObserver', ResizeObserverStub) + const h = makeHarness({ nodes: [user(1, 'q'), assistant(2, 'a')] }) + const view = render() + const scroller = view.container.querySelector('[class*="scroll"]') as HTMLDivElement + const metrics = installScrollMetrics(scroller, 9_931, 300) + expect(notify).toBeDefined() + scroller.scrollTop = 9_631 + fireEvent.scroll(scroller) + fireEvent(scroller, new Event('scrollend')) + const rect = vi.spyOn(HTMLElement.prototype, 'getBoundingClientRect') + rect.mockClear() + try { + metrics.setLayout(9_918, 9_631) + fireEvent.scroll(scroller) + metrics.setHeight(10_013) + act(() => { notify?.() }) + expect(scroller.scrollTop).toBe(9_713) + fireEvent.scroll(scroller) + metrics.setHeight(10_093) + act(() => { notify?.() }) + expect(scroller.scrollTop).toBe(9_793) + expect(rect).not.toHaveBeenCalled() + expect(h.chatScroll.read()).toBeNull() + } finally { + rect.mockRestore() + } + }) + + it('clears an away sample when a back-to-bottom delivery restores pinned ownership', () => { + const h = makeHarness({ nodes: [user(1, 'q'), assistant(2, 'a')] }) + const view = render() + const scroller = view.container.querySelector('[class*="scroll"]') as HTMLDivElement + const metrics = installScrollMetrics(scroller, 1_000, 300) + scroller.scrollTop = 700 + fireEvent.scroll(scroller) + scroller.scrollTop = 500 + fireEvent.scroll(scroller) + fireEvent(scroller, new Event('scrollend')) + scroller.scrollTop = 400 + fireEvent.scroll(scroller) + fireEvent.click(view.getByLabelText('回到底部')) + fireEvent.scroll(scroller) + metrics.setHeight(1_200) + act(() => { h.setSession({ running: true }) }) + expect(scroller.scrollTop).toBe(900) + expect(h.chatScroll.read()).toBeNull() + }) + + it('samples away-reader geometry on the interval or scrollend and cancels it on unmount', () => { + vi.useFakeTimers() + try { + const h = makeHarness({ nodes: [user(1, 'q'), assistant(2, 'a')] }) + const view = render() + const scroller = view.container.querySelector('[class*="scroll"]') as HTMLDivElement + installScrollMetrics(scroller, 1_000, 300) + scroller.scrollTop = 700 + fireEvent.scroll(scroller) + scroller.scrollTop = 500 + fireEvent.scroll(scroller) + expect(view.getByLabelText('回到底部')).toBeTruthy() + const rect = vi.spyOn(HTMLElement.prototype, 'getBoundingClientRect') + try { + act(() => { vi.advanceTimersByTime(500) }) + rect.mockClear() + scroller.scrollTop = 400 + fireEvent.scroll(scroller) + scroller.scrollTop = 300 + fireEvent.scroll(scroller) + act(() => { vi.advanceTimersByTime(499) }) + expect(rect).not.toHaveBeenCalled() + act(() => { vi.advanceTimersByTime(1) }) + expect(rect).toHaveBeenCalled() + rect.mockClear() + scroller.scrollTop = 200 + fireEvent.scroll(scroller) + expect(rect).not.toHaveBeenCalled() + fireEvent(scroller, new Event('scrollend')) + expect(rect).toHaveBeenCalled() + scroller.scrollTop = 100 + fireEvent.scroll(scroller) + view.unmount() + rect.mockClear() + act(() => { vi.advanceTimersByTime(500) }) + expect(rect).not.toHaveBeenCalled() + } finally { + rect.mockRestore() + } + } finally { + vi.useRealTimers() + } + }) + it('uses the last delivered top when compositor scrolling precedes scroll delivery', () => { const h = makeHarness({ nodes: [user(1, 'q'), assistant(2, 'a')] }) const view = render() From c599ef87c458b28bfc7b15ee6e9e978e1a941677 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 14:48:09 +0800 Subject: [PATCH 02/22] test(perf): baseline tool-heavy backend workflows --- ...backend-continuation-performance.i18n.yaml | 6 + ...-09-06-backend-continuation-performance.md | 60 ++++++++ ...-06-backend-continuation-performance.zh.md | 60 ++++++++ .../agent-continuation/README.i18n.yaml | 6 + benchmarks/agent-continuation/README.md | 31 ++++ benchmarks/agent-continuation/README.zh.md | 31 ++++ .../agent-continuation.bench.ts | 86 +++++++++++ .../agent-continuation.worker.ts | 144 ++++++++++++++++++ .../child-catalog.worker.ts | 95 ++++++++++++ .../agent-continuation/profile-adapter.ts | 42 +++++ .../profile-continuation.worker.ts | 85 +++++++++++ benchmarks/agent-continuation/workload.ts | 110 +++++++++++++ benchmarks/package.json | 3 + benchmarks/tsdown.config.ts | 12 ++ pnpm-lock.yaml | 9 ++ 15 files changed, 780 insertions(+) create mode 100644 .agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml create mode 100644 .agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md create mode 100644 .agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md create mode 100644 benchmarks/agent-continuation/README.i18n.yaml create mode 100644 benchmarks/agent-continuation/README.md create mode 100644 benchmarks/agent-continuation/README.zh.md create mode 100644 benchmarks/agent-continuation/agent-continuation.bench.ts create mode 100644 benchmarks/agent-continuation/agent-continuation.worker.ts create mode 100644 benchmarks/agent-continuation/child-catalog.worker.ts create mode 100644 benchmarks/agent-continuation/profile-adapter.ts create mode 100644 benchmarks/agent-continuation/profile-continuation.worker.ts create mode 100644 benchmarks/agent-continuation/workload.ts diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml new file mode 100644 index 0000000000..d6f7815fa1 --- /dev/null +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md +2026-09-06-backend-continuation-performance.md: 4c18e440c98135f140c4957a7c5e7c73188c694c +2026-09-06-backend-continuation-performance.zh.md: 2cf1793faf39ab0c3bc10b1932db15e64961e8b8 diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md new file mode 100644 index 0000000000..4c18e440c9 --- /dev/null +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md @@ -0,0 +1,60 @@ +# Agent Note: Performance baselines for tool-heavy backend continuation + +Status: implemented + +English | [中文](2026-09-06-backend-continuation-performance.zh.md) + +## Problem + +Opening one Session does not measure the repeated cost of preparing model requests after a long tool conversation, executing another tool-heavy turn, or discovering multiple inactive fork children. The [Session-opening gate](2026-09-04-session-open-performance-gate.md) covers first history and activation but deliberately stops before new model work. Its text/reasoning workload also lacks historical tool-call arguments and large tool results. + +## Decision + +The [agent-continuation benchmark](../../../../benchmarks/agent-continuation/agent-continuation.bench.ts) adds three scenario groups, including a shipped-profile variant, without changing product implementations. They use current-generation Zstandard Sessions authored through production append, stream accumulation, and persistence APIs. A separate seed process creates the deterministic source before measurement; each sample copies that source into its private root and starts a fresh compiled plain-Node worker. No recorded Session, ambient repository, network, private Harness home, or deployed GUI supplies input. + +The shared history has 800 completed two-step turns, four tool calls per turn, and 2,048-character tool results: 13,600 events and 5,600 model messages. Each assistant reply carries reasoning, text, and compact streamed records; tool replies additionally carry fragmented arguments. Fixed timestamps and ids describe the seed. Live synthetic replies use the real loop's clocks and ids without overriding process globals. + +| Case | Timed operation | Endpoint | +|---|---|---| +| Request history | After unmeasured cold resume, deliver 40 sequential text-only turns over the tool-heavy history, then flush | Idle Agent with all 40 model requests completed; reports turn and final-flush time separately | +| Tool continuation | Cold resume, 20 sequential turns with eight parallel-safe synthetic tool calls and a final reply per turn, then flush | Idle Agent with 40 model requests and 160 completed tool executions; reports resume, turns, and final flush separately | +| Shipped SDK workflow | Launch built dsh with the sdk-minimal profile, deliver 100 sequential turns with eight real file-view calls per turn, then close the SDK | SDK receives 200 assistant messages and 800 successful file results; includes Loader boot, stdio JSON-RPC, persistence, and shutdown | +| Child catalog | List 16 inactive seeded fork children twice through the real subagent and Session query services | Two complete healthy catalogs with observations released; each child inherits 80 tool-heavy turns and owns its descriptor after the exact fork cut | + +The tool execution pipeline, request preparation, Session projections required by those services, persistence, and catalog observations remain production code. Only the model adapter and bounded tool body are synthetic. The adapter retains a request counter, not request objects, so the fixture cannot manufacture a growing retention cost. Sequential input means each idle interval belongs to the one request delivered by this worker; it does not generalize idle to a per-message completion API under concurrent input. + +Five samples report raw wall time, CPU user/system time, peak RSS, endpoint counts, and the minimum, median, and maximum total wall time. Budgets enforce the unrounded median. Continuation additionally measures retained heap against an initialized Host: two explicit GCs separated by an event-loop yield precede and follow the timed operation, while the idle Agent remains reachable. The measured delta therefore includes the resident historical Session and live additions, not just newly appended turns. GC and teardown are outside timing; flush is inside. Request-history retention starts after resume and is diagnostic only. Catalog peak RSS is diagnostic; no retained-heap budget claims to measure already-released child observations. + +The parent bounds every child to 60 seconds, checks timeout, signal, exit, and report independently, awaits process close, and removes private roots after failures. Context and Agent teardown run in finally blocks. Seed processes cannot warm the measured process's caches. Filesystem caches are not forcibly evicted: cold means a fresh process, not cold physical storage. + +## Calibration evidence + +The implementation reference is `925e012340f033f0521e802ba8569ce6dd7ef1ac` on Apple M4 Pro, macOS arm64, Node 24.19.0. Two exclusive five-sample runs use the same seed and no product optimization. Durations below are milliseconds; source expectations round above the observed run medians rather than imposing an unimplemented optimization target. + +| Case | Run 1 raw totals | Run 2 raw totals | Medians | Reference expectation | CI budget | +|---|---|---|---|---:|---:| +| Request history | 209.134, 210.333, 208.959, 236.355, 238.685 | 222.833, 213.911, 208.089, 211.494, 209.137 | 210.333 / 211.494 | 220 | 550 | +| Tool continuation | 358.953, 324.790, 318.861, 320.119, 322.896 | 324.280, 321.952, 340.409, 325.470, 324.312 | 322.896 / 324.312 | 340 | 850 | +| Child catalog | 318.730, 309.006, 311.404, 308.565, 310.105 | 308.670, 310.030, 280.086, 303.084, 284.829 | 310.105 / 303.084 | 320 | 800 | + +Continuation retains approximately 22.295 MiB; its source expectation is 23 MiB and its budget is 28.75 MiB. Time expectations use the existing [calibration helper](../../../../benchmarks/support/calibration.ts): 2× shared CI time scale and 1.25× variance headroom. Memory uses only 1.25× headroom. The scale is inherited from the existing lane's calibration, not a new Linux measurement of these cases; CI evidence remains necessary when runner characteristics change. Baseline budgets protect the measured implementation; tighter budgets belong with a measured behavior-preserving fix. + +A separate plain-Node request-history CPU profile attributes 132.876 ms of sampled self time to deepFreeze called by buildRequest during a 211.300 ms operation. This identifies repeated traversal of already-frozen history as a focused investigation target, not a proven optimization result. Catalog first/repeat timings remain separate because a second listing still reads body-bearing seeded children after observations are released. + +The shipped SDK variant completes 100 turns, 200 requests, and 800 real file reads. Its five-sample smoke totals are 1,521.773, 1,463.465, 1,689.701, 1,365.485, and 1,417.106 ms (median 1,463.465 ms); a full-suite repeat reports 1,596.183, 1,784.536, 2,120.082, 1,405.365, and 1,355.894 ms (median 1,596.183 ms). Its 1,700 ms reference expectation yields a 4,250 ms CI budget. The repeat also slows the unchanged service cases, so it is validation under variable host load rather than evidence to relax their exclusive calibration. The SDK process receives an allowlisted environment and private home/workspace. A 40-second deadline starts SDK shutdown; every path awaits the same memoized close promise before the outer worker’s 60-second deadline. Profile timing includes boot, all turns, and shutdown, reported separately; no parent-process CPU or heap metric is presented as server memory. The adapter does not serialize requests for an external model provider. + +## Alternatives considered + +**Repeat existing migration and first-open variants.** Rejected: those twelve cases already distinguish read-only preparation from writable publication. These cases use the current generation and begin or continue actual model work, or enumerate a corpus rather than open one Session. + +**Measure only deriveMessages.** Rejected: its incremental cache does not include complete request freezing, adapter dispatch, live append, or persistence. Actual sequential requests protect the cost the Agent pays per step. + +**Use only unseeded children with warm projection-cache rows.** Rejected: that path bypasses body observations and misses the exact inherited-cut requirement of fork children. The catalog intentionally omits the optional projection cache and reports the seeded fallback path; it does not characterize cache-hit discovery. + +**Apply an optimization and its desired budget together with the first measurements.** Rejected: a baseline-only layer remains independently mergeable and records the current workload before attribution or implementation changes. Source constants cannot be overridden by environment variables. + +## Consequences + +The lane adds four cases in three scenario groups and twenty measured workers, plus two seed processes. The integrated continuation case spans resume through completed model/tool work and durable flush. The shipped SDK workflow additionally includes profile boot, SDK transport, real file tools, and shutdown; only its model adapter is synthetic. It starts a fresh Session because the public SDK prompt API creates rather than resumes stored identities. Neither path includes network model latency, provider-specific request serialization, optional user plugins, compaction, failed tool results, images, cancellation, or browser rendering. Functional tests retain responsibility for event contents, immutable messages, tool semantics, fork lineage, and read-only versus writable side effects; endpoint counts prevent timing a skipped workload without duplicating those assertions. + +This note supplements, rather than supersedes, the Session-opening gate's isolation and calibration rationale. No existing active decision is retired. diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md new file mode 100644 index 0000000000..2cf1793faf --- /dev/null +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md @@ -0,0 +1,60 @@ +# Agent Note: 工具密集后端续聊的性能基线 + +Status: implemented + +[English](2026-09-06-backend-continuation-performance.md) | 中文 + +## 问题 + +打开一个 Session 不能衡量长工具对话后重复准备模型请求、执行更多工具密集轮次或发现多个非活动 fork 子会话的成本。[Session 打开门禁](2026-09-04-session-open-performance-gate.zh.md)覆盖首屏历史和激活，但有意停在新的模型工作开始前。它的文本与推理负载也不包含历史工具调用参数和大型工具结果。 + +## 决定 + +[agent-continuation 基准](../../../../benchmarks/agent-continuation/agent-continuation.bench.ts)增加三个场景组，包含一个已发布 profile 变体，不修改产品实现。它们通过生产追加、流累积和持久化 API 构造当前代际的 Zstandard Session。独立播种进程在测量前生成确定性源数据；每个样本将其复制到私有根目录，并启动新的已编译纯 Node worker。输入不来自录制 Session、环境仓库、网络、私有 Harness 主目录或已部署 GUI。 + +共享历史包含 800 个已完成的双步骤轮次，每轮四次工具调用，工具结果为 2,048 字符：共 13,600 个事件和 5,600 条模型消息。每条助手回复携带推理、文本和紧凑流记录；请求工具的回复还携带分片参数。播种数据使用固定时间戳和 id。实时合成回复使用真实循环的时钟和 id，不覆盖进程全局状态。 + +| 用例 | 计时操作 | 终点 | +|---|---|---| +| 请求历史 | 在不计时的冷恢复后，向工具密集历史顺序提交 40 个纯文本轮次，然后 flush | 空闲 Agent，已完成全部 40 次模型请求；分别报告轮次和最终 flush 时间 | +| 工具续聊 | 冷恢复，顺序执行 20 个轮次，每轮八次可安全并行的合成工具调用和一条最终回复，然后 flush | 空闲 Agent，已完成 40 次模型请求和 160 次工具执行；分别报告恢复、轮次和最终 flush 时间 | +| 已发布 SDK 工作流 | 使用 sdk-minimal profile 启动已构建 dsh，顺序提交 100 个轮次，每轮八次真实文件查看调用，然后关闭 SDK | SDK 收到 200 条助手消息和 800 个成功文件结果；包含 Loader 启动、stdio JSON-RPC、持久化和关闭 | +| 子会话目录 | 通过真实 subagent 和 Session 查询服务，两次列出 16 个非活动、带种子的 fork 子会话 | 两份完整健康目录，观察已释放；每个子会话继承 80 个工具密集轮次，并在精确 fork 切点后拥有自己的描述符 | + +工具执行管线、请求准备、这些服务所需的 Session 投影、持久化和目录观察均保留生产代码。只有模型适配器和有界工具体是合成的。适配器只保留请求计数，不保留请求对象，因此 fixture（测试前置数据）不会制造不断增长的保留成本。顺序输入使每个空闲区间对应此 worker 提交的唯一请求；这不代表并发输入时可以把空闲状态推广为逐消息完成 API。 + +五个样本报告原始壁钟时间、CPU 用户态／内核态时间、峰值 RSS、终点计数及总壁钟时间的最小值、中位数和最大值。预算约束未经舍入的中位数。续聊还相对已初始化 Host 测量保留堆内存：计时操作前后各执行两次显式 GC，中间让出一次事件循环，空闲 Agent 始终可达。因此该增量包含常驻历史 Session 和实时追加，而不只是新轮次。GC 与资源释放不计时；flush 计时。请求历史的内存基线从恢复后开始，只作诊断。目录峰值 RSS 仅作诊断；没有保留堆预算声称衡量已经释放的子会话观察。 + +父进程为每个子进程设置 60 秒上限，独立检查超时、信号、退出状态和报告，等待进程关闭，并在失败后删除私有根目录。Context 和 Agent 在 finally 中释放。播种进程无法预热被测进程的缓存。不强制清除文件系统缓存：冷指新进程，不指冷物理存储。 + +## 校准证据 + +实现参考为 Apple M4 Pro、macOS arm64、Node 24.19.0 上的 `925e012340f033f0521e802ba8569ce6dd7ef1ac`。两轮独占的五样本运行使用相同播种数据，没有产品优化。下表时间单位为毫秒；源码期望值向上取整至实测各轮中位数以上，而不是施加尚未实现的优化目标。 + +| 用例 | 第一轮原始总时间 | 第二轮原始总时间 | 中位数 | 参考期望 | CI 预算 | +|---|---|---|---|---:|---:| +| 请求历史 | 209.134, 210.333, 208.959, 236.355, 238.685 | 222.833, 213.911, 208.089, 211.494, 209.137 | 210.333 / 211.494 | 220 | 550 | +| 工具续聊 | 358.953, 324.790, 318.861, 320.119, 322.896 | 324.280, 321.952, 340.409, 325.470, 324.312 | 322.896 / 324.312 | 340 | 850 | +| 子会话目录 | 318.730, 309.006, 311.404, 308.565, 310.105 | 308.670, 310.030, 280.086, 303.084, 284.829 | 310.105 / 303.084 | 320 | 800 | + +续聊保留约 22.295 MiB；源码期望值为 23 MiB，预算为 28.75 MiB。时间期望值使用现有[校准辅助函数](../../../../benchmarks/support/calibration.ts)：2× 共享 CI 时间比例和 1.25× 波动余量。内存只使用 1.25× 余量。比例继承现有通道的校准，并非这些用例的新 Linux 实测值；runner 特征变化时仍需 CI 证据。基线预算保护实测实现；更紧预算属于有测量依据且保持行为的修复。 + +独立的纯 Node 请求历史 CPU profile 在一次 211.300 ms 操作中，将 132.876 ms 采样自身时间归因于 buildRequest 调用的 deepFreeze。这把重复遍历已冻结历史定位为聚焦调查目标，不是已证实的优化结果。目录首次／重复时间分别保留，因为观察释放后第二次列举仍读取带种子子会话的正文。 + +已发布 SDK 变体完成 100 个轮次、200 次请求和 800 次真实文件读取。五样本 smoke 总时间为 1,521.773、1,463.465、1,689.701、1,365.485 和 1,417.106 ms（中位数 1,463.465 ms）；完整套件重复运行报告 1,596.183、1,784.536、2,120.082、1,405.365 和 1,355.894 ms（中位数 1,596.183 ms）。1,700 ms 参考期望对应 4,250 ms CI 预算。重复运行中未改变的服务用例也变慢，因此这是可变主机负载下的验证，不是放宽其独占校准预算的依据。SDK 进程使用白名单环境和私有主目录／工作区。40 秒截止时间启动 SDK 关闭；所有路径等待同一个记忆化 close Promise，并早于外层 worker 的 60 秒截止时间。Profile 时间包含启动、全部轮次和关闭，分别报告；不把父进程 CPU 或堆指标当作服务端内存。适配器不为外部模型服务商序列化请求。 + +## 考虑过的替代方案 + +**重复现有迁移和首次打开变体。** 拒绝：现有十二个用例已经区分只读准备与可写发布。这些用例使用当前代际并开始或继续实际模型工作，或者列举语料集合而不是打开单个 Session。 + +**只测 deriveMessages。** 拒绝：它的增量缓存不包含完整请求冻结、适配器分发、实时追加或持久化。实际顺序请求保护 Agent 每一步支付的成本。 + +**只使用投影缓存行已预热的无种子子会话。** 拒绝：该路径绕过正文观察，遗漏 fork 子会话的精确继承切点要求。目录用例有意不挂载可选投影缓存，报告带种子的回退路径；它不代表缓存命中的发现过程。 + +**将优化及其目标预算与首次测量一起应用。** 拒绝：纯基线层可以独立合并，并在归因或实现改变前记录当前负载。环境变量不能覆盖源码常量。 + +## 后果 + +通道增加三个场景组中的四个用例、二十个测量 worker 和两个播种进程。集成续聊用例覆盖恢复、完成模型／工具工作及持久化 flush。已发布 SDK 工作流额外包含 profile 启动、SDK 传输、真实文件工具和关闭；只有模型适配器是合成的。它创建新 Session，因为公共 SDK prompt API 创建而非恢复已存储身份。两条路径均不包含网络模型延迟、服务商专属请求序列化、可选用户插件、压缩、失败工具结果、图像、取消或浏览器渲染。功能测试仍负责事件内容、不可变消息、工具语义、fork 谱系以及只读／可写副作用；终点计数防止把跳过的工作当作测量结果，不重复这些断言。 + +本记录补充而非取代 Session 打开门禁的隔离和校准依据。不退役任何现有活跃决策。 diff --git a/benchmarks/agent-continuation/README.i18n.yaml b/benchmarks/agent-continuation/README.i18n.yaml new file mode 100644 index 0000000000..b5af858f60 --- /dev/null +++ b/benchmarks/agent-continuation/README.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write benchmarks/agent-continuation/README.md +README.md: 96f6915ebe2cfa3b334939e87d03828526026281 +README.zh.md: 5889e7b6d04c99f7eff388f2606c8f6bbb201c86 diff --git a/benchmarks/agent-continuation/README.md b/benchmarks/agent-continuation/README.md new file mode 100644 index 0000000000..96f6915ebe --- /dev/null +++ b/benchmarks/agent-continuation/README.md @@ -0,0 +1,31 @@ +# Backend continuation benchmarks + +English | [中文](README.zh.md) + +## Summary + +Measure long-history request processing, cold tool-heavy continuation, and repeated discovery of inactive fork children without network services or recorded user data. The SDK variant drives 100 turns and 800 real file reads through the shipped sdk-minimal profile; other cases isolate backend service costs. No case renders a browser. + +## Table of Contents + +- [Run](#run) +- [Measurements](#measurements) +- [Dev Note](#dev-note) + + + +## Run + +From the repository root, build the libraries and workers with `pnpm run build:bench`, then run `pnpm exec vitest run --config vitest.bench.config.ts benchmarks/agent-continuation/agent-continuation.bench.ts`. Do not overlap timing runs with builds or other benchmarks. + +The test reports all five fresh-process samples and enforces reviewed median budgets. A failed worker reports its exit, signal, timeout, and stderr; temporary roots are removed even on failure. The required benchmark lane discovers this file automatically. + + + +## Measurements + +[workload.ts](workload.ts) owns synthetic dimensions. [The Agent Note](../../.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md) owns timing endpoints, calibration evidence, memory interpretation, and exclusions. The model adapter does not perform provider serialization or network calls; integrated cases use synthetic tool bodies, while the SDK profile variant performs real file reads. + +## Dev Note + +None. diff --git a/benchmarks/agent-continuation/README.zh.md b/benchmarks/agent-continuation/README.zh.md new file mode 100644 index 0000000000..5889e7b6d0 --- /dev/null +++ b/benchmarks/agent-continuation/README.zh.md @@ -0,0 +1,31 @@ +# 后端续聊基准 + +[English](README.md) | 中文 + +## Summary + +在不使用网络服务或录制用户数据的情况下，测量长历史请求处理、冷工具密集续聊和重复发现非活动 fork 子会话。SDK 变体通过已发布 sdk-minimal profile 执行 100 个轮次和 800 次真实文件读取；其他用例隔离后端服务成本。所有用例均不渲染浏览器。 + +## Table of Contents + +- [运行](#run) +- [测量](#measurements) +- [Dev Note](#dev-note) + + + +## 运行 + +在仓库根目录使用 `pnpm run build:bench` 构建库和 worker，然后运行 `pnpm exec vitest run --config vitest.bench.config.ts benchmarks/agent-continuation/agent-continuation.bench.ts`。不要让计时运行与构建或其他基准重叠。 + +测试报告全部五个新进程样本，并约束经审查的中位数预算。worker 失败时报告退出状态、信号、超时和 stderr；失败时也会删除临时根目录。必需基准通道自动发现此文件。 + + + +## 测量 + +[workload.ts](workload.ts)拥有合成维度。[Agent Note](../../.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md)拥有计时终点、校准证据、内存解释和排除项。模型适配器不执行服务商序列化或网络调用；集成用例的合成工具经过真实执行管线，SDK profile 变体则执行真实文件读取。 + +## Dev Note + +无。 diff --git a/benchmarks/agent-continuation/agent-continuation.bench.ts b/benchmarks/agent-continuation/agent-continuation.bench.ts new file mode 100644 index 0000000000..9fde897549 --- /dev/null +++ b/benchmarks/agent-continuation/agent-continuation.bench.ts @@ -0,0 +1,86 @@ +/** Baseline budgets for long-history requests, tool continuation, and fork-child discovery. */ + +import { cp, mkdir, mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { runBuiltBenchmarkWorker } from '../support/built-worker.ts' +import { ciTimeBudget, PERFORMANCE_BUDGET_HEADROOM } from '../support/calibration.ts' +import type { ContinuationReport } from './agent-continuation.worker.ts' +import type { CatalogReport } from './child-catalog.worker.ts' +import type { ProfileReport } from './profile-continuation.worker.ts' +import { WORKLOAD } from './workload.ts' + +const ATTEMPTS = 5 +const WORKER_TIMEOUT_MS = 60_000 +/** M4 Pro / Node 24.19 baseline expectations, before shared CI scaling and variance headroom. */ +const EXPECTED_MS = { 'request-history': 220, 'tool-continuation': 340, catalog: 320, 'profile-continuation': 1_700 } as const +const EXPECTED_RETAINED_HEAP_MB = 23 +const WORKERS = join(import.meta.dirname, '..', '.dsh-build', 'agent-continuation') + +type Scenario = keyof typeof EXPECTED_MS +type Report = ContinuationReport | CatalogReport | ProfileReport + +function workerName(scenario: Scenario): string { + if (scenario === 'profile-continuation') return 'profile-continuation.worker.js' + return scenario === 'catalog' ? 'child-catalog.worker.js' : 'agent-continuation.worker.js' +} + +async function run(root: string, scenario: Scenario, mode: string): Promise { + const outcome = await runBuiltBenchmarkWorker({ + worker: join(WORKERS, workerName(scenario)), args: [root, mode], + timeoutMs: WORKER_TIMEOUT_MS, exposeGc: true, + }) + if (outcome.timedOut || outcome.signal !== null || outcome.exitCode !== 0 || outcome.report === undefined) { + throw new Error('backend worker failed: ' + JSON.stringify(outcome)) + } + return outcome.report +} + +function median(values: readonly number[]): number { + return [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)] as number +} + +describe('continuing tool-heavy Sessions with large histories', () => { + let scratch: string | undefined + const sources = new Map() + + beforeAll(async () => { + scratch = await mkdtemp(join(tmpdir(), 'dsh-agent-continuation-bench-')) + for (const scenario of ['request-history', 'catalog'] as const) { + const root = join(scratch, 'source-' + scenario) + await run(root, scenario, 'seed') + sources.set(scenario, root) + } + sources.set('tool-continuation', sources.get('request-history') as string) + }) + afterAll(async () => { + if (scratch !== undefined) await rm(scratch, { recursive: true, force: true }) + }) + + for (const scenario of ['request-history', 'tool-continuation', 'catalog', 'profile-continuation'] as const) { + it(scenario, async () => { + const samples: Report[] = [] + for (let attempt = 0; attempt < ATTEMPTS; attempt++) { + const root = join(scratch as string, scenario + '-' + String(attempt)) + if (scenario === 'profile-continuation') await mkdir(root) + else await cp(sources.get(scenario) as string, root, { recursive: true }) + try { samples.push(await run(root, scenario, scenario)) } + finally { await rm(root, { recursive: true, force: true }) } + } + const totalMs = samples.map(sample => sample.totalMs) + const budgetMs = ciTimeBudget(EXPECTED_MS[scenario]) + const retainedHeapBudgetMb = EXPECTED_RETAINED_HEAP_MB * PERFORMANCE_BUDGET_HEADROOM + console.log(JSON.stringify({ + benchmark: 'agent-continuation/' + scenario, workload: WORKLOAD, + samples, totalMs: { min: Math.min(...totalMs), median: median(totalMs), max: Math.max(...totalMs) }, + budgetMs, ...(scenario === 'tool-continuation' ? { retainedHeapBudgetMb } : {}), + })) + expect(median(totalMs)).toBeLessThanOrEqual(budgetMs) + if (scenario === 'tool-continuation') { + expect(median((samples as ContinuationReport[]).map(sample => sample.retainedHeapMb))) + .toBeLessThanOrEqual(retainedHeapBudgetMb) + } + }) + } +}) diff --git a/benchmarks/agent-continuation/agent-continuation.worker.ts b/benchmarks/agent-continuation/agent-continuation.worker.ts new file mode 100644 index 0000000000..7e676e9971 --- /dev/null +++ b/benchmarks/agent-continuation/agent-continuation.worker.ts @@ -0,0 +1,144 @@ +/** Plain-Node measurements of active request history and cold tool-heavy continuation. */ + +import { performance } from 'node:perf_hooks' +import { scheduler } from 'node:timers/promises' +import { Context } from '@deepseek-ai/cordis' +import AgentLoop from '@deepseek-ai/dsh-agent-loop' +import type { Agent, AgentHandle } from '@deepseek-ai/dsh-agent' +import { mountAgentLoopTestDependencies } from '@deepseek-ai/dsh-agent-loop-testkit' +import { createUserMessage, LlmAdapter } from '@deepseek-ai/dsh-llm' +import type { GenerateOptions, LlmResolvedModelInfo, StreamChunk } from '@deepseek-ai/dsh-llm' +import { SESSION_FORMAT_VERSION } from '@deepseek-ai/dsh-session' +import JsonlSessionPersistence from '@deepseek-ai/dsh-session-persistence-jsonl' +import SessionProjectionRegistry from '@deepseek-ai/dsh-session-projection' +import { defineContentToolFixture } from '@deepseek-ai/dsh-tools' +import { assertBuiltBenchmarkRuntime } from '../support/built-worker.ts' +import { PARENT_ID, response, resultText, syntheticHistory, TIME_ZERO, WORKLOAD } from './workload.ts' + +/** Raw timing and retained-memory report from one isolated backend process. */ +export interface ContinuationReport { + readonly totalMs: number + readonly resumeMs: number + readonly turnsMs: number + readonly flushMs: number + readonly cpuUserMs: number + readonly cpuSystemMs: number + readonly retainedHeapMb: number + readonly peakRssMb: number + readonly requests: number + readonly toolCalls: number + readonly events: number +} + +class SyntheticAdapter extends LlmAdapter { + requests = 0 + constructor(private readonly toolsPerTurn: number) { super() } + + override resolveModel(provider: string, model: string): Promise { + return Promise.resolve({ provider, id: model, name: model }) + } + + async * stream(_options: GenerateOptions): AsyncIterable { + const tools = this.toolsPerTurn > 0 && this.requests % 2 === 0 ? this.toolsPerTurn : 0 + const reply = response(100_000 + this.requests++, tools) + yield* reply.chunks + } +} + +async function collectHeap(): Promise { + if (globalThis.gc === undefined) throw new Error('backend benchmark requires --expose-gc') + globalThis.gc() + await scheduler.yield() + globalThis.gc() + return process.memoryUsage().heapUsed / 1_048_576 +} + +async function seed(root: string): Promise { + const ctx = new Context() + try { + await mountAgentLoopTestDependencies(ctx) + await ctx.plugin(JsonlSessionPersistence, { root, compression: 'zstd' }) + const handle = await ctx.sessionPersistence.create({ + version: SESSION_FORMAT_VERSION, id: PARENT_ID, createdAt: TIME_ZERO, cwd: '/bench', isSeeded: false, + }, {}) + try { + await handle.append(syntheticHistory(WORKLOAD.historyTurns)) + await handle.flush() + } finally { await handle.close() } + } finally { await ctx.fiber.dispose() } +} + +async function runTurns(agent: Agent, turns: number): Promise { + for (let turn = 0; turn < turns; turn++) { + agent.followup(createUserMessage({ content: [{ type: 'text', text: 'Continue synthetic task ' + String(turn) }], source: { kind: 'user' } })) + await agent.whenIdle() + } +} + +async function measure(root: string, scenario: string): Promise { + const ctx = new Context() + let handle: AgentHandle | undefined + const toolHeavy = scenario === 'tool-continuation' + const adapter = new SyntheticAdapter(toolHeavy ? WORKLOAD.toolsPerLiveTurn : 0) + let toolCalls = 0 + try { + await ctx.plugin(SessionProjectionRegistry) + await mountAgentLoopTestDependencies(ctx) + await ctx.plugin(JsonlSessionPersistence, { root, compression: 'zstd' }) + await ctx.plugin(AgentLoop, { agents: [] }) + ctx.effect(() => ctx.llm.registerAdapter(['bench'], adapter)) + ctx.effect(() => ctx.tools.register(defineContentToolFixture({ + name: 'bench_tool', description: 'Read a bounded synthetic module.', + parameters: { ordinal: { type: 'number', required: true } }, + isConcurrencySafe: () => true, + execute(args) { + toolCalls++ + return Promise.resolve([{ type: 'text', text: resultText(args.ordinal) }]) + }, + }))) + if (!toolHeavy) { + handle = await ctx.agents.resume({ resumeSessionId: PARENT_ID, agentOptions: { provider: 'bench', model: 'bench' } }) + } + const beforeHeap = await collectHeap() + const cpuStart = process.cpuUsage() + const start = performance.now() + if (handle === undefined) { + handle = await ctx.agents.resume({ resumeSessionId: PARENT_ID, agentOptions: { provider: 'bench', model: 'bench' } }) + } + const resumed = performance.now() + await runTurns(handle.agent, toolHeavy ? WORKLOAD.continuationTurns : WORKLOAD.requestTurns) + const turnsDone = performance.now() + await ctx.sessions.flush(handle.agent.session) + const end = performance.now() + const cpu = process.cpuUsage(cpuStart) + const retainedHeapMb = (await collectHeap()) - beforeHeap + if (adapter.requests !== (toolHeavy ? WORKLOAD.continuationTurns * 2 : WORKLOAD.requestTurns) + || toolCalls !== (toolHeavy ? WORKLOAD.continuationTurns * WORKLOAD.toolsPerLiveTurn : 0)) { + throw new Error('backend benchmark did not complete every requested model/tool step') + } + return { + totalMs: end - start, resumeMs: resumed - start, turnsMs: turnsDone - resumed, flushMs: end - turnsDone, + cpuUserMs: cpu.user / 1_000, cpuSystemMs: cpu.system / 1_000, + retainedHeapMb, peakRssMb: process.resourceUsage().maxRSS / 1_024, + requests: adapter.requests, toolCalls, events: handle.agent.session.seq, + } + } finally { + await handle?.dispose() + await ctx.fiber.dispose() + } +} + +assertBuiltBenchmarkRuntime(import.meta.url, Object.fromEntries([ + '@deepseek-ai/dsh-agent-loop', '@deepseek-ai/dsh-session', '@deepseek-ai/dsh-llm', + '@deepseek-ai/dsh-tools', '@deepseek-ai/dsh-session-persistence-jsonl', +].map(name => [name, import.meta.resolve(name)]))) +const [root, scenario] = process.argv.slice(2) +if (root === undefined || scenario === undefined || !['seed', 'request-history', 'tool-continuation'].includes(scenario)) { + throw new Error('usage: agent-continuation.worker.js ') +} +if (scenario === 'seed') { + await seed(root) + process.stdout.write(JSON.stringify({ seeded: true }) + '\n') +} else { + process.stdout.write(JSON.stringify(await measure(root, scenario)) + '\n') +} diff --git a/benchmarks/agent-continuation/child-catalog.worker.ts b/benchmarks/agent-continuation/child-catalog.worker.ts new file mode 100644 index 0000000000..e12cb97d2f --- /dev/null +++ b/benchmarks/agent-continuation/child-catalog.worker.ts @@ -0,0 +1,95 @@ +/** Cold catalog observations of persisted fork children with tool-heavy inherited histories. */ + +import { performance } from 'node:perf_hooks' +import { Context } from '@deepseek-ai/cordis' +import SessionStore, { SESSION_FORMAT_VERSION, SessionId, SessionLogOffset, SessionSeq } from '@deepseek-ai/dsh-session' +import type { SessionEvent } from '@deepseek-ai/dsh-session' +import JsonlSessionPersistence from '@deepseek-ai/dsh-session-persistence-jsonl' +import SessionProjectionRegistry from '@deepseek-ai/dsh-session-projection' +import SessionQueryEngine from '@deepseek-ai/dsh-session-query' +import SubagentRuntime, { SUBAGENT_DESCRIPTOR_VERSION } from '@deepseek-ai/dsh-subagent' +import { assertBuiltBenchmarkRuntime } from '../support/built-worker.ts' +import { PARENT_ID, syntheticHistory, TIME_ZERO, WORKLOAD } from './workload.ts' + +/** Two complete catalog reads in one fresh Host, with every child observation released. */ +export interface CatalogReport { + readonly totalMs: number + readonly firstMs: number + readonly repeatMs: number + readonly cpuUserMs: number + readonly cpuSystemMs: number + readonly children: number + readonly peakRssMb: number +} + +class CatalogQuery extends SessionQueryEngine { + override searchSessions(): Promise { + return Promise.reject(new Error('search is outside the child-catalog benchmark')) + } + override searchEvents(): Promise { + return Promise.reject(new Error('search is outside the child-catalog benchmark')) + } +} + +async function seed(ctx: Context): Promise { + const inherited = syntheticHistory(WORKLOAD.childHistoryTurns) + for (let child = 0; child < WORKLOAD.children; child++) { + const id = SessionId('bench-child-' + String(child)) + const events: SessionEvent[] = [ + ...inherited, + { type: 'session/end-seed', seq: SessionSeq(inherited.length), time: TIME_ZERO + inherited.length, data: { inherited: true } }, + { type: 'subagent/descriptor', seq: SessionSeq(inherited.length + 1), time: TIME_ZERO + inherited.length + 1, data: { + version: SUBAGENT_DESCRIPTOR_VERSION, mode: 'continuable', provider: 'fork', label: 'Synthetic child ' + String(child), + } }, + ] + const handle = await ctx.sessionPersistence.create({ + version: SESSION_FORMAT_VERSION, id, createdAt: TIME_ZERO + child, cwd: '/bench', + parentSession: PARENT_ID, isSeeded: true, origin: 'subagent', delegationDepth: 1, + }, { inheritedEventCount: SessionLogOffset(inherited.length) }) + try { + await handle.append(events) + await handle.flush() + } finally { await handle.close() } + } +} + +async function run(root: string, mode: string): Promise { + const ctx = new Context() + try { + await ctx.plugin(SessionStore) + await ctx.plugin(SessionProjectionRegistry) + await ctx.plugin(JsonlSessionPersistence, { root, compression: 'zstd' }) + await ctx.plugin(CatalogQuery) + await ctx.plugin(SubagentRuntime) + if (mode === 'seed') { + await seed(ctx) + return { seeded: true } + } + const cpuStart = process.cpuUsage() + const start = performance.now() + const first = await ctx.subagents.listChildren(PARENT_ID) + const firstDone = performance.now() + const repeated = await ctx.subagents.listChildren(PARENT_ID) + const end = performance.now() + const cpu = process.cpuUsage(cpuStart) + if (first.length !== WORKLOAD.children || repeated.length !== WORKLOAD.children + || [...first, ...repeated].some(row => row.kind !== 'child')) { + throw new Error('child-catalog benchmark did not reach the complete healthy catalog') + } + return { + totalMs: end - start, firstMs: firstDone - start, repeatMs: end - firstDone, + cpuUserMs: cpu.user / 1_000, cpuSystemMs: cpu.system / 1_000, + children: first.length, peakRssMb: process.resourceUsage().maxRSS / 1_024, + } + } finally { await ctx.fiber.dispose() } +} + +assertBuiltBenchmarkRuntime(import.meta.url, Object.fromEntries([ + '@deepseek-ai/dsh-subagent', '@deepseek-ai/dsh-session-query', + '@deepseek-ai/dsh-session-persistence-jsonl', +].map(name => [name, import.meta.resolve(name)]))) +const [root, mode] = process.argv.slice(2) +if (root === undefined || (mode !== 'seed' && mode !== 'catalog')) { + throw new Error('usage: child-catalog.worker.js ') +} +process.stdout.write(JSON.stringify(await run(root, mode)) + '\n') diff --git a/benchmarks/agent-continuation/profile-adapter.ts b/benchmarks/agent-continuation/profile-adapter.ts new file mode 100644 index 0000000000..8c42c27fb9 --- /dev/null +++ b/benchmarks/agent-continuation/profile-adapter.ts @@ -0,0 +1,42 @@ +/** Compiled synthetic model for the shipped sdk-minimal profile; tools remain production plugins. */ + +import type { Context } from '@deepseek-ai/cordis' +import { LlmAdapter, ToolCallId } from '@deepseek-ai/dsh-llm' +import type { GenerateOptions, LlmResolvedModelInfo, StreamChunk } from '@deepseek-ai/dsh-llm' +import { response, WORKLOAD } from './workload.ts' + +class ProfileAdapter extends LlmAdapter { + private requests = 0 + override resolveModel(provider: string, model: string): Promise { + return Promise.resolve({ provider, id: model, name: model, contextWindow: 1_000_000 }) + } + + async * stream(_options: GenerateOptions): AsyncIterable { + const serial = this.requests++ + if (serial % 2 === 1) { + yield* response(200_000 + serial, 0).chunks + return + } + for (let index = 0; index < WORKLOAD.toolsPerLiveTurn; index++) { + const id = ToolCallId('profile-call-' + String(serial) + '-' + String(index)) + const args = JSON.stringify({ command: 'view', path: process.cwd() + '/synthetic.txt' }) + yield { type: 'block-start', index, blockType: 'tool-call' } + yield { type: 'tool-call-delta', index, id, name: 'str_replace_editor', argumentsDelta: args } + yield { type: 'block-end', index, block: { type: 'tool-call', id, name: 'str_replace_editor', arguments: args } } + } + yield { type: 'finish', reason: { kind: 'tool-calls' } } + } +} + +/** Loader plugin identity. */ +export const name = 'backend-profile-benchmark-model' +/** The scripted provider requires the production LLM registry. */ +export const inject = ['llm'] + +/** + * Register the synthetic provider without changing any runtime services or tools. + * @param ctx - profile-owned plugin context. + */ +export function apply(ctx: Context): void { + ctx.effect(() => ctx.llm.registerAdapter(['bench'], new ProfileAdapter())) +} diff --git a/benchmarks/agent-continuation/profile-continuation.worker.ts b/benchmarks/agent-continuation/profile-continuation.worker.ts new file mode 100644 index 0000000000..4e0fade29b --- /dev/null +++ b/benchmarks/agent-continuation/profile-continuation.worker.ts @@ -0,0 +1,85 @@ +/** End-to-end SDK continuation through the built dsh sdk-minimal profile and real file tools. */ + +import { mkdir, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { performance } from 'node:perf_hooks' +import { DeepSeekHarness } from '@deepseek-ai/dsh-sdk-client' +import { assertBuiltBenchmarkRuntime } from '../support/built-worker.ts' +import { PARENT_ID, resultText, WORKLOAD } from './workload.ts' + +/** Parent-observed wall time, including profile launch and SDK shutdown. */ +export interface ProfileReport { + readonly totalMs: number + readonly bootMs: number + readonly turnsMs: number + readonly closeMs: number + readonly requests: number + readonly toolCalls: number +} + +async function run(root: string): Promise { + const home = join(root, 'home') + const cwd = join(root, 'workspace') + await mkdir(cwd, { recursive: true }) + await mkdir(home, { recursive: true }) + await writeFile(join(cwd, 'synthetic.txt'), resultText(0)) + const patch = join(root, 'profile.patch.yml') + await writeFile(patch, [ + '- id: llm-deepseek', ' disabled: true', + '- id: sessions', ' config:', ' root: ' + JSON.stringify(join(root, 'profile-sessions')), ' compression: zstd', + '- insert:', ' - id: benchmark-model', ' name: ' + JSON.stringify(join(import.meta.dirname, 'profile-adapter.js')), + '', + ].join('\n')) + const env: NodeJS.ProcessEnv = { + PATH: process.env.PATH, HOME: home, USERPROFILE: home, + DSH_AGENTS_HOME: join(home, 'agents'), + } + const harness = new DeepSeekHarness({ + dshBin: join(import.meta.dirname, '..', '..', '..', 'apps', 'cli', 'lib', 'bin.js'), + profile: 'sdk-minimal', dshHome: home, processCwd: cwd, cwd, + provider: 'bench', model: 'bench', patches: [patch], env, + initializeTimeoutMs: 15_000, requestTimeoutMs: 15_000, + }) + let closing: Promise | undefined + const close = (): Promise => closing ??= harness.close() + let expired = false + const deadline = setTimeout(() => { + expired = true + // The awaited finally close below reports shutdown failures; this only requests cancellation. + void close().catch(() => undefined) + }, 40_000) + let requests = 0 + let toolCalls = 0 + const start = performance.now() + try { + await harness.start() + const booted = performance.now() + for (let turn = 0; turn < WORKLOAD.profileTurns; turn++) { + const result = await harness.run('Read the synthetic file ' + String(turn), { sessionId: PARENT_ID }) + requests += result.events.filter(event => event.type === 'assistant/message').length + for (const event of result.events) { + if (event.type !== 'tool/result') continue + const result = event.data.message.content[0] + if (result.isError || !result.content.some(block => block.type === 'text' && block.text.includes('export const synthetic = 42;'))) { + throw new Error('profile benchmark did not read the synthetic file') + } + toolCalls++ + } + } + const turnsDone = performance.now() + await close() + const end = performance.now() + if (expired || requests !== WORKLOAD.profileTurns * 2 || toolCalls !== WORKLOAD.profileTurns * WORKLOAD.toolsPerLiveTurn) { + throw new Error('profile benchmark did not finish every model request and real tool call') + } + return { totalMs: end - start, bootMs: booted - start, turnsMs: turnsDone - booted, closeMs: end - turnsDone, requests, toolCalls } + } finally { + clearTimeout(deadline) + await close() + } +} + +assertBuiltBenchmarkRuntime(import.meta.url, { '@deepseek-ai/dsh-sdk-client': import.meta.resolve('@deepseek-ai/dsh-sdk-client') }) +const [root] = process.argv.slice(2) +if (root === undefined) throw new Error('usage: profile-continuation.worker.js ') +process.stdout.write(JSON.stringify(await run(root)) + '\n') diff --git a/benchmarks/agent-continuation/workload.ts b/benchmarks/agent-continuation/workload.ts new file mode 100644 index 0000000000..6785585574 --- /dev/null +++ b/benchmarks/agent-continuation/workload.ts @@ -0,0 +1,110 @@ +/** Reviewed synthetic tool history shared by continuation and child-catalog measurements. */ + +import { AssistantStreamAccumulator } from '@deepseek-ai/dsh-llm/assistant-stream' +import { MessageId, ToolCallId } from '@deepseek-ai/dsh-llm' +import type { ContentBlock, StreamChunk } from '@deepseek-ai/dsh-llm' +import { Session, SessionId } from '@deepseek-ai/dsh-session' +import type { SessionEvent } from '@deepseek-ai/dsh-session' + +/** Workload dimensions, independent of environment and recorded user material. */ +export const WORKLOAD = { + historyTurns: 800, + toolsPerHistoricalTurn: 4, + toolResultChars: 2_048, + requestTurns: 40, + continuationTurns: 20, + profileTurns: 100, + toolsPerLiveTurn: 8, + children: 16, + childHistoryTurns: 80, +} as const + +/** Fixed clock used only to author persisted synthetic input. */ +export const TIME_ZERO = 1_700_000_000_000 +/** Durable parent identity of the measured continuation. */ +export const PARENT_ID = SessionId('bench-parent') + +/** + * Construct a deterministic model reply without retaining past requests. + * @param serial - unique response ordinal. + * @param tools - number of synthetic tool calls, or zero for a final text reply. + * @returns streamed chunks and their known final blocks. + */ +export function response(serial: number, tools: number): { chunks: StreamChunk[]; content: ContentBlock[] } { + const content: ContentBlock[] = [ + { type: 'reasoning', text: 'Inspect the synthetic result. '.repeat(8) }, + { type: 'text', text: 'Synthetic response. '.repeat(8) }, + ...Array.from({ length: tools }, (_, index): ContentBlock => ({ + type: 'tool-call', id: ToolCallId('call-' + String(serial) + '-' + String(index)), name: 'bench_tool', + arguments: JSON.stringify({ ordinal: serial * 100 + index }), + })), + ] + const chunks: StreamChunk[] = [] + content.forEach((block, index) => { + chunks.push({ type: 'block-start', index, blockType: block.type }) + if (block.type === 'text' || block.type === 'reasoning') { + for (let offset = 0; offset < block.text.length; offset += 16) { + chunks.push({ type: block.type === 'text' ? 'text-delta' : 'reasoning-delta', index, text: block.text.slice(offset, offset + 16) }) + } + } else if (block.type === 'tool-call') { + for (let offset = 0; offset < block.arguments.length; offset += 8) { + chunks.push({ type: 'tool-call-delta', index, id: block.id, name: block.name, argumentsDelta: block.arguments.slice(offset, offset + 8) }) + } + } + chunks.push({ type: 'block-end', index, block }) + }) + chunks.push({ type: 'usage', usage: { inputTokens: 10_000, outputTokens: 100 } }) + chunks.push({ type: 'finish', reason: { kind: tools === 0 ? 'stop' : 'tool-calls' } }) + return { chunks, content } +} + +/** + * Author completed two-step turns through production Session append and stream compaction. + * @param turns - completed historical turns. + * @returns detached current-generation events with fixed ids, timestamps and payloads. + */ +export function syntheticHistory(turns: number): SessionEvent[] { + const session = Session.create(PARENT_ID) + for (let turn = 1; turn <= turns; turn++) { + session.append('turn/start', { turn }) + session.append('step/start', { turn, step: 1 }) + session.append('user/message', { + id: MessageId('prompt-' + String(turn)), role: 'user', + content: [{ type: 'text', text: 'Inspect synthetic module ' + String(turn) }], source: { kind: 'user' }, + }, { surfaceOp: 'append' }) + for (const step of [1, 2]) { + if (step === 2) session.append('step/start', { turn, step }) + const reply = response(turn * 2 + step, step === 1 ? WORKLOAD.toolsPerHistoricalTurn : 0) + const stream = new AssistantStreamAccumulator() + reply.chunks.forEach((chunk, index) => { stream.push({ time: TIME_ZERO + turn * 1_000 + step * 100 + index, chunk }) }) + session.append('assistant/message', { + turn, step, + message: { id: MessageId('reply-' + String(turn) + '-' + String(step)), role: 'assistant', content: reply.content, source: { kind: 'model', provider: 'bench', model: 'bench' } }, + stream: [...stream.snapshot()], + }, { surfaceOp: 'append' }) + for (const block of reply.content) { + if (block.type !== 'tool-call') continue + const call = session.append('tool/call', { turn, step, callId: block.id, name: block.name, arguments: block.arguments }) + session.append('tool/result', { + turn, step, + message: { + id: MessageId('result-' + block.id), role: 'user', source: { kind: 'tool', callId: block.id }, + content: [{ type: 'tool-result', toolCallId: block.id, content: [{ type: 'text', text: resultText(turn) }], isError: false }], + }, + }, { surfaceOp: 'append', sourceEventSeqs: [call.seq] }) + } + session.append('step/end', { turn, step }) + } + session.append('turn/end', { turn, reason: { kind: 'completed' } }) + } + return session.snapshotEvents().map(event => ({ ...event, time: TIME_ZERO + event.seq })) +} + +/** + * Build a bounded synthetic file-read result with a varying prefix. + * @param ordinal - deterministic result identifier. + * @returns exactly the reviewed number of UTF-16 characters. + */ +export function resultText(ordinal: number): string { + return ('module ' + String(ordinal) + '\n' + 'export const synthetic = 42;\n'.repeat(100)).slice(0, WORKLOAD.toolResultChars) +} diff --git a/benchmarks/package.json b/benchmarks/package.json index 673146977e..2cf0eec369 100644 --- a/benchmarks/package.json +++ b/benchmarks/package.json @@ -15,6 +15,7 @@ "@deepseek-ai/dsh-client-ui-chat": "workspace:^", "@deepseek-ai/dsh-deque": "workspace:^", "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/dsh-sdk-client": "workspace:^", "@deepseek-ai/dsh-session": "workspace:^", "@deepseek-ai/dsh-session-persistence": "workspace:^", "@deepseek-ai/dsh-session-persistence-jsonl": "workspace:^", @@ -23,6 +24,8 @@ "@deepseek-ai/dsh-session-stats": "workspace:^", "@deepseek-ai/dsh-session-title": "workspace:^", "@deepseek-ai/dsh-session-turn-outline": "workspace:^", + "@deepseek-ai/dsh-subagent": "workspace:^", + "@deepseek-ai/dsh-tools": "workspace:^", "@deepseek-ai/dsh-token-meter": "workspace:^", "@deepseek-ai/dsh-typert-protocol": "workspace:^" }, diff --git a/benchmarks/tsdown.config.ts b/benchmarks/tsdown.config.ts index 2f3112308f..3c8d6a9ab5 100644 --- a/benchmarks/tsdown.config.ts +++ b/benchmarks/tsdown.config.ts @@ -14,6 +14,18 @@ const shared = { /** Compile measured benchmark workers while keeping workspace packages on their built `lib` entries. */ export default defineConfig([ + { + ...shared, + entry: { + 'agent-continuation.worker': 'agent-continuation/agent-continuation.worker.ts', + 'child-catalog.worker': 'agent-continuation/child-catalog.worker.ts', + 'profile-continuation.worker': 'agent-continuation/profile-continuation.worker.ts', + 'profile-adapter': 'agent-continuation/profile-adapter.ts', + }, + outDir: '.dsh-build/agent-continuation', + clean: true, + tsconfig: 'tsconfig.host.json', + }, { ...shared, entry: { 'session-open.worker': 'session-open/session-open.worker.ts' }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f8b88cc454..cf86833ddb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -581,6 +581,9 @@ importers: '@deepseek-ai/dsh-llm': specifier: workspace:^ version: link:../packages/llm/llm + '@deepseek-ai/dsh-sdk-client': + specifier: workspace:^ + version: link:../packages/sdk/client '@deepseek-ai/dsh-session': specifier: workspace:^ version: link:../packages/core/session @@ -605,9 +608,15 @@ importers: '@deepseek-ai/dsh-session-turn-outline': specifier: workspace:^ version: link:../packages/session/session-turn-outline + '@deepseek-ai/dsh-subagent': + specifier: workspace:^ + version: link:../packages/subagent/subagent '@deepseek-ai/dsh-token-meter': specifier: workspace:^ version: link:../packages/llm/token-meter + '@deepseek-ai/dsh-tools': + specifier: workspace:^ + version: link:../packages/core/tools '@deepseek-ai/dsh-typert-protocol': specifier: workspace:^ version: link:../packages/typert/protocol From 1507f828887b55cad8b65fa81c0381b3fa476469 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 15:34:24 +0800 Subject: [PATCH 03/22] docs(perf): record backend CI evidence and align workload prose --- .../2026-09-06-backend-continuation-performance.i18n.yaml | 4 ++-- .../testing/2026-09-06-backend-continuation-performance.md | 4 +++- .../testing/2026-09-06-backend-continuation-performance.zh.md | 4 +++- benchmarks/agent-continuation/README.i18n.yaml | 4 ++-- benchmarks/agent-continuation/README.md | 2 +- benchmarks/agent-continuation/README.zh.md | 2 +- 6 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml index d6f7815fa1..30d8a4d77a 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md -2026-09-06-backend-continuation-performance.md: 4c18e440c98135f140c4957a7c5e7c73188c694c -2026-09-06-backend-continuation-performance.zh.md: 2cf1793faf39ab0c3bc10b1932db15e64961e8b8 +2026-09-06-backend-continuation-performance.md: 0ad7f5267136181e5d45d4bc6f452f6cf9a744ca +2026-09-06-backend-continuation-performance.zh.md: e07cd154d1d879cca010fb0e36872205710c37a4 diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md index 4c18e440c9..0ad7f52671 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md @@ -12,7 +12,7 @@ Opening one Session does not measure the repeated cost of preparing model reques The [agent-continuation benchmark](../../../../benchmarks/agent-continuation/agent-continuation.bench.ts) adds three scenario groups, including a shipped-profile variant, without changing product implementations. They use current-generation Zstandard Sessions authored through production append, stream accumulation, and persistence APIs. A separate seed process creates the deterministic source before measurement; each sample copies that source into its private root and starts a fresh compiled plain-Node worker. No recorded Session, ambient repository, network, private Harness home, or deployed GUI supplies input. -The shared history has 800 completed two-step turns, four tool calls per turn, and 2,048-character tool results: 13,600 events and 5,600 model messages. Each assistant reply carries reasoning, text, and compact streamed records; tool replies additionally carry fragmented arguments. Fixed timestamps and ids describe the seed. Live synthetic replies use the real loop's clocks and ids without overriding process globals. +The shared history has 800 completed two-step turns, four tool calls per turn, and 2,048-character tool results: 13,600 events and 5,600 conversation messages. Each assistant reply carries reasoning, text, and compact streamed records; tool replies additionally carry fragmented arguments. Fixed timestamps and ids describe the seed. Live synthetic replies use the real loop's clocks and ids without overriding process globals. | Case | Timed operation | Endpoint | |---|---|---| @@ -43,6 +43,8 @@ A separate plain-Node request-history CPU profile attributes 132.876 ms of sampl The shipped SDK variant completes 100 turns, 200 requests, and 800 real file reads. Its five-sample smoke totals are 1,521.773, 1,463.465, 1,689.701, 1,365.485, and 1,417.106 ms (median 1,463.465 ms); a full-suite repeat reports 1,596.183, 1,784.536, 2,120.082, 1,405.365, and 1,355.894 ms (median 1,596.183 ms). Its 1,700 ms reference expectation yields a 4,250 ms CI budget. The repeat also slows the unchanged service cases, so it is validation under variable host load rather than evidence to relax their exclusive calibration. The SDK process receives an allowlisted environment and private home/workspace. A 40-second deadline starts SDK shutdown; every path awaits the same memoized close promise before the outer worker’s 60-second deadline. Profile timing includes boot, all turns, and shutdown, reported separately; no parent-process CPU or heap metric is presented as server memory. The adapter does not serialize requests for an external model provider. +The first Linux x64 CI measurement at commit `1dc3296eba631d51fbb3bb50e249bf3cc0fce9f6` ran on `VM-7-113-ubuntu-ci-10` with Node 24.18.1 ([run 34017868081, attempt 1, job 101444810498](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34017868081/job/101444810498)). The SDK median was 2,753.441 ms against its 4,250 ms budget, and tool-continuation retained-heap median was 22.274 MiB against 28.75 MiB. Request-history and tool-continuation time budgets failed: 785.498 ms against 550 ms and 1,077.285 ms against 850 ms, respectively. The unchanged Session-reopen open phase also failed at 31.6 ms against 30 ms. [Attempt 2, job 101447076381](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34017868081/job/101447076381) passed every benchmark on the same commit and unchanged budgets, but used `VM-7-113-ubuntu-ci-29` with Node 24.19.0. The gate runner suppressed successful child output, so that attempt supplies a passing verdict rather than raw medians. The changed runner and Node version prevent attributing the difference solely to contention or claiming stable repeated CI calibration; neither the budgets nor the shared scale are changed on this evidence. + ## Alternatives considered **Repeat existing migration and first-open variants.** Rejected: those twelve cases already distinguish read-only preparation from writable publication. These cases use the current generation and begin or continue actual model work, or enumerate a corpus rather than open one Session. diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md index 2cf1793faf..e07cd154d1 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md @@ -12,7 +12,7 @@ Status: implemented [agent-continuation 基准](../../../../benchmarks/agent-continuation/agent-continuation.bench.ts)增加三个场景组，包含一个已发布 profile 变体，不修改产品实现。它们通过生产追加、流累积和持久化 API 构造当前代际的 Zstandard Session。独立播种进程在测量前生成确定性源数据；每个样本将其复制到私有根目录，并启动新的已编译纯 Node worker。输入不来自录制 Session、环境仓库、网络、私有 Harness 主目录或已部署 GUI。 -共享历史包含 800 个已完成的双步骤轮次，每轮四次工具调用，工具结果为 2,048 字符：共 13,600 个事件和 5,600 条模型消息。每条助手回复携带推理、文本和紧凑流记录；请求工具的回复还携带分片参数。播种数据使用固定时间戳和 id。实时合成回复使用真实循环的时钟和 id，不覆盖进程全局状态。 +共享历史包含 800 个已完成的双步骤轮次，每轮四次工具调用，工具结果为 2,048 字符：共 13,600 个事件和 5,600 条对话消息。每条助手回复携带推理、文本和紧凑流记录；请求工具的回复还携带分片参数。播种数据使用固定时间戳和 id。实时合成回复使用真实循环的时钟和 id，不覆盖进程全局状态。 | 用例 | 计时操作 | 终点 | |---|---|---| @@ -43,6 +43,8 @@ Status: implemented 已发布 SDK 变体完成 100 个轮次、200 次请求和 800 次真实文件读取。五样本 smoke 总时间为 1,521.773、1,463.465、1,689.701、1,365.485 和 1,417.106 ms（中位数 1,463.465 ms）；完整套件重复运行报告 1,596.183、1,784.536、2,120.082、1,405.365 和 1,355.894 ms（中位数 1,596.183 ms）。1,700 ms 参考期望对应 4,250 ms CI 预算。重复运行中未改变的服务用例也变慢，因此这是可变主机负载下的验证，不是放宽其独占校准预算的依据。SDK 进程使用白名单环境和私有主目录／工作区。40 秒截止时间启动 SDK 关闭；所有路径等待同一个记忆化 close Promise，并早于外层 worker 的 60 秒截止时间。Profile 时间包含启动、全部轮次和关闭，分别报告；不把父进程 CPU 或堆指标当作服务端内存。适配器不为外部模型服务商序列化请求。 +提交 `1dc3296eba631d51fbb3bb50e249bf3cc0fce9f6` 的首次 Linux x64 CI 测量使用 `VM-7-113-ubuntu-ci-10` 和 Node 24.18.1（[run 34017868081，attempt 1，job 101444810498](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34017868081/job/101444810498)）。SDK 中位数为 2,753.441 ms，预算为 4,250 ms；工具续聊保留堆中位数为 22.274 MiB，预算为 28.75 MiB。请求历史与工具续聊时间预算失败：分别为 785.498 ms 对 550 ms、1,077.285 ms 对 850 ms。未修改的 Session 重开 open 阶段也以 31.6 ms 对 30 ms 失败。[Attempt 2，job 101447076381](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34017868081/job/101447076381) 在同一提交和未修改预算下通过全部基准，但使用 `VM-7-113-ubuntu-ci-29` 和 Node 24.19.0。门禁运行器隐藏成功子进程的输出，因此该次运行只提供通过结论，不提供原始中位数。Runner 与 Node 版本同时变化，不能把差异仅归因于资源争用，也不能宣称已获得稳定的重复 CI 校准；这些证据不改变预算或共享比例。 + ## 考虑过的替代方案 **重复现有迁移和首次打开变体。** 拒绝：现有十二个用例已经区分只读准备与可写发布。这些用例使用当前代际并开始或继续实际模型工作，或者列举语料集合而不是打开单个 Session。 diff --git a/benchmarks/agent-continuation/README.i18n.yaml b/benchmarks/agent-continuation/README.i18n.yaml index b5af858f60..41fa944b7e 100644 --- a/benchmarks/agent-continuation/README.i18n.yaml +++ b/benchmarks/agent-continuation/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write benchmarks/agent-continuation/README.md -README.md: 96f6915ebe2cfa3b334939e87d03828526026281 -README.zh.md: 5889e7b6d04c99f7eff388f2606c8f6bbb201c86 +README.md: 13fc21ea486bea55a93011a10edaca1cbe40ff47 +README.zh.md: f3f1fdcfb95f2948add2ebec7e1ea711172b7870 diff --git a/benchmarks/agent-continuation/README.md b/benchmarks/agent-continuation/README.md index 96f6915ebe..13fc21ea48 100644 --- a/benchmarks/agent-continuation/README.md +++ b/benchmarks/agent-continuation/README.md @@ -24,7 +24,7 @@ The test reports all five fresh-process samples and enforces reviewed median bud ## Measurements -[workload.ts](workload.ts) owns synthetic dimensions. [The Agent Note](../../.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md) owns timing endpoints, calibration evidence, memory interpretation, and exclusions. The model adapter does not perform provider serialization or network calls; integrated cases use synthetic tool bodies, while the SDK profile variant performs real file reads. +[workload.ts](workload.ts) owns synthetic dimensions. [The Agent Note](../../.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md) owns timing endpoints, calibration evidence, memory interpretation, and exclusions. The model adapter does not perform provider serialization or network calls; integrated cases run synthetic tool bodies through the real tool-execution pipeline, while the SDK profile variant performs real file reads. ## Dev Note diff --git a/benchmarks/agent-continuation/README.zh.md b/benchmarks/agent-continuation/README.zh.md index 5889e7b6d0..f3f1fdcfb9 100644 --- a/benchmarks/agent-continuation/README.zh.md +++ b/benchmarks/agent-continuation/README.zh.md @@ -24,7 +24,7 @@ ## 测量 -[workload.ts](workload.ts)拥有合成维度。[Agent Note](../../.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md)拥有计时终点、校准证据、内存解释和排除项。模型适配器不执行服务商序列化或网络调用；集成用例的合成工具经过真实执行管线，SDK profile 变体则执行真实文件读取。 +[workload.ts](workload.ts)拥有合成维度。[Agent Note](../../.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md)拥有计时终点、校准证据、内存解释和排除项。模型适配器不执行服务商序列化或网络调用；集成用例通过真实工具执行管线运行合成工具体，SDK profile 变体则执行真实文件读取。 ## Dev Note From daa7f60630197efe8a7636a3b423dac9f06acda4 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 20:48:24 +0800 Subject: [PATCH 04/22] test(perf): calibrate catalog for standard hosted CI --- ...backend-continuation-performance.i18n.yaml | 4 +-- ...-09-06-backend-continuation-performance.md | 6 ++-- ...-06-backend-continuation-performance.zh.md | 6 ++-- .../agent-continuation/README.i18n.yaml | 4 +-- benchmarks/agent-continuation/README.md | 2 +- benchmarks/agent-continuation/README.zh.md | 2 +- .../agent-continuation.bench.ts | 33 ++++++++++++++++--- 7 files changed, 43 insertions(+), 14 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml index 30d8a4d77a..031e846f05 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md -2026-09-06-backend-continuation-performance.md: 0ad7f5267136181e5d45d4bc6f452f6cf9a744ca -2026-09-06-backend-continuation-performance.zh.md: e07cd154d1d879cca010fb0e36872205710c37a4 +2026-09-06-backend-continuation-performance.md: c247c7b9c4e2612603e0fb04b4f1a03b80314407 +2026-09-06-backend-continuation-performance.zh.md: bba1b4fc8a742ee261be6d09f61dd754568ebe0b diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md index 0ad7f52671..c247c7b9c4 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md @@ -31,13 +31,13 @@ The parent bounds every child to 60 seconds, checks timeout, signal, exit, and r The implementation reference is `925e012340f033f0521e802ba8569ce6dd7ef1ac` on Apple M4 Pro, macOS arm64, Node 24.19.0. Two exclusive five-sample runs use the same seed and no product optimization. Durations below are milliseconds; source expectations round above the observed run medians rather than imposing an unimplemented optimization target. -| Case | Run 1 raw totals | Run 2 raw totals | Medians | Reference expectation | CI budget | +| Case | Run 1 raw totals | Run 2 raw totals | Medians | Historical M4 expectation | Historical scaled budget | |---|---|---|---|---:|---:| | Request history | 209.134, 210.333, 208.959, 236.355, 238.685 | 222.833, 213.911, 208.089, 211.494, 209.137 | 210.333 / 211.494 | 220 | 550 | | Tool continuation | 358.953, 324.790, 318.861, 320.119, 322.896 | 324.280, 321.952, 340.409, 325.470, 324.312 | 322.896 / 324.312 | 340 | 850 | | Child catalog | 318.730, 309.006, 311.404, 308.565, 310.105 | 308.670, 310.030, 280.086, 303.084, 284.829 | 310.105 / 303.084 | 320 | 800 | -Continuation retains approximately 22.295 MiB; its source expectation is 23 MiB and its budget is 28.75 MiB. Time expectations use the existing [calibration helper](../../../../benchmarks/support/calibration.ts): 2× shared CI time scale and 1.25× variance headroom. Memory uses only 1.25× headroom. The scale is inherited from the existing lane's calibration, not a new Linux measurement of these cases; CI evidence remains necessary when runner characteristics change. Baseline budgets protect the measured implementation; tighter budgets belong with a measured behavior-preserving fix. +Continuation retains approximately 22.295 MiB; its source expectation is 23 MiB and its budget is 28.75 MiB. Request-history, tool-continuation, and SDK time expectations use the existing [calibration helper](../../../../benchmarks/support/calibration.ts): 2× shared CI time scale and 1.25× variance headroom. Memory uses only 1.25× headroom. The scale is inherited from the existing lane's calibration, not a new Linux measurement of these cases; CI evidence remains necessary when runner characteristics change. Baseline budgets protect the measured implementation; tighter budgets belong with a measured behavior-preserving fix. A separate plain-Node request-history CPU profile attributes 132.876 ms of sampled self time to deepFreeze called by buildRequest during a 211.300 ms operation. This identifies repeated traversal of already-frozen history as a focused investigation target, not a proven optimization result. Catalog first/repeat timings remain separate because a second listing still reads body-bearing seeded children after observations are released. @@ -45,6 +45,8 @@ The shipped SDK variant completes 100 turns, 200 requests, and 800 real file rea The first Linux x64 CI measurement at commit `1dc3296eba631d51fbb3bb50e249bf3cc0fce9f6` ran on `VM-7-113-ubuntu-ci-10` with Node 24.18.1 ([run 34017868081, attempt 1, job 101444810498](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34017868081/job/101444810498)). The SDK median was 2,753.441 ms against its 4,250 ms budget, and tool-continuation retained-heap median was 22.274 MiB against 28.75 MiB. Request-history and tool-continuation time budgets failed: 785.498 ms against 550 ms and 1,077.285 ms against 850 ms, respectively. The unchanged Session-reopen open phase also failed at 31.6 ms against 30 ms. [Attempt 2, job 101447076381](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34017868081/job/101447076381) passed every benchmark on the same commit and unchanged budgets, but used `VM-7-113-ubuntu-ci-29` with Node 24.19.0. The gate runner suppressed successful child output, so that attempt supplies a passing verdict rather than raw medians. The changed runner and Node version prevent attributing the difference solely to contention or claiming stable repeated CI calibration; neither the budgets nor the shared scale are changed on this evidence. +Catalog uses an explicit 900 ms expected CI duration and only the existing 1.25× headroom, yielding 1,125 ms without applying the reference-machine scale again. The standard two-CPU hosted `ubuntu-24.04` [run 34033336380, job 101487280801](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336380/job/101487280801) reports five unchanged-catalog totals of 797.374, 883.157, 858.364, 790.569, and 904.579 ms: median 858.364 ms exceeds the historical 800 ms budget. The 320 ms M4 expectation above remains historical evidence, not a CI measurement. This follows the explicit-CI calibration used by Session reopening (50 ms expected CI); no shared factor, other scenario budget, workload, timing endpoint, or product implementation changes. Deterministic controls use the same assertion as the measured verdict: the unrounded recorded median passes 1,125 ms and fails 800 ms, while a synthetic 1,400 ms median fails 1,125 ms. A passing run on a faster host does not calibrate the standard hosted runner. + ## Alternatives considered **Repeat existing migration and first-open variants.** Rejected: those twelve cases already distinguish read-only preparation from writable publication. These cases use the current generation and begin or continue actual model work, or enumerate a corpus rather than open one Session. diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md index e07cd154d1..bba1b4fc8a 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md @@ -31,13 +31,13 @@ Status: implemented 实现参考为 Apple M4 Pro、macOS arm64、Node 24.19.0 上的 `925e012340f033f0521e802ba8569ce6dd7ef1ac`。两轮独占的五样本运行使用相同播种数据，没有产品优化。下表时间单位为毫秒；源码期望值向上取整至实测各轮中位数以上，而不是施加尚未实现的优化目标。 -| 用例 | 第一轮原始总时间 | 第二轮原始总时间 | 中位数 | 参考期望 | CI 预算 | +| 用例 | 第一轮原始总时间 | 第二轮原始总时间 | 中位数 | 历史 M4 期望 | 历史缩放预算 | |---|---|---|---|---:|---:| | 请求历史 | 209.134, 210.333, 208.959, 236.355, 238.685 | 222.833, 213.911, 208.089, 211.494, 209.137 | 210.333 / 211.494 | 220 | 550 | | 工具续聊 | 358.953, 324.790, 318.861, 320.119, 322.896 | 324.280, 321.952, 340.409, 325.470, 324.312 | 322.896 / 324.312 | 340 | 850 | | 子会话目录 | 318.730, 309.006, 311.404, 308.565, 310.105 | 308.670, 310.030, 280.086, 303.084, 284.829 | 310.105 / 303.084 | 320 | 800 | -续聊保留约 22.295 MiB；源码期望值为 23 MiB，预算为 28.75 MiB。时间期望值使用现有[校准辅助函数](../../../../benchmarks/support/calibration.ts)：2× 共享 CI 时间比例和 1.25× 波动余量。内存只使用 1.25× 余量。比例继承现有通道的校准，并非这些用例的新 Linux 实测值；runner 特征变化时仍需 CI 证据。基线预算保护实测实现；更紧预算属于有测量依据且保持行为的修复。 +续聊保留约 22.295 MiB；源码期望值为 23 MiB，预算为 28.75 MiB。请求历史、工具续聊及 SDK 时间期望值使用现有[校准辅助函数](../../../../benchmarks/support/calibration.ts)：2× 共享 CI 时间比例和 1.25× 波动余量。内存只使用 1.25× 余量。比例继承现有通道的校准，并非这些用例的新 Linux 实测值；runner 特征变化时仍需 CI 证据。基线预算保护实测实现；更紧预算属于有测量依据且保持行为的修复。 独立的纯 Node 请求历史 CPU profile 在一次 211.300 ms 操作中，将 132.876 ms 采样自身时间归因于 buildRequest 调用的 deepFreeze。这把重复遍历已冻结历史定位为聚焦调查目标，不是已证实的优化结果。目录首次／重复时间分别保留，因为观察释放后第二次列举仍读取带种子子会话的正文。 @@ -45,6 +45,8 @@ Status: implemented 提交 `1dc3296eba631d51fbb3bb50e249bf3cc0fce9f6` 的首次 Linux x64 CI 测量使用 `VM-7-113-ubuntu-ci-10` 和 Node 24.18.1（[run 34017868081，attempt 1，job 101444810498](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34017868081/job/101444810498)）。SDK 中位数为 2,753.441 ms，预算为 4,250 ms；工具续聊保留堆中位数为 22.274 MiB，预算为 28.75 MiB。请求历史与工具续聊时间预算失败：分别为 785.498 ms 对 550 ms、1,077.285 ms 对 850 ms。未修改的 Session 重开 open 阶段也以 31.6 ms 对 30 ms 失败。[Attempt 2，job 101447076381](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34017868081/job/101447076381) 在同一提交和未修改预算下通过全部基准，但使用 `VM-7-113-ubuntu-ci-29` 和 Node 24.19.0。门禁运行器隐藏成功子进程的输出，因此该次运行只提供通过结论，不提供原始中位数。Runner 与 Node 版本同时变化，不能把差异仅归因于资源争用，也不能宣称已获得稳定的重复 CI 校准；这些证据不改变预算或共享比例。 +目录用例使用显式的 900 ms CI 期望时间，仅乘现有 1.25× 余量，得到 1,125 ms，不再应用参考机器比例。标准双 CPU 托管 `ubuntu-24.04` 的 [run 34033336380，job 101487280801](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336380/job/101487280801) 报告未修改目录实现的五个总时间为 797.374、883.157、858.364、790.569 和 904.579 ms：中位数 858.364 ms 超出历史 800 ms 预算。上表 320 ms M4 期望保留为历史证据，不是 CI 测量。此方法与 Session 重开使用的显式 CI 校准一致（CI 期望为 50 ms）；共享系数、其他场景预算、负载、计时终点和产品实现均不改变。确定性对照与实测判定使用同一断言：未经舍入的录制中位数通过 1,125 ms 并被 800 ms 拒绝，合成的 1,400 ms 中位数则被 1,125 ms 拒绝。更快主机上的通过结果不能校准标准托管 runner。 + ## 考虑过的替代方案 **重复现有迁移和首次打开变体。** 拒绝：现有十二个用例已经区分只读准备与可写发布。这些用例使用当前代际并开始或继续实际模型工作，或者列举语料集合而不是打开单个 Session。 diff --git a/benchmarks/agent-continuation/README.i18n.yaml b/benchmarks/agent-continuation/README.i18n.yaml index 41fa944b7e..e93a36a019 100644 --- a/benchmarks/agent-continuation/README.i18n.yaml +++ b/benchmarks/agent-continuation/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write benchmarks/agent-continuation/README.md -README.md: 13fc21ea486bea55a93011a10edaca1cbe40ff47 -README.zh.md: f3f1fdcfb95f2948add2ebec7e1ea711172b7870 +README.md: 0544489a5a5af14d74b349926eaa3f0c0bc9580d +README.zh.md: fbe42a23502bdb72ca45b706dfdf345ee0690ca0 diff --git a/benchmarks/agent-continuation/README.md b/benchmarks/agent-continuation/README.md index 13fc21ea48..0544489a5a 100644 --- a/benchmarks/agent-continuation/README.md +++ b/benchmarks/agent-continuation/README.md @@ -18,7 +18,7 @@ Measure long-history request processing, cold tool-heavy continuation, and repea From the repository root, build the libraries and workers with `pnpm run build:bench`, then run `pnpm exec vitest run --config vitest.bench.config.ts benchmarks/agent-continuation/agent-continuation.bench.ts`. Do not overlap timing runs with builds or other benchmarks. -The test reports all five fresh-process samples and enforces reviewed median budgets. A failed worker reports its exit, signal, timeout, and stderr; temporary roots are removed even on failure. The required benchmark lane discovers this file automatically. +The test reports all five fresh-process samples and enforces reviewed median budgets. Catalog uses a 900 ms standard hosted CI expectation with 1.25× headroom (1,125 ms); other time budgets use reference-machine scaling. A failed worker reports its exit, signal, timeout, and stderr; temporary roots are removed even on failure. The required benchmark lane discovers this file automatically. diff --git a/benchmarks/agent-continuation/README.zh.md b/benchmarks/agent-continuation/README.zh.md index f3f1fdcfb9..fbe42a2350 100644 --- a/benchmarks/agent-continuation/README.zh.md +++ b/benchmarks/agent-continuation/README.zh.md @@ -18,7 +18,7 @@ 在仓库根目录使用 `pnpm run build:bench` 构建库和 worker，然后运行 `pnpm exec vitest run --config vitest.bench.config.ts benchmarks/agent-continuation/agent-continuation.bench.ts`。不要让计时运行与构建或其他基准重叠。 -测试报告全部五个新进程样本，并约束经审查的中位数预算。worker 失败时报告退出状态、信号、超时和 stderr；失败时也会删除临时根目录。必需基准通道自动发现此文件。 +测试报告全部五个新进程样本，并约束经审查的中位数预算。目录用例使用标准托管 CI 的 900 ms 期望值与 1.25× 余量（1,125 ms）；其他时间预算使用参考机器缩放。worker 失败时报告退出状态、信号、超时和 stderr；失败时也会删除临时根目录。必需基准通道自动发现此文件。 diff --git a/benchmarks/agent-continuation/agent-continuation.bench.ts b/benchmarks/agent-continuation/agent-continuation.bench.ts index 9fde897549..bdd2d859ef 100644 --- a/benchmarks/agent-continuation/agent-continuation.bench.ts +++ b/benchmarks/agent-continuation/agent-continuation.bench.ts @@ -14,11 +14,14 @@ import { WORKLOAD } from './workload.ts' const ATTEMPTS = 5 const WORKER_TIMEOUT_MS = 60_000 /** M4 Pro / Node 24.19 baseline expectations, before shared CI scaling and variance headroom. */ -const EXPECTED_MS = { 'request-history': 220, 'tool-continuation': 340, catalog: 320, 'profile-continuation': 1_700 } as const +const EXPECTED_MS = { 'request-history': 220, 'tool-continuation': 340, 'profile-continuation': 1_700 } as const +/** Standard two-CPU hosted CI catalog median is 858.364 ms; 900 ms is the rounded expectation. */ +const EXPECTED_CATALOG_CI_MS = 900 +const CATALOG_BUDGET_MS = Math.ceil(EXPECTED_CATALOG_CI_MS * PERFORMANCE_BUDGET_HEADROOM) const EXPECTED_RETAINED_HEAP_MB = 23 const WORKERS = join(import.meta.dirname, '..', '.dsh-build', 'agent-continuation') -type Scenario = keyof typeof EXPECTED_MS +type Scenario = keyof typeof EXPECTED_MS | 'catalog' type Report = ContinuationReport | CatalogReport | ProfileReport function workerName(scenario: Scenario): string { @@ -41,6 +44,28 @@ function median(values: readonly number[]): number { return [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)] as number } +function expectTotalWithinBudget(value: number, budget: number): void { + expect(value).toBeLessThanOrEqual(budget) +} + +describe('standard hosted catalog calibration', () => { + it('accepts the recorded two-CPU samples that exceed the historical budget', () => { + const recordedMedian = median([797.373945, 883.157358, 858.363927, 790.568538, 904.5785669999999]) + + expect(recordedMedian).toBe(858.363927) + expect(() => expectTotalWithinBudget(recordedMedian, 800)).toThrow() + expectTotalWithinBudget(recordedMedian, CATALOG_BUDGET_MS) + expect(CATALOG_BUDGET_MS).toBe(1_125) + }) + + it('rejects a synthetic material catalog regression', () => { + const regressionMedian = median([1_380, 1_400, 1_420, 1_410, 1_390]) + + expect(regressionMedian).toBe(1_400) + expect(() => expectTotalWithinBudget(regressionMedian, CATALOG_BUDGET_MS)).toThrow() + }) +}) + describe('continuing tool-heavy Sessions with large histories', () => { let scratch: string | undefined const sources = new Map() @@ -69,14 +94,14 @@ describe('continuing tool-heavy Sessions with large histories', () => { finally { await rm(root, { recursive: true, force: true }) } } const totalMs = samples.map(sample => sample.totalMs) - const budgetMs = ciTimeBudget(EXPECTED_MS[scenario]) + const budgetMs = scenario === 'catalog' ? CATALOG_BUDGET_MS : ciTimeBudget(EXPECTED_MS[scenario]) const retainedHeapBudgetMb = EXPECTED_RETAINED_HEAP_MB * PERFORMANCE_BUDGET_HEADROOM console.log(JSON.stringify({ benchmark: 'agent-continuation/' + scenario, workload: WORKLOAD, samples, totalMs: { min: Math.min(...totalMs), median: median(totalMs), max: Math.max(...totalMs) }, budgetMs, ...(scenario === 'tool-continuation' ? { retainedHeapBudgetMb } : {}), })) - expect(median(totalMs)).toBeLessThanOrEqual(budgetMs) + expectTotalWithinBudget(median(totalMs), budgetMs) if (scenario === 'tool-continuation') { expect(median((samples as ContinuationReport[]).map(sample => sample.retainedHeapMb))) .toBeLessThanOrEqual(retainedHeapBudgetMb) From 2f0088357d09df68b5577ca12b43f70b612eb892 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 21:03:59 +0800 Subject: [PATCH 05/22] test(perf): calibrate tool continuation for hosted CI --- ...backend-continuation-performance.i18n.yaml | 4 ++-- ...-09-06-backend-continuation-performance.md | 6 +++-- ...-06-backend-continuation-performance.zh.md | 6 +++-- .../agent-continuation/README.i18n.yaml | 4 ++-- benchmarks/agent-continuation/README.md | 2 +- benchmarks/agent-continuation/README.zh.md | 2 +- .../agent-continuation.bench.ts | 22 ++++++++++++++++--- 7 files changed, 33 insertions(+), 13 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml index 031e846f05..72b2e73a3b 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md -2026-09-06-backend-continuation-performance.md: c247c7b9c4e2612603e0fb04b4f1a03b80314407 -2026-09-06-backend-continuation-performance.zh.md: bba1b4fc8a742ee261be6d09f61dd754568ebe0b +2026-09-06-backend-continuation-performance.md: 619b583d3d0c9f035b2e78012d1a138adcc4b9f2 +2026-09-06-backend-continuation-performance.zh.md: 2008480ef4d51d1c276776043787a50aaea4e611 diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md index c247c7b9c4..619b583d3d 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md @@ -37,7 +37,7 @@ The implementation reference is `925e012340f033f0521e802ba8569ce6dd7ef1ac` on Ap | Tool continuation | 358.953, 324.790, 318.861, 320.119, 322.896 | 324.280, 321.952, 340.409, 325.470, 324.312 | 322.896 / 324.312 | 340 | 850 | | Child catalog | 318.730, 309.006, 311.404, 308.565, 310.105 | 308.670, 310.030, 280.086, 303.084, 284.829 | 310.105 / 303.084 | 320 | 800 | -Continuation retains approximately 22.295 MiB; its source expectation is 23 MiB and its budget is 28.75 MiB. Request-history, tool-continuation, and SDK time expectations use the existing [calibration helper](../../../../benchmarks/support/calibration.ts): 2× shared CI time scale and 1.25× variance headroom. Memory uses only 1.25× headroom. The scale is inherited from the existing lane's calibration, not a new Linux measurement of these cases; CI evidence remains necessary when runner characteristics change. Baseline budgets protect the measured implementation; tighter budgets belong with a measured behavior-preserving fix. +Continuation retains approximately 22.295 MiB; its source expectation is 23 MiB and its budget is 28.75 MiB. Request-history and SDK time expectations use the existing [calibration helper](../../../../benchmarks/support/calibration.ts): 2× shared CI time scale and 1.25× variance headroom. Memory uses only 1.25× headroom. The scale is inherited from the existing lane's calibration, not a new Linux measurement of these cases; CI evidence remains necessary when runner characteristics change. Baseline budgets protect the measured implementation; tighter budgets belong with a measured behavior-preserving fix. A separate plain-Node request-history CPU profile attributes 132.876 ms of sampled self time to deepFreeze called by buildRequest during a 211.300 ms operation. This identifies repeated traversal of already-frozen history as a focused investigation target, not a proven optimization result. Catalog first/repeat timings remain separate because a second listing still reads body-bearing seeded children after observations are released. @@ -45,7 +45,9 @@ The shipped SDK variant completes 100 turns, 200 requests, and 800 real file rea The first Linux x64 CI measurement at commit `1dc3296eba631d51fbb3bb50e249bf3cc0fce9f6` ran on `VM-7-113-ubuntu-ci-10` with Node 24.18.1 ([run 34017868081, attempt 1, job 101444810498](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34017868081/job/101444810498)). The SDK median was 2,753.441 ms against its 4,250 ms budget, and tool-continuation retained-heap median was 22.274 MiB against 28.75 MiB. Request-history and tool-continuation time budgets failed: 785.498 ms against 550 ms and 1,077.285 ms against 850 ms, respectively. The unchanged Session-reopen open phase also failed at 31.6 ms against 30 ms. [Attempt 2, job 101447076381](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34017868081/job/101447076381) passed every benchmark on the same commit and unchanged budgets, but used `VM-7-113-ubuntu-ci-29` with Node 24.19.0. The gate runner suppressed successful child output, so that attempt supplies a passing verdict rather than raw medians. The changed runner and Node version prevent attributing the difference solely to contention or claiming stable repeated CI calibration; neither the budgets nor the shared scale are changed on this evidence. -Catalog uses an explicit 900 ms expected CI duration and only the existing 1.25× headroom, yielding 1,125 ms without applying the reference-machine scale again. The standard two-CPU hosted `ubuntu-24.04` [run 34033336380, job 101487280801](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336380/job/101487280801) reports five unchanged-catalog totals of 797.374, 883.157, 858.364, 790.569, and 904.579 ms: median 858.364 ms exceeds the historical 800 ms budget. The 320 ms M4 expectation above remains historical evidence, not a CI measurement. This follows the explicit-CI calibration used by Session reopening (50 ms expected CI); no shared factor, other scenario budget, workload, timing endpoint, or product implementation changes. Deterministic controls use the same assertion as the measured verdict: the unrounded recorded median passes 1,125 ms and fails 800 ms, while a synthetic 1,400 ms median fails 1,125 ms. A passing run on a faster host does not calibrate the standard hosted runner. +Catalog uses an explicit 900 ms expected CI duration and only the existing 1.25× headroom, yielding 1,125 ms without applying the reference-machine scale again. The standard two-CPU hosted `ubuntu-24.04` [run 34033336380, job 101487280801](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336380/job/101487280801) reports five unchanged-catalog totals of 797.374, 883.157, 858.364, 790.569, and 904.579 ms: median 858.364 ms exceeds the historical 800 ms budget. The 320 ms M4 expectation above remains historical evidence, not a CI measurement. This follows the explicit-CI calibration used by Session reopening (50 ms expected CI); shared factors, workloads, timing endpoints, and product implementations remain unchanged. Deterministic controls use the same assertion as the measured verdict: the unrounded recorded median passes 1,125 ms and fails 800 ms, while a synthetic 1,400 ms median fails 1,125 ms. A passing run on a faster host does not calibrate the standard hosted runner. + +Tool continuation also uses a 900 ms expected CI duration with 1.25× headroom (1,125 ms). At unchanged implementation `79c052ab29`, standard two-CPU hosted [run 34034524265, job 101490056074](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34034524265/job/101490056074) reports totals of 917.007, 892.091, 887.839, 905.659, and 898.252 ms: median 898.252 ms exceeds the historical 850 ms budget. The 340 ms M4 expectation remains historical evidence. The same measured-verdict assertion accepts the recorded unrounded median under 1,125 ms, rejects it under 850 ms, and rejects a synthetic 1,400 ms regression. Workload, timing, product code, and the 28.75 MiB retained-heap budget remain unchanged. ## Alternatives considered diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md index bba1b4fc8a..2008480ef4 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md @@ -37,7 +37,7 @@ Status: implemented | 工具续聊 | 358.953, 324.790, 318.861, 320.119, 322.896 | 324.280, 321.952, 340.409, 325.470, 324.312 | 322.896 / 324.312 | 340 | 850 | | 子会话目录 | 318.730, 309.006, 311.404, 308.565, 310.105 | 308.670, 310.030, 280.086, 303.084, 284.829 | 310.105 / 303.084 | 320 | 800 | -续聊保留约 22.295 MiB；源码期望值为 23 MiB，预算为 28.75 MiB。请求历史、工具续聊及 SDK 时间期望值使用现有[校准辅助函数](../../../../benchmarks/support/calibration.ts)：2× 共享 CI 时间比例和 1.25× 波动余量。内存只使用 1.25× 余量。比例继承现有通道的校准，并非这些用例的新 Linux 实测值；runner 特征变化时仍需 CI 证据。基线预算保护实测实现；更紧预算属于有测量依据且保持行为的修复。 +续聊保留约 22.295 MiB；源码期望值为 23 MiB，预算为 28.75 MiB。请求历史及 SDK 时间期望值使用现有[校准辅助函数](../../../../benchmarks/support/calibration.ts)：2× 共享 CI 时间比例和 1.25× 波动余量。内存只使用 1.25× 余量。比例继承现有通道的校准，并非这些用例的新 Linux 实测值；runner 特征变化时仍需 CI 证据。基线预算保护实测实现；更紧预算属于有测量依据且保持行为的修复。 独立的纯 Node 请求历史 CPU profile 在一次 211.300 ms 操作中，将 132.876 ms 采样自身时间归因于 buildRequest 调用的 deepFreeze。这把重复遍历已冻结历史定位为聚焦调查目标，不是已证实的优化结果。目录首次／重复时间分别保留，因为观察释放后第二次列举仍读取带种子子会话的正文。 @@ -45,7 +45,9 @@ Status: implemented 提交 `1dc3296eba631d51fbb3bb50e249bf3cc0fce9f6` 的首次 Linux x64 CI 测量使用 `VM-7-113-ubuntu-ci-10` 和 Node 24.18.1（[run 34017868081，attempt 1，job 101444810498](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34017868081/job/101444810498)）。SDK 中位数为 2,753.441 ms，预算为 4,250 ms；工具续聊保留堆中位数为 22.274 MiB，预算为 28.75 MiB。请求历史与工具续聊时间预算失败：分别为 785.498 ms 对 550 ms、1,077.285 ms 对 850 ms。未修改的 Session 重开 open 阶段也以 31.6 ms 对 30 ms 失败。[Attempt 2，job 101447076381](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34017868081/job/101447076381) 在同一提交和未修改预算下通过全部基准，但使用 `VM-7-113-ubuntu-ci-29` 和 Node 24.19.0。门禁运行器隐藏成功子进程的输出，因此该次运行只提供通过结论，不提供原始中位数。Runner 与 Node 版本同时变化，不能把差异仅归因于资源争用，也不能宣称已获得稳定的重复 CI 校准；这些证据不改变预算或共享比例。 -目录用例使用显式的 900 ms CI 期望时间，仅乘现有 1.25× 余量，得到 1,125 ms，不再应用参考机器比例。标准双 CPU 托管 `ubuntu-24.04` 的 [run 34033336380，job 101487280801](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336380/job/101487280801) 报告未修改目录实现的五个总时间为 797.374、883.157、858.364、790.569 和 904.579 ms：中位数 858.364 ms 超出历史 800 ms 预算。上表 320 ms M4 期望保留为历史证据，不是 CI 测量。此方法与 Session 重开使用的显式 CI 校准一致（CI 期望为 50 ms）；共享系数、其他场景预算、负载、计时终点和产品实现均不改变。确定性对照与实测判定使用同一断言：未经舍入的录制中位数通过 1,125 ms 并被 800 ms 拒绝，合成的 1,400 ms 中位数则被 1,125 ms 拒绝。更快主机上的通过结果不能校准标准托管 runner。 +目录用例使用显式的 900 ms CI 期望时间，仅乘现有 1.25× 余量，得到 1,125 ms，不再应用参考机器比例。标准双 CPU 托管 `ubuntu-24.04` 的 [run 34033336380，job 101487280801](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336380/job/101487280801) 报告未修改目录实现的五个总时间为 797.374、883.157、858.364、790.569 和 904.579 ms：中位数 858.364 ms 超出历史 800 ms 预算。上表 320 ms M4 期望保留为历史证据，不是 CI 测量。此方法与 Session 重开使用的显式 CI 校准一致（CI 期望为 50 ms）；共享系数、负载、计时终点和产品实现均不改变。确定性对照与实测判定使用同一断言：未经舍入的录制中位数通过 1,125 ms 并被 800 ms 拒绝，合成的 1,400 ms 中位数则被 1,125 ms 拒绝。更快主机上的通过结果不能校准标准托管 runner。 + +工具续聊同样使用 900 ms CI 期望时间与 1.25× 余量（1,125 ms）。未修改实现的 `79c052ab29` 在标准双 CPU 托管 [run 34034524265，job 101490056074](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34034524265/job/101490056074) 中报告总时间为 917.007、892.091、887.839、905.659 和 898.252 ms：中位数 898.252 ms 超出历史 850 ms 预算。340 ms M4 期望保留为历史证据。与实测判定相同的断言在 1,125 ms 下接受未经舍入的录制中位数，在 850 ms 下拒绝它，并拒绝合成的 1,400 ms 回退。负载、计时、产品代码和 28.75 MiB 保留堆预算均不改变。 ## 考虑过的替代方案 diff --git a/benchmarks/agent-continuation/README.i18n.yaml b/benchmarks/agent-continuation/README.i18n.yaml index e93a36a019..286c34415c 100644 --- a/benchmarks/agent-continuation/README.i18n.yaml +++ b/benchmarks/agent-continuation/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write benchmarks/agent-continuation/README.md -README.md: 0544489a5a5af14d74b349926eaa3f0c0bc9580d -README.zh.md: fbe42a23502bdb72ca45b706dfdf345ee0690ca0 +README.md: 7c5ed357e2089f6d7238667b3b8310570983c77c +README.zh.md: bcbbff353e9faf5e181e1e75896200363f5c626b diff --git a/benchmarks/agent-continuation/README.md b/benchmarks/agent-continuation/README.md index 0544489a5a..7c5ed357e2 100644 --- a/benchmarks/agent-continuation/README.md +++ b/benchmarks/agent-continuation/README.md @@ -18,7 +18,7 @@ Measure long-history request processing, cold tool-heavy continuation, and repea From the repository root, build the libraries and workers with `pnpm run build:bench`, then run `pnpm exec vitest run --config vitest.bench.config.ts benchmarks/agent-continuation/agent-continuation.bench.ts`. Do not overlap timing runs with builds or other benchmarks. -The test reports all five fresh-process samples and enforces reviewed median budgets. Catalog uses a 900 ms standard hosted CI expectation with 1.25× headroom (1,125 ms); other time budgets use reference-machine scaling. A failed worker reports its exit, signal, timeout, and stderr; temporary roots are removed even on failure. The required benchmark lane discovers this file automatically. +The test reports all five fresh-process samples and enforces reviewed median budgets. Catalog and tool continuation each use a 900 ms standard hosted CI expectation with 1.25× headroom (1,125 ms); other time budgets use reference-machine scaling. A failed worker reports its exit, signal, timeout, and stderr; temporary roots are removed even on failure. The required benchmark lane discovers this file automatically. diff --git a/benchmarks/agent-continuation/README.zh.md b/benchmarks/agent-continuation/README.zh.md index fbe42a2350..bcbbff353e 100644 --- a/benchmarks/agent-continuation/README.zh.md +++ b/benchmarks/agent-continuation/README.zh.md @@ -18,7 +18,7 @@ 在仓库根目录使用 `pnpm run build:bench` 构建库和 worker，然后运行 `pnpm exec vitest run --config vitest.bench.config.ts benchmarks/agent-continuation/agent-continuation.bench.ts`。不要让计时运行与构建或其他基准重叠。 -测试报告全部五个新进程样本，并约束经审查的中位数预算。目录用例使用标准托管 CI 的 900 ms 期望值与 1.25× 余量（1,125 ms）；其他时间预算使用参考机器缩放。worker 失败时报告退出状态、信号、超时和 stderr；失败时也会删除临时根目录。必需基准通道自动发现此文件。 +测试报告全部五个新进程样本，并约束经审查的中位数预算。目录和工具续聊用例均使用标准托管 CI 的 900 ms 期望值与 1.25× 余量（1,125 ms）；其他时间预算使用参考机器缩放。worker 失败时报告退出状态、信号、超时和 stderr；失败时也会删除临时根目录。必需基准通道自动发现此文件。 diff --git a/benchmarks/agent-continuation/agent-continuation.bench.ts b/benchmarks/agent-continuation/agent-continuation.bench.ts index bdd2d859ef..6545b6039f 100644 --- a/benchmarks/agent-continuation/agent-continuation.bench.ts +++ b/benchmarks/agent-continuation/agent-continuation.bench.ts @@ -14,14 +14,17 @@ import { WORKLOAD } from './workload.ts' const ATTEMPTS = 5 const WORKER_TIMEOUT_MS = 60_000 /** M4 Pro / Node 24.19 baseline expectations, before shared CI scaling and variance headroom. */ -const EXPECTED_MS = { 'request-history': 220, 'tool-continuation': 340, 'profile-continuation': 1_700 } as const +const EXPECTED_MS = { 'request-history': 220, 'profile-continuation': 1_700 } as const +/** Standard two-CPU hosted CI tool-continuation median is 898.252 ms. */ +const EXPECTED_TOOL_CONTINUATION_CI_MS = 900 +const TOOL_CONTINUATION_BUDGET_MS = Math.ceil(EXPECTED_TOOL_CONTINUATION_CI_MS * PERFORMANCE_BUDGET_HEADROOM) /** Standard two-CPU hosted CI catalog median is 858.364 ms; 900 ms is the rounded expectation. */ const EXPECTED_CATALOG_CI_MS = 900 const CATALOG_BUDGET_MS = Math.ceil(EXPECTED_CATALOG_CI_MS * PERFORMANCE_BUDGET_HEADROOM) const EXPECTED_RETAINED_HEAP_MB = 23 const WORKERS = join(import.meta.dirname, '..', '.dsh-build', 'agent-continuation') -type Scenario = keyof typeof EXPECTED_MS | 'catalog' +type Scenario = keyof typeof EXPECTED_MS | 'catalog' | 'tool-continuation' type Report = ContinuationReport | CatalogReport | ProfileReport function workerName(scenario: Scenario): string { @@ -66,6 +69,18 @@ describe('standard hosted catalog calibration', () => { }) }) +describe('standard hosted tool-continuation calibration', () => { + it('accepts recorded two-CPU samples but rejects a material regression', () => { + const recordedMedian = median([917.006744, 892.091482, 887.838867, 905.6594390000001, 898.2517579999999]) + + expect(recordedMedian).toBe(898.2517579999999) + expect(() => expectTotalWithinBudget(recordedMedian, 850)).toThrow() + expectTotalWithinBudget(recordedMedian, TOOL_CONTINUATION_BUDGET_MS) + expect(TOOL_CONTINUATION_BUDGET_MS).toBe(1_125) + expect(() => expectTotalWithinBudget(1_400, TOOL_CONTINUATION_BUDGET_MS)).toThrow() + }) +}) + describe('continuing tool-heavy Sessions with large histories', () => { let scratch: string | undefined const sources = new Map() @@ -94,7 +109,8 @@ describe('continuing tool-heavy Sessions with large histories', () => { finally { await rm(root, { recursive: true, force: true }) } } const totalMs = samples.map(sample => sample.totalMs) - const budgetMs = scenario === 'catalog' ? CATALOG_BUDGET_MS : ciTimeBudget(EXPECTED_MS[scenario]) + const budgetMs = scenario === 'catalog' ? CATALOG_BUDGET_MS + : scenario === 'tool-continuation' ? TOOL_CONTINUATION_BUDGET_MS : ciTimeBudget(EXPECTED_MS[scenario]) const retainedHeapBudgetMb = EXPECTED_RETAINED_HEAP_MB * PERFORMANCE_BUDGET_HEADROOM console.log(JSON.stringify({ benchmark: 'agent-continuation/' + scenario, workload: WORKLOAD, From 84914c316dfed4722e1b861f10d7fb172b66833e Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 21:19:25 +0800 Subject: [PATCH 06/22] test(perf): calibrate baseline requests for hosted CI --- ...backend-continuation-performance.i18n.yaml | 4 ++-- ...-09-06-backend-continuation-performance.md | 4 +++- ...-06-backend-continuation-performance.zh.md | 4 +++- .../agent-continuation/README.i18n.yaml | 4 ++-- benchmarks/agent-continuation/README.md | 2 +- benchmarks/agent-continuation/README.zh.md | 2 +- .../agent-continuation.bench.ts | 22 ++++++++++++++++--- 7 files changed, 31 insertions(+), 11 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml index 72b2e73a3b..03917f21fd 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md -2026-09-06-backend-continuation-performance.md: 619b583d3d0c9f035b2e78012d1a138adcc4b9f2 -2026-09-06-backend-continuation-performance.zh.md: 2008480ef4d51d1c276776043787a50aaea4e611 +2026-09-06-backend-continuation-performance.md: f8dddda687185d6b6504cd2423d307df02db9dfe +2026-09-06-backend-continuation-performance.zh.md: 2e87e9157f6a6a834cfcef42c3f35b9f42199e6d diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md index 619b583d3d..f8dddda687 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md @@ -37,7 +37,7 @@ The implementation reference is `925e012340f033f0521e802ba8569ce6dd7ef1ac` on Ap | Tool continuation | 358.953, 324.790, 318.861, 320.119, 322.896 | 324.280, 321.952, 340.409, 325.470, 324.312 | 322.896 / 324.312 | 340 | 850 | | Child catalog | 318.730, 309.006, 311.404, 308.565, 310.105 | 308.670, 310.030, 280.086, 303.084, 284.829 | 310.105 / 303.084 | 320 | 800 | -Continuation retains approximately 22.295 MiB; its source expectation is 23 MiB and its budget is 28.75 MiB. Request-history and SDK time expectations use the existing [calibration helper](../../../../benchmarks/support/calibration.ts): 2× shared CI time scale and 1.25× variance headroom. Memory uses only 1.25× headroom. The scale is inherited from the existing lane's calibration, not a new Linux measurement of these cases; CI evidence remains necessary when runner characteristics change. Baseline budgets protect the measured implementation; tighter budgets belong with a measured behavior-preserving fix. +Continuation retains approximately 22.295 MiB; its source expectation is 23 MiB and its budget is 28.75 MiB. The SDK time expectation uses the existing [calibration helper](../../../../benchmarks/support/calibration.ts): 2× shared CI time scale and 1.25× variance headroom. Memory uses only 1.25× headroom. The scale is inherited from the existing lane's calibration, not a new Linux measurement of these cases; CI evidence remains necessary when runner characteristics change. Baseline budgets protect the measured implementation; tighter budgets belong with a measured behavior-preserving fix. A separate plain-Node request-history CPU profile attributes 132.876 ms of sampled self time to deepFreeze called by buildRequest during a 211.300 ms operation. This identifies repeated traversal of already-frozen history as a focused investigation target, not a proven optimization result. Catalog first/repeat timings remain separate because a second listing still reads body-bearing seeded children after observations are released. @@ -49,6 +49,8 @@ Catalog uses an explicit 900 ms expected CI duration and only the existing 1.25 Tool continuation also uses a 900 ms expected CI duration with 1.25× headroom (1,125 ms). At unchanged implementation `79c052ab29`, standard two-CPU hosted [run 34034524265, job 101490056074](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34034524265/job/101490056074) reports totals of 917.007, 892.091, 887.839, 905.659, and 898.252 ms: median 898.252 ms exceeds the historical 850 ms budget. The 340 ms M4 expectation remains historical evidence. The same measured-verdict assertion accepts the recorded unrounded median under 1,125 ms, rejects it under 850 ms, and rejects a synthetic 1,400 ms regression. Workload, timing, product code, and the 28.75 MiB retained-heap budget remain unchanged. +Baseline request history uses a 600 ms expected CI duration with 1.25× headroom (750 ms). At unchanged implementation `54d1190a75`, standard two-CPU hosted [run 34035306987, job 101492163630](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34035306987/job/101492163630) reports totals of 618.598, 618.606, 582.035, 582.304, and 581.832 ms: median 582.304 ms exceeds the historical 550 ms budget. The 220 ms M4 expectation remains historical evidence. The same measured-verdict assertion accepts the recorded unrounded median under 750 ms, rejects it under 550 ms, and rejects a synthetic 900 ms regression. This calibrates the unoptimized baseline only; workload, timing, product code, and memory budgets remain unchanged. + ## Alternatives considered **Repeat existing migration and first-open variants.** Rejected: those twelve cases already distinguish read-only preparation from writable publication. These cases use the current generation and begin or continue actual model work, or enumerate a corpus rather than open one Session. diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md index 2008480ef4..2e87e9157f 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md @@ -37,7 +37,7 @@ Status: implemented | 工具续聊 | 358.953, 324.790, 318.861, 320.119, 322.896 | 324.280, 321.952, 340.409, 325.470, 324.312 | 322.896 / 324.312 | 340 | 850 | | 子会话目录 | 318.730, 309.006, 311.404, 308.565, 310.105 | 308.670, 310.030, 280.086, 303.084, 284.829 | 310.105 / 303.084 | 320 | 800 | -续聊保留约 22.295 MiB；源码期望值为 23 MiB，预算为 28.75 MiB。请求历史及 SDK 时间期望值使用现有[校准辅助函数](../../../../benchmarks/support/calibration.ts)：2× 共享 CI 时间比例和 1.25× 波动余量。内存只使用 1.25× 余量。比例继承现有通道的校准，并非这些用例的新 Linux 实测值；runner 特征变化时仍需 CI 证据。基线预算保护实测实现；更紧预算属于有测量依据且保持行为的修复。 +续聊保留约 22.295 MiB；源码期望值为 23 MiB，预算为 28.75 MiB。SDK 时间期望值使用现有[校准辅助函数](../../../../benchmarks/support/calibration.ts)：2× 共享 CI 时间比例和 1.25× 波动余量。内存只使用 1.25× 余量。比例继承现有通道的校准，并非这些用例的新 Linux 实测值；runner 特征变化时仍需 CI 证据。基线预算保护实测实现；更紧预算属于有测量依据且保持行为的修复。 独立的纯 Node 请求历史 CPU profile 在一次 211.300 ms 操作中，将 132.876 ms 采样自身时间归因于 buildRequest 调用的 deepFreeze。这把重复遍历已冻结历史定位为聚焦调查目标，不是已证实的优化结果。目录首次／重复时间分别保留，因为观察释放后第二次列举仍读取带种子子会话的正文。 @@ -49,6 +49,8 @@ Status: implemented 工具续聊同样使用 900 ms CI 期望时间与 1.25× 余量（1,125 ms）。未修改实现的 `79c052ab29` 在标准双 CPU 托管 [run 34034524265，job 101490056074](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34034524265/job/101490056074) 中报告总时间为 917.007、892.091、887.839、905.659 和 898.252 ms：中位数 898.252 ms 超出历史 850 ms 预算。340 ms M4 期望保留为历史证据。与实测判定相同的断言在 1,125 ms 下接受未经舍入的录制中位数，在 850 ms 下拒绝它，并拒绝合成的 1,400 ms 回退。负载、计时、产品代码和 28.75 MiB 保留堆预算均不改变。 +基线请求历史使用 600 ms CI 期望时间与 1.25× 余量（750 ms）。未修改实现的 `54d1190a75` 在标准双 CPU 托管 [run 34035306987，job 101492163630](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34035306987/job/101492163630) 中报告总时间为 618.598、618.606、582.035、582.304 和 581.832 ms：中位数 582.304 ms 超出历史 550 ms 预算。220 ms M4 期望保留为历史证据。与实测判定相同的断言在 750 ms 下接受未经舍入的录制中位数，在 550 ms 下拒绝它，并拒绝合成的 900 ms 回退。此校准仅针对未优化基线；负载、计时、产品代码和内存预算均不改变。 + ## 考虑过的替代方案 **重复现有迁移和首次打开变体。** 拒绝：现有十二个用例已经区分只读准备与可写发布。这些用例使用当前代际并开始或继续实际模型工作，或者列举语料集合而不是打开单个 Session。 diff --git a/benchmarks/agent-continuation/README.i18n.yaml b/benchmarks/agent-continuation/README.i18n.yaml index 286c34415c..e6fb5cd78c 100644 --- a/benchmarks/agent-continuation/README.i18n.yaml +++ b/benchmarks/agent-continuation/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write benchmarks/agent-continuation/README.md -README.md: 7c5ed357e2089f6d7238667b3b8310570983c77c -README.zh.md: bcbbff353e9faf5e181e1e75896200363f5c626b +README.md: 9854de19c3465c9ed30fcce8d80e8d7b3ef3d864 +README.zh.md: 91904419b0514b8f47c230e6748109c4be570ef7 diff --git a/benchmarks/agent-continuation/README.md b/benchmarks/agent-continuation/README.md index 7c5ed357e2..9854de19c3 100644 --- a/benchmarks/agent-continuation/README.md +++ b/benchmarks/agent-continuation/README.md @@ -18,7 +18,7 @@ Measure long-history request processing, cold tool-heavy continuation, and repea From the repository root, build the libraries and workers with `pnpm run build:bench`, then run `pnpm exec vitest run --config vitest.bench.config.ts benchmarks/agent-continuation/agent-continuation.bench.ts`. Do not overlap timing runs with builds or other benchmarks. -The test reports all five fresh-process samples and enforces reviewed median budgets. Catalog and tool continuation each use a 900 ms standard hosted CI expectation with 1.25× headroom (1,125 ms); other time budgets use reference-machine scaling. A failed worker reports its exit, signal, timeout, and stderr; temporary roots are removed even on failure. The required benchmark lane discovers this file automatically. +The test reports all five fresh-process samples and enforces reviewed median budgets. Catalog and tool continuation each use a 900 ms standard hosted CI expectation with 1.25× headroom (1,125 ms); baseline request history uses 600 ms with the same headroom (750 ms). The SDK time budget uses reference-machine scaling. A failed worker reports its exit, signal, timeout, and stderr; temporary roots are removed even on failure. The required benchmark lane discovers this file automatically. diff --git a/benchmarks/agent-continuation/README.zh.md b/benchmarks/agent-continuation/README.zh.md index bcbbff353e..91904419b0 100644 --- a/benchmarks/agent-continuation/README.zh.md +++ b/benchmarks/agent-continuation/README.zh.md @@ -18,7 +18,7 @@ 在仓库根目录使用 `pnpm run build:bench` 构建库和 worker，然后运行 `pnpm exec vitest run --config vitest.bench.config.ts benchmarks/agent-continuation/agent-continuation.bench.ts`。不要让计时运行与构建或其他基准重叠。 -测试报告全部五个新进程样本，并约束经审查的中位数预算。目录和工具续聊用例均使用标准托管 CI 的 900 ms 期望值与 1.25× 余量（1,125 ms）；其他时间预算使用参考机器缩放。worker 失败时报告退出状态、信号、超时和 stderr；失败时也会删除临时根目录。必需基准通道自动发现此文件。 +测试报告全部五个新进程样本，并约束经审查的中位数预算。目录和工具续聊用例均使用标准托管 CI 的 900 ms 期望值与 1.25× 余量（1,125 ms）；基线请求历史使用 600 ms 与相同余量（750 ms）。SDK 时间预算使用参考机器缩放。worker 失败时报告退出状态、信号、超时和 stderr；失败时也会删除临时根目录。必需基准通道自动发现此文件。 diff --git a/benchmarks/agent-continuation/agent-continuation.bench.ts b/benchmarks/agent-continuation/agent-continuation.bench.ts index 6545b6039f..ed681c2ce6 100644 --- a/benchmarks/agent-continuation/agent-continuation.bench.ts +++ b/benchmarks/agent-continuation/agent-continuation.bench.ts @@ -14,7 +14,10 @@ import { WORKLOAD } from './workload.ts' const ATTEMPTS = 5 const WORKER_TIMEOUT_MS = 60_000 /** M4 Pro / Node 24.19 baseline expectations, before shared CI scaling and variance headroom. */ -const EXPECTED_MS = { 'request-history': 220, 'profile-continuation': 1_700 } as const +const EXPECTED_MS = { 'profile-continuation': 1_700 } as const +/** Standard two-CPU hosted CI baseline request-history median is 582.304 ms. */ +const EXPECTED_BASELINE_REQUEST_CI_MS = 600 +const BASELINE_REQUEST_BUDGET_MS = Math.ceil(EXPECTED_BASELINE_REQUEST_CI_MS * PERFORMANCE_BUDGET_HEADROOM) /** Standard two-CPU hosted CI tool-continuation median is 898.252 ms. */ const EXPECTED_TOOL_CONTINUATION_CI_MS = 900 const TOOL_CONTINUATION_BUDGET_MS = Math.ceil(EXPECTED_TOOL_CONTINUATION_CI_MS * PERFORMANCE_BUDGET_HEADROOM) @@ -24,7 +27,7 @@ const CATALOG_BUDGET_MS = Math.ceil(EXPECTED_CATALOG_CI_MS * PERFORMANCE_BUDGET_ const EXPECTED_RETAINED_HEAP_MB = 23 const WORKERS = join(import.meta.dirname, '..', '.dsh-build', 'agent-continuation') -type Scenario = keyof typeof EXPECTED_MS | 'catalog' | 'tool-continuation' +type Scenario = keyof typeof EXPECTED_MS | 'catalog' | 'tool-continuation' | 'request-history' type Report = ContinuationReport | CatalogReport | ProfileReport function workerName(scenario: Scenario): string { @@ -81,6 +84,18 @@ describe('standard hosted tool-continuation calibration', () => { }) }) +describe('standard hosted baseline request-history calibration', () => { + it('accepts recorded two-CPU samples but rejects a material regression', () => { + const recordedMedian = median([618.598065, 618.606407, 582.0351149999999, 582.303506, 581.8318300000001]) + + expect(recordedMedian).toBe(582.303506) + expect(() => expectTotalWithinBudget(recordedMedian, 550)).toThrow() + expectTotalWithinBudget(recordedMedian, BASELINE_REQUEST_BUDGET_MS) + expect(BASELINE_REQUEST_BUDGET_MS).toBe(750) + expect(() => expectTotalWithinBudget(900, BASELINE_REQUEST_BUDGET_MS)).toThrow() + }) +}) + describe('continuing tool-heavy Sessions with large histories', () => { let scratch: string | undefined const sources = new Map() @@ -110,7 +125,8 @@ describe('continuing tool-heavy Sessions with large histories', () => { } const totalMs = samples.map(sample => sample.totalMs) const budgetMs = scenario === 'catalog' ? CATALOG_BUDGET_MS - : scenario === 'tool-continuation' ? TOOL_CONTINUATION_BUDGET_MS : ciTimeBudget(EXPECTED_MS[scenario]) + : scenario === 'tool-continuation' ? TOOL_CONTINUATION_BUDGET_MS + : scenario === 'request-history' ? BASELINE_REQUEST_BUDGET_MS : ciTimeBudget(EXPECTED_MS[scenario]) const retainedHeapBudgetMb = EXPECTED_RETAINED_HEAP_MB * PERFORMANCE_BUDGET_HEADROOM console.log(JSON.stringify({ benchmark: 'agent-continuation/' + scenario, workload: WORKLOAD, From 73edce1ae7ad0cbf8813d4d65b288317a16a7f5c Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 15:37:50 +0800 Subject: [PATCH 07/22] perf(agent-loop): reuse proven message freezes per agent --- ...6-07-05-reconstructable-requests.i18n.yaml | 4 +- .../2026-07-05-reconstructable-requests.md | 2 +- .../2026-07-05-reconstructable-requests.zh.md | 2 +- ...-agent-request-freeze-provenance.i18n.yaml | 6 + ...6-09-06-agent-request-freeze-provenance.md | 55 ++++ ...9-06-agent-request-freeze-provenance.zh.md | 55 ++++ docs/architecture.i18n.yaml | 4 +- docs/architecture.md | 2 + docs/architecture.zh.md | 2 + packages/core/agent-loop/README.i18n.yaml | 4 +- packages/core/agent-loop/README.md | 2 + packages/core/agent-loop/README.zh.md | 2 + packages/core/agent-loop/src/agent.ts | 15 +- .../agent-loop/tests/request-freeze.spec.ts | 234 ++++++++++++++++++ 14 files changed, 379 insertions(+), 10 deletions(-) create mode 100644 .agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.i18n.yaml create mode 100644 .agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.md create mode 100644 .agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.zh.md create mode 100644 packages/core/agent-loop/tests/request-freeze.spec.ts diff --git a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.i18n.yaml b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.i18n.yaml index c1a0acb3d7..12c82b20bb 100644 --- a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.md -2026-07-05-reconstructable-requests.md: bc8ba640c400b18598f18aa303f2bd1b5c5b9cdc -2026-07-05-reconstructable-requests.zh.md: de1802aac83f1e0980172d2d541a093f2d729e4a +2026-07-05-reconstructable-requests.md: bca93a60bf07484d73f1faf50359b72a0d00b9a3 +2026-07-05-reconstructable-requests.zh.md: c9d2a4a5d05456df8b0bd065bade8a41dd7e4e84 diff --git a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.md b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.md index bc8ba640c4..bca93a60bf 100644 --- a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.md +++ b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.md @@ -24,7 +24,7 @@ Prefix-cache stability is corollary #1, not the headline: an append-only log pro `EpochHeader` records the request's non-history state: call config, rendered system prompt, and tool schemas, with empty values canonicalized to absence. Adapter-supplied effort and token defaults retain their `adapterDefaults` provenance; a Web model selection restored from the log omits an adapter-owned effort so the next resolution cannot reclassify the same effective config as an explicit selection and a false change. `request/header` always writes a full snapshot: the first loop instance uses reason `initial`, later instances use `resume`, an in-instance change uses `change`, and an unchanged envelope beginning an explicitly declared message series or following a surface replacement uses `series`. A `change` snapshot carries `startsSeries: true` when the changed request also starts a series, preserving the two independent facts without a duplicate header. Ordinary append-only later Turns, further same-series Steps, and retries inherit the latest snapshot. `foldRequestHeader` selects the latest snapshot. Legacy `request/header-delta` events and the removed `fallback` reason are rejected when appended or loaded. -Each proposed step first claims its inbox batch and runs `agent/pre-step`. Rejection opens no step; enter opens `step/start`, records the final message batch as `user/message` events, and may use `startsRequestSeries: true` to declare a distinct series. The step then assembles the system prompt and tools, while `agent/request` may replace only the frozen call-config seed. The loop records the owed initial, resume, change, or series full snapshot, builds `GenerateOptions` from derived messages and that header, and deep-freezes it while leaving `AbortSignal` live. The first call config starts from explicit `AgentOptions`, preserving fork overrides and resume reconfiguration; later calls start from the folded header. +Each proposed step first claims its inbox batch and runs `agent/pre-step`. Rejection opens no step; enter opens `step/start`, records the final message batch as `user/message` events, and may use `startsRequestSeries: true` to declare a distinct series. The step then assembles the system prompt and tools, while `agent/request` may replace only the frozen call-config seed. The loop records the owed initial, resume, change, or series full snapshot, builds `GenerateOptions` from derived messages and that header, and freezes it while leaving `AbortSignal` live. The [request-freeze provenance decision](../simplification/2026-09-06-agent-request-freeze-provenance.md) owns reuse of completed message freezes and per-request local header freezing. The first call config starts from explicit `AgentOptions`, preserving fork overrides and resume reconfiguration; later calls start from the folded header. **The open step is the reconstruction boundary.** Its entered `user/message` batch and any newly written `request/header` precede request dispatch. Injection after the atomic claim joins a later request, while a listener that must affect this request returns messages through `agent/pre-step`. Header reconstruction selects the step's `request/header`, or carries the prior snapshot when no new header is written. diff --git a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.zh.md b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.zh.md index de1802aac8..c9d2a4a5d0 100644 --- a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.zh.md +++ b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.zh.md @@ -24,7 +24,7 @@ Status: implemented `EpochHeader` 记录请求的非历史状态：调用配置、渲染后的系统提示词和工具 schema，空值规范化为缺失。适配器提供的推理强度与 token 默认值会保留其 `adapterDefaults` 来源信息；Web 从日志恢复模型选择时会省略适配器持有的推理强度，因此下一次解析不会把相同的有效配置重新归类为显式选择并产生虚假变更。`request/header` 始终写入完整快照：首个循环实例使用 reason `initial`，后续实例使用 `resume`，实例内变更使用 `change`，内容未变的封装显式开启消息序列或跟随表层替换时使用 `series`。如果发生变化的请求同时开启序列，`change` 快照会携带 `startsSeries: true`，无需重复 header 即可保留这两个独立事实。普通的仅追加后续 Turn、同一序列内后续的 Step 与重试沿用最新快照。`foldRequestHeader` 选择最新快照。旧的 `request/header-delta` 事件和已移除的 `fallback` reason 在追加或加载时都会被拒绝。 -每个拟议步骤先领取其 inbox 批次，再运行 `agent/pre-step`。reject 不打开步骤；enter 打开 `step/start`，把最终消息批次记录为 `user/message` 事件，并可使用 `startsRequestSeries: true` 声明独立序列。随后步骤组装系统提示词与工具，`agent/request` 只能替换冻结的调用配置种子。循环记录所需的 initial、resume、change 或 series 完整快照，从派生消息与该 header 构建 `GenerateOptions`，对其深度冻结但保持 `AbortSignal` 活跃。首次调用配置从显式的 `AgentOptions` 出发，保留 fork 覆盖和恢复重配置；后续调用从折叠后的 header 出发。 +每个拟议步骤先领取其 inbox 批次，再运行 `agent/pre-step`。reject 不打开步骤；enter 打开 `step/start`，把最终消息批次记录为 `user/message` 事件，并可使用 `startsRequestSeries: true` 声明独立序列。随后步骤组装系统提示词与工具，`agent/request` 只能替换冻结的调用配置种子。循环记录所需的 initial、resume、change 或 series 完整快照，从派生消息与该 header 构建 `GenerateOptions`，冻结请求但保持 `AbortSignal` 活跃。[请求冻结来源证明决策](../simplification/2026-09-06-agent-request-freeze-provenance.zh.md)拥有消息完整冻结的复用规则和每次请求的本地 header 冻结规则。首次调用配置从显式的 `AgentOptions` 出发，保留 fork 覆盖和恢复重配置；后续调用从折叠后的 header 出发。 **已打开步骤是重建边界。** 进入步骤的 `user/message` 批次与任何新写入的 `request/header` 都位于请求分派之前。原子领取后发生的注入加入后续请求；必须影响本次请求的监听器则通过 `agent/pre-step` 返回消息。header 重建选择该步骤的 `request/header`，或在无新 header 写入时沿用前一个快照。 diff --git a/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.i18n.yaml b/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.i18n.yaml new file mode 100644 index 0000000000..9cc7f50ddb --- /dev/null +++ b/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.md +2026-09-06-agent-request-freeze-provenance.md: bfefe39a0c481250d45318c02199cbd947c4eb4f +2026-09-06-agent-request-freeze-provenance.zh.md: 4a333c845d11e2e1cbe8ef2325f92f56f48b203d diff --git a/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.md b/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.md new file mode 100644 index 0000000000..bfefe39a0c --- /dev/null +++ b/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.md @@ -0,0 +1,55 @@ +# Agent Note: Reuse only loop-proven message freezes + +Status: implemented + +English | [中文](2026-09-06-agent-request-freeze-provenance.zh.md) + +## Problem + +Long tool conversations repeatedly traverse immutable history while constructing requests. The [backend continuation baseline](../testing/2026-09-06-backend-continuation-performance.md) attributes 132.876 ms of sampled CPU self time to `buildRequest`'s `deepFreeze` during a 211.300 ms request-history operation. Skipping all frozen roots is unsafe: restore adopts independently owned graphs without freezing them, and a shallow-frozen message can still contain mutable content. + +## Decision + +Each `ReactLoopAgent` owns a private WeakSet containing only identities whose complete `deepFreeze` call succeeded in that instance. Every unseen message is deep-frozen in place, then added. Later requests reuse that proof. A fresh loop proves each identity again; equal message ids do not establish object identity. Weak references add no ownership of compacted history. + +The loop deep-freezes the small local canonical header on every request. `canonicalHeader` shares nested values, and `Session.append` freezes a separate snapshot: neither operation proves the local tools or a `NO_ADAPTER` fallback's stop array immutable. The loop separately freezes its fresh messages array and request envelope, retains `markAgentLoopRequest`, and leaves the live `AbortSignal` mutable. Restored message identity and containing event-wrapper mutability remain unchanged. + +This specializes request construction, not Session ownership or general `deepFreeze` behavior. `Session.deriveMessages` and `fromRestore` remain unchanged. LLM file, image, and replay projections retain their own freezes because their newly produced values have no loop-local proof. The [reconstructable-request decision](../architecture/2026-07-05-reconstructable-requests.md) continues to own observable immutability and logged request reconstruction. + +## Measurement evidence + +Apple M4 Pro, macOS arm64, Node 24.19.0; independent worktree dependencies and built artifacts. The exact parent Agent source at 1dc3296eba is rebuilt for the negative control, then the optimized source is restored and rebuilt. Each row retains all five fresh-process totals in sampling order; all timings are milliseconds. Exclusive slots do not overlap repository builds or sibling benchmarks. + +| Implementation and UTC interval (2026-09-06) | Request-history raw totals | Median | 175 ms verdict | +|---|---|---:|---| +| Optimized, 07:15:40–07:15:51 | 65.737375, 67.292833, 68.035208, 65.380417, 67.919167 | 67.292833 | Pass | +| Original, 07:17:06–07:17:10 | 249.050708, 238.275291, 242.172084, 250.093166, 246.130875 | 246.130875 | Fail | +| Optimized repeat, 07:18:17–07:18:20 | 66.693500, 67.402083, 68.665000, 66.642083, 66.609125 | 66.693500 | Pass | + +The same 800-turn, four-tools-per-historical-turn history and 40 live requests complete in every sample: 13,923 events, no live tool calls. The repeat median is 72.9% below the isolated original. A 70 ms source expectation rounds above both optimized medians; the existing 2× CI scale and 1.25× headroom produce 175 ms. This is local calibration, not proof that the shared scale fits every CI runner; the required CI lane owns runner validation. No other case or memory budget changes here. + +The first optimized slot also measures cold tool continuation: totals 185.839958, 185.235583, 185.865917, 189.213459, 185.279417; median 185.839958 ms. Every sample completes 40 requests and 160 tool calls with 14,143 events. Retained heap samples are 22.591591, 22.590355, 22.594795, 22.591743, 22.594681 MiB, below the unchanged 28.75 MiB budget. The earlier baseline's approximately 22.295 MiB highlights the small provenance-table cost; weak keys prevent the table itself retaining replaced messages. + +The same slot's shipped SDK profile completes 100 turns, 200 requests, and 800 real reads per sample. Totals are 1428.555292, 1160.396333, 1139.843500, 1135.834750, 1155.890334 ms; median 1155.890334 ms. The first sample includes 461.829250 ms boot time versus 164–169 ms for the others and is retained, not discarded. Provider serialization, network time, and browser rendering remain excluded as specified by the baseline owner. + +An earlier original-code run at 06:58:28 UTC overlaps a sibling build because of scheduling-message latency: totals 264.269792, 282.442000, 365.836334, 293.172791, 288.719500 ms; median 288.719500 ms. It also fails 175 ms but is not calibration evidence. The isolated original row replaces that comparison, without removing or averaging away the contaminated samples. + +## Alternatives considered + +**Return immediately for `Object.isFrozen`.** A frozen root does not prove its descendants frozen. Applying this shortcut to the shared helper would weaken every caller, including restore and projection paths. + +**Trust every Session message or cache message ids.** Restore explicitly permits owned unfrozen data; replacements can preserve an id while changing identity and content. Only completed traversal of that exact object proves the request's requirement. + +**Retain a strong Set or share a global proof cache.** Strong references extend old history lifetime. Global caching expands ownership beyond the Agent and is unnecessary for repeated requests from one loop. + +**Remove downstream projection freezes.** Projected file/image/replay messages are distinct values with separate ownership. Optimizing them requires their own evidence and is not implied by freezing canonical history. + +## Consequences + +Request construction still scans message identities and allocates a fresh array; it avoids recursively traversing already-proven history. Each loop pays one complete traversal for restored history. Local headers remain a per-request cost. Message values, request markers, previous request snapshots, cancellation, and serialized SDK outputs keep their existing behavior. + +The [focused tests](../../../../packages/core/agent-loop/tests/request-freeze.spec.ts) exercise shallow-frozen restored roots with mutable descendants, wrapper identity and mutability, successful-only provenance, repeated requests, same-id compaction replacements, a fresh loop, nested tool schemas, adapter and `NO_ADAPTER` stop arrays, held requests, and live cancellation. Reconstruction and cancellation suites cover adjacent loop semantics. Performance measurements use the unchanged [continuation workload](../../../../benchmarks/agent-continuation/workload.ts), not a smaller synthetic microbenchmark. + +Validation runs 646 Agent-loop and LLM tests with 100% statement, branch, function, and line coverage of agent.ts. Keyless TypeScript SDK bash-tool and multi-turn snapshots pass against rebuilt libraries. Python sdk-minimal and sdk-snapshot checks pass against an independently packaged node24-macos-arm64 executable. Neither SDK requires an expected-output change. The packaging deploy temporarily removes workspace dependency links; a frozen-lockfile install restores them before source checks, without a tracked dependency change. + +The active immutability, message-identity, observable-state-machine, and backend-baseline notes remain independently useful; none is fully superseded or archived. This note specializes the request-freezing mechanism and cross-links its reconstructability owner. diff --git a/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.zh.md b/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.zh.md new file mode 100644 index 0000000000..4a333c845d --- /dev/null +++ b/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.zh.md @@ -0,0 +1,55 @@ +# Agent Note: 仅复用循环已证明的消息冻结 + +Status: implemented + +[English](2026-09-06-agent-request-freeze-provenance.md) | 中文 + +## 问题 + +长工具对话在构造请求时反复遍历不可变历史。[后端续跑基线](../testing/2026-09-06-backend-continuation-performance.zh.md)在一次 211.300 ms 的请求历史操作中，将 132.876 ms 的采样 CPU 自耗时归因于 `buildRequest` 的 `deepFreeze`。跳过所有已冻结根对象并不安全：恢复操作会接管独立拥有的对象图而不冻结它们，浅冻结消息仍可能包含可变内容。 + +## 决策 + +每个 `ReactLoopAgent` 拥有私有 WeakSet，其中只记录该实例中完整 `deepFreeze` 调用成功的对象身份。每个未见消息先原地深冻结，再加入集合。后续请求复用该证明。新循环会重新证明每个对象；消息 id 相等不能证明对象身份相同。弱引用不会增加对已压缩历史的所有权。 + +循环每次请求都深冻结较小的本地规范化 header。`canonicalHeader` 共享嵌套值，`Session.append` 冻结的是独立快照：两者都不能证明本地 tools 或 `NO_ADAPTER` 回退中的 stop 数组不可变。循环分别冻结新消息数组与请求封装，保留 `markAgentLoopRequest`，并保持实时 `AbortSignal` 可变。恢复消息的对象身份及其外围事件包装对象的可变性保持不变。 + +该决策专门优化请求构造，不改变 Session 所有权或通用 `deepFreeze` 行为。`Session.deriveMessages` 与 `fromRestore` 保持不变。LLM（大语言模型）的文件、图像和回放投影保留各自的冻结，因为它们新生成的值没有循环本地证明。[可重建请求决策](../architecture/2026-07-05-reconstructable-requests.zh.md)继续拥有可观察的不可变性与基于日志的请求重建规则。 + +## 测量证据 + +Apple M4 Pro、macOS arm64、Node 24.19.0；worktree 使用独立依赖和构建产物。负对照重新构建 1dc3296eba 中精确的父版本 Agent 源码，随后恢复并重新构建优化源码。每行按采样顺序保留全部五个新进程总耗时；时间单位均为毫秒。独占时段不与仓库构建或其他基准重叠。 + +| 实现与 UTC 时段（2026-09-06） | 请求历史原始总耗时 | 中位数 | 175 ms 判定 | +|---|---|---:|---| +| 优化版，07:15:40–07:15:51 | 65.737375, 67.292833, 68.035208, 65.380417, 67.919167 | 67.292833 | 通过 | +| 原版，07:17:06–07:17:10 | 249.050708, 238.275291, 242.172084, 250.093166, 246.130875 | 246.130875 | 失败 | +| 优化版复测，07:18:17–07:18:20 | 66.693500, 67.402083, 68.665000, 66.642083, 66.609125 | 66.693500 | 通过 | + +每个样本都完成相同的 800 轮历史（每个历史轮次四个工具）和 40 个实时请求：13,923 个事件，无实时工具调用。复测中位数比独占原版低 72.9%。70 ms 的源码期望值向上取整并高于两次优化版中位数；现有 2× CI 系数和 1.25× 余量得到 175 ms。这是本地校准，不能证明共享系数适合所有 CI 运行器；必跑 CI 测试负责验证运行器。本文不改变其他场景或内存预算。 + +首个优化版时段还测量冷启动工具续跑：总耗时 185.839958, 185.235583, 185.865917, 189.213459, 185.279417；中位数 185.839958 ms。每个样本都完成 40 个请求、160 个工具调用和 14,143 个事件。保留堆样本为 22.591591, 22.590355, 22.594795, 22.591743, 22.594681 MiB，低于不变的 28.75 MiB 预算。先前基线约 22.295 MiB，显示了证明表的小额成本；弱键防止表本身保留已替换消息。 + +同一时段的随产品发布 SDK profile 每个样本都完成 100 轮、200 个请求和 800 次真实读取。总耗时为 1428.555292, 1160.396333, 1139.843500, 1135.834750, 1155.890334 ms；中位数 1155.890334 ms。首个样本包含 461.829250 ms 启动时间，其他样本为 164–169 ms；首个样本被保留而非丢弃。供应商序列化、网络时间和浏览器渲染仍按基线所属说明排除。 + +较早的原版运行始于 06:58:28 UTC，因调度消息延迟而与其他构建重叠：总耗时 264.269792, 282.442000, 365.836334, 293.172791, 288.719500 ms；中位数 288.719500 ms。它也超过 175 ms，但不属于校准证据。独占原版行替代该比较，没有删除受污染样本或通过取平均掩盖它们。 + +## 考虑过的替代方案 + +**`Object.isFrozen` 为真时立即返回。** 已冻结根对象不能证明其后代已冻结。在共享辅助函数中使用此捷径会削弱所有调用方，包括恢复与投影路径。 + +**信任所有 Session 消息或缓存消息 id。** 恢复明确允许拥有独立所有权的未冻结数据；替换操作可保留 id，同时改变对象身份与内容。只有对该精确对象完成遍历才能证明请求要求。 + +**保留强引用 Set 或共享全局证明缓存。** 强引用会延长旧历史的生命周期。全局缓存将所有权扩大到 agent（智能体）之外，对同一循环的重复请求并无必要。 + +**移除下游投影冻结。** 投影后的文件／图像／回放消息是拥有独立所有权的不同值。优化它们需要独立证据，不能由规范历史已冻结推导出来。 + +## 影响 + +请求构造仍扫描消息身份并分配新数组，但避免递归遍历已证明的历史。每个循环都为恢复历史支付一次完整遍历成本。本地 header 仍是每次请求的成本。消息值、请求标记、先前请求快照、取消及 SDK 序列化输出保持现有行为。 + +[聚焦测试](../../../../packages/core/agent-loop/tests/request-freeze.spec.ts)覆盖具有可变后代的浅冻结恢复根对象、包装对象身份与可变性、仅成功遍历的证明、重复请求、同 id 压缩替换、新循环、嵌套工具 schema、适配器与 `NO_ADAPTER` stop 数组、持有的旧请求以及实时取消。重建与取消测试集覆盖相邻循环语义。性能测量采用不变的[续跑工作负载](../../../../benchmarks/agent-continuation/workload.ts)，而非缩小的合成微基准。 + +验证运行了 646 个 agent loop 与 LLM 测试，agent.ts 的语句、分支、函数和行覆盖率均为 100%。无密钥 TypeScript SDK bash-tool 与 multi-turn 快照通过重新构建的库执行并通过。Python sdk-minimal 与 sdk-snapshot 检查使用独立打包的 node24-macos-arm64 可执行文件并通过。两个 SDK 均无需修改期望输出。打包部署暂时移除了工作区依赖链接；执行冻结 lockfile 安装可在源码检查前恢复它们，无需修改受版本管理的依赖文件。 + +现行不可变性、消息身份、可观察状态机和后端基线说明仍各自具有价值；没有说明被完全取代或归档。本文专门规定请求冻结机制，并与可重建性所属说明交叉链接。 diff --git a/docs/architecture.i18n.yaml b/docs/architecture.i18n.yaml index 46d8c636b6..12a7a1d617 100644 --- a/docs/architecture.i18n.yaml +++ b/docs/architecture.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/architecture.md -architecture.md: bbd6a7e09b6af2fe5e90acab33ad220d3f1b62d1 -architecture.zh.md: b05670f8f715c5c3dd5c8cdd76ec81e9d426ace6 +architecture.md: fcb9c1e59b60dc059ab66b64ac26acd3c9157c96 +architecture.zh.md: d7a0a3833ebf9397837967065249d7fe1650d707 diff --git a/docs/architecture.md b/docs/architecture.md index bbd6a7e09b..fcb9c1e59b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -100,6 +100,8 @@ Input reaches the driver through one inbox. Some messages wake it immediately; i `agent/pre-step` decides what the model sees. Listeners may rewrite the claimed messages or reject them outright; a rejected or empty first claim still closes a durable turn that spent no step, so the log records the attempt. An enter decision may also set `startsRequestSeries` to begin a distinct model-message series: the loop then logs a fresh `request/header` (reason `series`, or `change` carrying `startsSeries: true` when the envelope changed too). A listener that rebuilds a downstream enter decision must spread it (`{ ...decision, messages }`) so the declaration survives. Each step reads the prompt sections and tool schemas that plugins registered. +The loop sends immutable requests while keeping cancellation live. It reuses message-freeze provenance only for identities it has fully frozen; [agent-loop](../packages/core/agent-loop/README.md) owns the request construction rules. + Details: the [sequence diagram](agent-lifecycle.md), the [tool pipeline](tool-execution-pipeline.md), and [cancellation and error recovery](subsystems/core.md#the-agent-handle). ## Session log diff --git a/docs/architecture.zh.md b/docs/architecture.zh.md index b05670f8f7..d7a0a3833e 100644 --- a/docs/architecture.zh.md +++ b/docs/architecture.zh.md @@ -104,6 +104,8 @@ turn/end `agent/pre-step` 决定模型看到什么。监听器可以改写已领取的消息，也可以直接拒绝它们；首次领取被拒绝或被改写为空时，仍会关闭一个不含步骤的持久轮次，因此日志会记录这次尝试。enter 决策还可以设置 `startsRequestSeries` 来开启独立的模型消息序列：loop 会随之记录一个新的 `request/header`（原因为 `series`，或在封装同时变化时为携带 `startsSeries: true` 的 `change`）。重建下游 enter 决策的监听器必须展开它（`{ ...decision, messages }`），该声明才能存活。每个步骤读取插件注册的提示词片段和工具 schema。 +循环发送不可变请求，同时保留实时取消能力。只有已由该循环完整冻结的消息对象身份才能复用冻结证明；[agent-loop](../packages/core/agent-loop/README.zh.md)拥有请求构造规则。 + 详情见[时序图](agent-lifecycle.zh.md)、[工具流水线](tool-execution-pipeline.zh.md)和[取消与错误恢复](subsystems/core.zh.md#the-agent-handle)。 ## 会话日志 diff --git a/packages/core/agent-loop/README.i18n.yaml b/packages/core/agent-loop/README.i18n.yaml index 95a4edc69e..b1cc6893b2 100644 --- a/packages/core/agent-loop/README.i18n.yaml +++ b/packages/core/agent-loop/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/core/agent-loop/README.md -README.md: c5985f585f92f38cb27ae1385b1052dbce1d9b9d -README.zh.md: 615cdd5a6e3e71ae129d782403f09d0309913929 +README.md: 55966a4b0eed0c1cc3484314e809de79341072de +README.zh.md: 1d27bf3743f54d8fe66a58e75565fc85d94ae1ed diff --git a/packages/core/agent-loop/README.md b/packages/core/agent-loop/README.md index c5985f585f..55966a4b0e 100644 --- a/packages/core/agent-loop/README.md +++ b/packages/core/agent-loop/README.md @@ -90,6 +90,8 @@ The package is the one concrete implementation of the public `Agent` contract. I After `agent/request`, `ctx.llm.prepareCall()` validates adapter-owned fields and resolves reasoning-effort and output-token defaults under the active turn signal. The loop retains that exact adapter through resolution, `request/header` logging, and dispatch. It writes a full header for the first request, a changed envelope, an explicit message-series start, a request after surface replacement, and resume; unchanged steps, retries, and ordinary later turns in the same series inherit the latest header. Before the next waterfall, the loop removes adapter-default fields so the current route resolves them again, while explicit settings persist. An unhandled route still fails with `NO_ADAPTER`. +The loop deep-freezes each derived message identity on its first request and reuses that proof only within the same agent. Restored messages keep their identity; request construction does not freeze their containing event wrappers. Each request freezes its local canonical header, fresh message array, and envelope while leaving the cancellation signal live. The [request-freeze decision](../../../.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.md) explains ownership and measurement. + ### Source map | File | Role | diff --git a/packages/core/agent-loop/README.zh.md b/packages/core/agent-loop/README.zh.md index 615cdd5a6e..1d27bf3743 100644 --- a/packages/core/agent-loop/README.zh.md +++ b/packages/core/agent-loop/README.zh.md @@ -90,6 +90,8 @@ const handle = await ctx.agents.create({ `agent/request` 返回后，`ctx.llm.prepareCall()` 会在活跃轮次信号下校验适配器持有的字段，并解析推理强度和输出 token 默认值。循环会在解析、`request/header` 记录与分派期间保留同一个适配器。循环会为首次请求、变化的 envelope、显式消息序列起点、表层替换后的请求及恢复写入完整 header；同一序列内内容未变的步骤、重试与普通后续轮次继承最新 header。下一次 waterfall 前，循环移除适配器默认字段，使当前路由重新解析它们；显式设置则保留。未处理的路由仍以 `NO_ADAPTER` 失败。 +循环在每个派生消息对象首次进入请求时执行深冻结，并且仅在同一 agent 内复用该证明。恢复的消息保留对象身份；构造请求不会冻结包含消息的事件包装对象。每个请求都会冻结本地规范化 header、新消息数组和请求封装，同时保留取消信号的可变性。[请求冻结决策](../../../.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.zh.md)解释了所有权与测量依据。 + ### 源码地图 | 文件 | 职责 | diff --git a/packages/core/agent-loop/src/agent.ts b/packages/core/agent-loop/src/agent.ts index 5dd560cd99..634a1a52de 100644 --- a/packages/core/agent-loop/src/agent.ts +++ b/packages/core/agent-loop/src/agent.ts @@ -87,6 +87,8 @@ export class ReactLoopAgent implements Agent { /** Process-local revision of assistant frames for this attached Session. */ private assistantStreamRevision = 0 private assistantAttemptCounter = 0 + /** Identities fully frozen by this loop; weak references do not retain replaced history. */ + private readonly frozenMessages = new WeakSet() constructor( private loopCtx: Context, @@ -483,7 +485,8 @@ export class ReactLoopAgent implements Agent { /** * Compose one frozen request and bind it to the adapter registration that - * resolved its exact-model defaults. + * resolved its exact-model defaults. Message identities retain their first + * successful deep freeze; each local header is frozen afresh. The signal stays live. */ private async buildRequest( turn: number, @@ -576,7 +579,15 @@ export class ReactLoopAgent implements Agent { } signal.throwIfAborted() - const request = markAgentLoopRequest(deepFreeze({ + // canonicalHeader is shallow; append logs a detached snapshot, not these local values. + deepFreeze(header) + for (const message of boundaryMessages) { + if (this.frozenMessages.has(message)) continue + deepFreeze(message) + this.frozenMessages.add(message) + } + Object.freeze(boundaryMessages) + const request = markAgentLoopRequest(Object.freeze({ ...header.config, messages: boundaryMessages, ...header.system !== undefined ? { system: header.system } : {}, diff --git a/packages/core/agent-loop/tests/request-freeze.spec.ts b/packages/core/agent-loop/tests/request-freeze.spec.ts new file mode 100644 index 0000000000..cfb2c64da2 --- /dev/null +++ b/packages/core/agent-loop/tests/request-freeze.spec.ts @@ -0,0 +1,234 @@ +/** Request immutability through the real loop, including adopted restore graphs. */ + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { Context } from '@deepseek-ai/cordis' +import AgentLoop from '@deepseek-ai/dsh-agent-loop' +import type { Agent } from '@deepseek-ai/dsh-agent' +import { mountAgentLoopTestDependencies } from '@deepseek-ai/dsh-agent-loop-testkit' +import { createAssistantMessage, createUserMessage, isAgentLoopRequest } from '@deepseek-ai/dsh-llm' +import type { GenerateOptions, ToolSchema } from '@deepseek-ai/dsh-llm' +import { Session, SessionId, SessionLogOffset, SESSION_FORMAT_VERSION } from '@deepseek-ai/dsh-session' +import SessionProjectionRegistry from '@deepseek-ai/dsh-session-projection' +import * as values from '@deepseek-ai/dsh-util-values' +import { ReactLoopAgent } from '../src/agent.ts' +import { MockAdapter, textResponse } from './mock-adapter.ts' + +const cleanups: (() => Promise)[] = [] +afterEach(async () => { + try { + for (const cleanup of cleanups.reverse()) await cleanup() + } finally { + cleanups.length = 0 + vi.restoreAllMocks() + } +}) + +async function harness(adapter?: MockAdapter): Promise { + const ctx = new Context() + cleanups.push(() => ctx.fiber.dispose()) + await mountAgentLoopTestDependencies(ctx) + await ctx.plugin(SessionProjectionRegistry) + await ctx.plugin(AgentLoop, { agents: [] }) + if (adapter) ctx.effect(() => ctx.llm.registerAdapter(['mock'], adapter)) + return ctx +} + +async function send(agent: Agent, text: string): Promise { + agent.followup(createUserMessage({ content: [{ type: 'text', text }], source: { kind: 'user' } })) + await agent.whenIdle() +} + +function expectFrozen(value: unknown): void { + if (value === null || typeof value !== 'object' || value instanceof AbortSignal) return + expect(Object.isFrozen(value)).toBe(true) + for (const child of Object.values(value)) expectFrozen(child) +} + +describe('loop-owned request freezing', () => { + it('adopts restored identities, freezes nested messages at dispatch, and leaves event wrappers mutable', async () => { + const ctx = await harness(new MockAdapter([textResponse('one'), textResponse('two'), textResponse('three'), textResponse('four')])) + const id = SessionId('restored-freeze') + const seed = Session.create(id) + seed.append('user/message', createUserMessage({ + content: [{ type: 'text', text: 'restored user' }], source: { kind: 'user' }, + }), { surfaceOp: 'append' }) + seed.append('assistant/message', { + turn: 1, step: 1, + message: createAssistantMessage({ + content: [{ type: 'text', text: 'restored assistant' }], + source: { provider: 'mock', model: 'mock', replayState: { nested: ['opaque'] } }, + }), + stream: [], + }, { surfaceOp: 'append' }) + const events = structuredClone(seed.snapshotEvents()) + const userEvent = events.find(event => event.type === 'user/message')! + const assistantEvent = events.find(event => event.type === 'assistant/message')! + Object.freeze(userEvent.data) + const freeze = vi.spyOn(values, 'deepFreeze') + const session = Session.fromRestore(id, events, { + id, version: SESSION_FORMAT_VERSION, createdAt: 1, cwd: '/test', isSeeded: false, + }, SessionLogOffset(0), 'detached') + const before = session.deriveMessages() + expect(before[0]).toBe(userEvent.data) + expect(before[1]).toBe(assistantEvent.data.message) + expect(Object.isFrozen(before)).toBe(false) + expect(Object.isFrozen(userEvent.data.content)).toBe(false) + expect(Object.isFrozen(assistantEvent.data.message)).toBe(false) + ctx.effect(() => ctx.sessions.enter(session)) + const agent = new ReactLoopAgent(ctx, id, { provider: 'mock', model: 'mock' }, session) + cleanups.push(async () => { + agent.cancel({ kind: 'disposed' }) + await agent.whenIdle() + await agent.scope.dispose() + }) + const requests: GenerateOptions[] = [] + const errors: unknown[] = [] + ctx.on('agent/error', ({ error }) => { errors.push(error) }) + ctx.on('llm/stream', (request, next) => { + expect(isAgentLoopRequest(request)).toBe(true) + expectFrozen(request) + requests.push(request) + return next() + }) + await send(agent, 'first') + expect(errors).toEqual([]) + expect(requests).toHaveLength(1) + const first = requests[0]! + expect(first.messages[0]).toBe(before[0]) + expect(first.messages[1]).toBe(before[1]) + expect(Object.isFrozen(userEvent)).toBe(false) + expect(Object.isFrozen(assistantEvent.data)).toBe(false) + expect(Object.isFrozen(assistantEvent.data.stream)).toBe(false) + userEvent.time += 1 + assistantEvent.data.stream.push({ type: 'chunk', time: 2, chunk: { type: 'finish', reason: { kind: 'stop' } } }) + before.pop() + const held = JSON.stringify(first.messages) + await send(agent, 'second') + expect(requests).toHaveLength(2) + expect(requests[1]!.messages).not.toBe(first.messages) + expect(requests[1]!.messages[0]).toBe(first.messages[0]) + expect(requests[1]!.messages.length).toBeGreaterThan(first.messages.length) + const nodes = session.surface.nodes + const replacement = session.append('user/message', { + ...userEvent.data, content: [{ type: 'text', text: 'compacted' }], + }, { + surfaceOp: { op: 'replace', start: nodes[0]!, end: nodes[1]! }, + sourceEventSeqs: [nodes[0]!, nodes[1]!], + }) + await send(agent, 'third') + expect(requests).toHaveLength(3) + expect(requests[2]!.messages[0]).toBe(replacement.data) + expect(requests[2]!.messages[0]!.id).toBe(first.messages[0]!.id) + expect(requests[2]!.messages[0]).not.toBe(first.messages[0]) + expect(JSON.stringify(first.messages)).toBe(held) + expect(Object.isFrozen(session.deriveMessages())).toBe(false) + expect(freeze.mock.calls.filter(([value]) => value === userEvent.data)).toHaveLength(1) + expect(freeze.mock.calls.filter(([value]) => value === replacement.data)).toHaveLength(1) + const resumed = new ReactLoopAgent(ctx, id, { provider: 'mock', model: 'mock' }, session) + cleanups.push(async () => { + resumed.cancel({ kind: 'disposed' }) + await resumed.whenIdle() + await resumed.scope.dispose() + }) + await send(resumed, 'fresh loop') + expect(requests).toHaveLength(4) + expect(freeze.mock.calls.filter(([value]) => value === replacement.data)).toHaveLength(2) + }) + + it('retries freezing an identity whose previous traversal failed', async () => { + const ctx = await harness(new MockAdapter([textResponse('done')])) + const agent = await ctx.agentLoop.create(SessionId('freeze-failure'), { provider: 'mock', model: 'mock' }) + const message = agent.session.append('user/message', createUserMessage({ + content: [{ type: 'text', text: 'history' }], source: { kind: 'user' }, + }), { surfaceOp: 'append' }).data + const realFreeze = values.deepFreeze + let traversals = 0 + vi.spyOn(values, 'deepFreeze').mockImplementation((value) => { + if (value === message && ++traversals === 1) throw new Error('freeze traversal failed') + return realFreeze(value) + }) + const errors: unknown[] = [] + const requests: GenerateOptions[] = [] + ctx.on('agent/error', ({ error }) => { errors.push(error) }) + ctx.on('llm/stream', (request, next) => { requests.push(request); return next() }) + await send(agent, 'failed turn') + expect(errors).toEqual([new Error('freeze traversal failed')]) + expect(requests).toHaveLength(0) + await send(agent, 'retry turn') + expect(requests).toHaveLength(1) + expect(traversals).toBe(2) + expect(requests[0]!.messages[0]).toBe(message) + expectFrozen(requests[0]) + }) + + it.each([true, false])('freezes each local header with an adapter present: %s', async (registered) => { + const adapter = registered ? new MockAdapter([textResponse('one'), textResponse('two')]) : undefined + const ctx = await harness(adapter) + const schemas: ToolSchema[][] = [] + const stops: string[][] = [] + ctx.on('system-prompt/assemble', async (_assembly, _context, next) => { + const assembly = await next() + const tools: ToolSchema[] = [{ name: 'nested', description: 'test', parameters: { + type: 'object', properties: { value: { type: 'array', items: { type: 'string', enum: ['a', 'b'] } } }, + } }] + schemas.push(tools) + return { ...assembly, tools } + }) + ctx.on('agent/request', async (_payload, next) => { + const config = await next() + const stop = ['stop'] + stops.push(stop) + return { ...config, stop } + }) + const requests: GenerateOptions[] = [] + const errors: unknown[] = [] + ctx.on('agent/error', ({ error }) => { errors.push(error) }) + ctx.on('llm/stream', (request, next) => { + expect(isAgentLoopRequest(request)).toBe(true) + expectFrozen(request) + requests.push(request) + return registered ? next() : (async function* () { yield* textResponse('virtual') })() + }) + const agent = await ctx.agentLoop.create(SessionId('headers'), { provider: 'mock', model: 'mock' }) + await send(agent, 'first') + await send(agent, 'second') + expect(errors).toEqual([]) + expect(requests).toHaveLength(2) + for (const [index, request] of requests.entries()) { + expect(request.tools).toBe(schemas[index]) + expectFrozen(schemas[index]) + expect(() => request.stop!.push('mutate')).toThrow(TypeError) + if (!registered) expect(request.stop).toBe(stops[index]) + } + expect(agent.session.snapshotEvents().filter(event => event.type === 'request/header')).toHaveLength(1) + expect(agent.session.requestHeader()!.tools).not.toBe(requests[0]!.tools) + expect(agent.session.requestHeader()!.config.stop).not.toBe(requests[0]!.stop) + }) + + it('keeps the live request signal mutable and observes cancellation after dispatch', async () => { + const ctx = await harness(new MockAdapter(['hang'])) + const agent = await ctx.agentLoop.create(SessionId('cancel-freeze'), { provider: 'mock', model: 'mock' }) + const started = Promise.withResolvers() + ctx.on('llm/stream', (request, next) => { started.resolve(request); return next() }) + agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } })) + try { + const request = await started.promise + expect(Object.isFrozen(request)).toBe(true) + expect(Object.isFrozen(request.signal)).toBe(false) + expect(request.signal!.aborted).toBe(false) + const aborted = Promise.withResolvers() + request.signal!.addEventListener('abort', () => { aborted.resolve(undefined) }, { once: true }) + agent.cancel({ kind: 'user' }) + await aborted.promise + await agent.whenIdle() + expect(request.signal!.aborted).toBe(true) + expect(request.signal!.reason).toEqual({ kind: 'user' }) + expect(agent.session.snapshotEvents().at(-1)).toMatchObject({ + type: 'turn/end', data: { reason: { kind: 'aborted', reason: { kind: 'user' } } }, + }) + } finally { + agent.cancel({ kind: 'disposed' }) + await agent.whenIdle() + } + }) +}) From 8e270960ed99ef16800174f0cc92ef4008044d34 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 20:48:10 +0800 Subject: [PATCH 08/22] test(perf): calibrate request history on standard hosted CI --- ...-agent-request-freeze-provenance.i18n.yaml | 4 +- ...6-09-06-agent-request-freeze-provenance.md | 12 +++++- ...9-06-agent-request-freeze-provenance.zh.md | 12 +++++- .../agent-continuation.bench.ts | 42 ++++++++++++++++--- 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.i18n.yaml b/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.i18n.yaml index 9cc7f50ddb..5f3f78b3bf 100644 --- a/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.i18n.yaml +++ b/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.md -2026-09-06-agent-request-freeze-provenance.md: bfefe39a0c481250d45318c02199cbd947c4eb4f -2026-09-06-agent-request-freeze-provenance.zh.md: 4a333c845d11e2e1cbe8ef2325f92f56f48b203d +2026-09-06-agent-request-freeze-provenance.md: 7a4816df61f6490647aba6f0603719e1b4662a20 +2026-09-06-agent-request-freeze-provenance.zh.md: 239d7e69df1596010ef0f3c8789250f654a75cb1 diff --git a/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.md b/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.md index bfefe39a0c..7a4816df61 100644 --- a/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.md +++ b/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.md @@ -26,7 +26,7 @@ Apple M4 Pro, macOS arm64, Node 24.19.0; independent worktree dependencies and b | Original, 07:17:06–07:17:10 | 249.050708, 238.275291, 242.172084, 250.093166, 246.130875 | 246.130875 | Fail | | Optimized repeat, 07:18:17–07:18:20 | 66.693500, 67.402083, 68.665000, 66.642083, 66.609125 | 66.693500 | Pass | -The same 800-turn, four-tools-per-historical-turn history and 40 live requests complete in every sample: 13,923 events, no live tool calls. The repeat median is 72.9% below the isolated original. A 70 ms source expectation rounds above both optimized medians; the existing 2× CI scale and 1.25× headroom produce 175 ms. This is local calibration, not proof that the shared scale fits every CI runner; the required CI lane owns runner validation. No other case or memory budget changes here. +The same 800-turn, four-tools-per-historical-turn history and 40 live requests complete in every sample: 13,923 events, no live tool calls. The repeat median is 72.9% below the isolated original. The historical 70 ms M4 expectation rounds above both optimized medians; applying the shared 2× CI scale and 1.25× headroom produced the 175 ms budget used in the table. These remain local reference measurements, not hosted-runner expectations. The explicit hosted calibration below owns the enforced request-history budget; no other case or memory budget changes here. The first optimized slot also measures cold tool continuation: totals 185.839958, 185.235583, 185.865917, 189.213459, 185.279417; median 185.839958 ms. Every sample completes 40 requests and 160 tool calls with 14,143 events. Retained heap samples are 22.591591, 22.590355, 22.594795, 22.591743, 22.594681 MiB, below the unchanged 28.75 MiB budget. The earlier baseline's approximately 22.295 MiB highlights the small provenance-table cost; weak keys prevent the table itself retaining replaced messages. @@ -34,6 +34,16 @@ The same slot's shipped SDK profile completes 100 turns, 200 requests, and 800 r An earlier original-code run at 06:58:28 UTC overlaps a sibling build because of scheduling-message latency: totals 264.269792, 282.442000, 365.836334, 293.172791, 288.719500 ms; median 288.719500 ms. It also fails 175 ms but is not calibration evidence. The isolated original row replaces that comparison, without removing or averaging away the contaminated samples. +### Standard hosted CI calibration + +The standard two-CPU `ubuntu-24.04` lane runs Node 24.20.0. [Run 34033336380, job 101487280801](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336380/job/101487280801) measures the optimized request path at merge commit `8fba64d9ae06d1a9a778a95487bb915d24cb0644` in Azure eastus: 183.355397, 184.468253, 185.042397, 182.160790, 182.924728 ms; median 183.355397 ms. Every sample completes the same 40 requests and 13,923 events. All five exceed the historical 175 ms budget without changing the WeakSet implementation or workload. + +A second hosted run of the same request implementation, [run 34033336246, job 101487216170](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336246/job/101487216170), records 145.644577, 144.204300, 143.072572, 145.985903, 146.834474 ms; median 145.644577 ms. It uses the same Ubuntu image and Node version but a different worker in Azure westus3 at merge commit `c366e49`. This faster run does not replace the eastus evidence or establish why the workers differ. The older self-hosted `VM-7-113-ubuntu-ci-9` run with Node 24.18.1 ([run 34021903421, job 101456015028](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34021903421/job/101456015028)) records 110.025154, 119.958978, 108.266860, 107.557950, 108.538902 ms; median 108.538902 ms. Its runner and Node version do not calibrate the standard hosted lane. + +The request-history CI expectation is 190 ms, rounded above this observed range. The enforced median budget is `ceil(190 × 1.25) = 238 ms`; the shared 2× reference-machine scale does not apply again to a CI measurement. This matches the direct-CI calibration method of the [63 ms Session-reopen budget](../../../../benchmarks/session-open/session-open.bench.ts), rather than relabeling the M4 reference as hosted evidence. The 238 ms budget remains below the isolated original implementation’s 246.130875 ms M4 median. + +Deterministic controls call the same `assertRequestHistoryBudget` assertion as the timed case. They accept the recorded hosted median and maximum (185.042397 ms), reject the recorded original M4 median, and reject a synthetic 250 ms median from 248, 250, 252, 251, 249 ms inputs. The synthetic inputs model a material regression; they are not runtime measurements. Replaying recorded values verifies the assertion, not a new hosted run. The acceptance control fails at 175 ms before calibration; all three controls and the five request-freeze behavior tests pass at 238 ms. + ## Alternatives considered **Return immediately for `Object.isFrozen`.** A frozen root does not prove its descendants frozen. Applying this shortcut to the shared helper would weaken every caller, including restore and projection paths. diff --git a/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.zh.md b/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.zh.md index 4a333c845d..239d7e69df 100644 --- a/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.zh.md +++ b/.agents/notes/implemented/simplification/2026-09-06-agent-request-freeze-provenance.zh.md @@ -26,7 +26,7 @@ Apple M4 Pro、macOS arm64、Node 24.19.0；worktree 使用独立依赖和构建 | 原版，07:17:06–07:17:10 | 249.050708, 238.275291, 242.172084, 250.093166, 246.130875 | 246.130875 | 失败 | | 优化版复测，07:18:17–07:18:20 | 66.693500, 67.402083, 68.665000, 66.642083, 66.609125 | 66.693500 | 通过 | -每个样本都完成相同的 800 轮历史（每个历史轮次四个工具）和 40 个实时请求：13,923 个事件，无实时工具调用。复测中位数比独占原版低 72.9%。70 ms 的源码期望值向上取整并高于两次优化版中位数；现有 2× CI 系数和 1.25× 余量得到 175 ms。这是本地校准，不能证明共享系数适合所有 CI 运行器；必跑 CI 测试负责验证运行器。本文不改变其他场景或内存预算。 +每个样本都完成相同的 800 轮历史（每个历史轮次四个工具）和 40 个实时请求：13,923 个事件，无实时工具调用。复测中位数比独占原版低 72.9%。历史 M4 期望值 70 ms 向上取整并高于两次优化版中位数；应用共享 2× CI 系数和 1.25× 余量，得到表中使用的 175 ms 预算。这些仍是本地参考测量，而非托管运行器期望值。下文的显式托管校准拥有实际执行的请求历史预算；本文不改变其他场景或内存预算。 首个优化版时段还测量冷启动工具续跑：总耗时 185.839958, 185.235583, 185.865917, 189.213459, 185.279417；中位数 185.839958 ms。每个样本都完成 40 个请求、160 个工具调用和 14,143 个事件。保留堆样本为 22.591591, 22.590355, 22.594795, 22.591743, 22.594681 MiB，低于不变的 28.75 MiB 预算。先前基线约 22.295 MiB，显示了证明表的小额成本；弱键防止表本身保留已替换消息。 @@ -34,6 +34,16 @@ Apple M4 Pro、macOS arm64、Node 24.19.0；worktree 使用独立依赖和构建 较早的原版运行始于 06:58:28 UTC，因调度消息延迟而与其他构建重叠：总耗时 264.269792, 282.442000, 365.836334, 293.172791, 288.719500 ms；中位数 288.719500 ms。它也超过 175 ms，但不属于校准证据。独占原版行替代该比较，没有删除受污染样本或通过取平均掩盖它们。 +### 标准托管 CI 校准 + +标准双 CPU `ubuntu-24.04` 测试通道运行 Node 24.20.0。[运行 34033336380、任务 101487280801](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336380/job/101487280801)在 Azure eastus 上测量合并提交 `8fba64d9ae06d1a9a778a95487bb915d24cb0644` 的优化请求路径：183.355397, 184.468253, 185.042397, 182.160790, 182.924728 ms；中位数 183.355397 ms。每个样本都完成相同的 40 个请求和 13,923 个事件。在 WeakSet 实现与工作负载未变的情况下，全部五个样本均超过历史 175 ms 预算。 + +相同请求实现的另一次托管运行，[运行 34033336246、任务 101487216170](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336246/job/101487216170)，记录了 145.644577, 144.204300, 143.072572, 145.985903, 146.834474 ms；中位数 145.644577 ms。它在合并提交 `c366e49` 上使用相同的 Ubuntu 镜像和 Node 版本，但运行于 Azure westus3 的另一台工作机。较快的运行不能替代 eastus 证据，也不能证明工作机差异的原因。较早的自托管 `VM-7-113-ubuntu-ci-9` 运行使用 Node 24.18.1（[运行 34021903421、任务 101456015028](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34021903421/job/101456015028)），记录了 110.025154, 119.958978, 108.266860, 107.557950, 108.538902 ms；中位数 108.538902 ms。其运行器和 Node 版本不能校准标准托管通道。 + +请求历史的 CI 期望值为 190 ms，向上取整并高于该观测范围。实际执行的中位数预算为 `ceil(190 × 1.25) = 238 ms`；CI 测量不再应用共享的参考机器 2× 系数。这与 [Session 重开 63 ms 预算](../../../../benchmarks/session-open/session-open.bench.ts)的直接 CI 校准方法一致，而非将 M4 参考值重新标注为托管证据。238 ms 预算仍低于独占原版实现的 M4 中位数 246.130875 ms。 + +确定性对照调用与计时场景相同的 `assertRequestHistoryBudget` 断言。它们接受已记录的托管中位数和最大值（185.042397 ms），拒绝已记录的原版 M4 中位数，并拒绝由 248, 250, 252, 251, 249 ms 输入得到的合成 250 ms 中位数。合成输入模拟显著回归，并非运行时测量。回放已记录数值验证的是断言，而非新的托管运行。接受对照在校准前以 175 ms 预算失败；三个对照和五个请求冻结行为测试在 238 ms 预算下均通过。 + ## 考虑过的替代方案 **`Object.isFrozen` 为真时立即返回。** 已冻结根对象不能证明其后代已冻结。在共享辅助函数中使用此捷径会削弱所有调用方，包括恢复与投影路径。 diff --git a/benchmarks/agent-continuation/agent-continuation.bench.ts b/benchmarks/agent-continuation/agent-continuation.bench.ts index ed681c2ce6..957c694819 100644 --- a/benchmarks/agent-continuation/agent-continuation.bench.ts +++ b/benchmarks/agent-continuation/agent-continuation.bench.ts @@ -24,10 +24,13 @@ const TOOL_CONTINUATION_BUDGET_MS = Math.ceil(EXPECTED_TOOL_CONTINUATION_CI_MS * /** Standard two-CPU hosted CI catalog median is 858.364 ms; 900 ms is the rounded expectation. */ const EXPECTED_CATALOG_CI_MS = 900 const CATALOG_BUDGET_MS = Math.ceil(EXPECTED_CATALOG_CI_MS * PERFORMANCE_BUDGET_HEADROOM) +/** Two-CPU ubuntu-24.04 / Node 24.20 samples span 182.161–185.042 ms; rounded CI expectation. */ +const EXPECTED_REQUEST_HISTORY_CI_MS = 190 +const REQUEST_HISTORY_BUDGET_MS = Math.ceil(EXPECTED_REQUEST_HISTORY_CI_MS * PERFORMANCE_BUDGET_HEADROOM) const EXPECTED_RETAINED_HEAP_MB = 23 const WORKERS = join(import.meta.dirname, '..', '.dsh-build', 'agent-continuation') -type Scenario = keyof typeof EXPECTED_MS | 'catalog' | 'tool-continuation' | 'request-history' +type Scenario = 'request-history' | 'catalog' | 'tool-continuation' | keyof typeof EXPECTED_MS type Report = ContinuationReport | CatalogReport | ProfileReport function workerName(scenario: Scenario): string { @@ -96,6 +99,34 @@ describe('standard hosted baseline request-history calibration', () => { }) }) +function assertRequestHistoryBudget(value: number): void { + expect(value).toBeLessThanOrEqual(REQUEST_HISTORY_BUDGET_MS) +} + +describe('standard hosted request-history calibration', () => { + it('accepts the recorded two-CPU samples above the historical budget', () => { + const recorded = [183.355397, 184.468253, 185.042397, 182.160790, 182.924728] + const recordedMedian = median(recorded) + + expect(recordedMedian).toBe(183.355397) + expect(recordedMedian).toBeGreaterThan(ciTimeBudget(70)) + assertRequestHistoryBudget(recordedMedian) + assertRequestHistoryBudget(Math.max(...recorded)) + expect(REQUEST_HISTORY_BUDGET_MS).toBe(238) + }) + + it('rejects a synthetic material request-history regression', () => { + const regressionMedian = median([248, 250, 252, 251, 249]) + expect(() => assertRequestHistoryBudget(regressionMedian)).toThrow() + }) + + it('rejects the recorded original implementation on the M4 reference', () => { + const originalMedian = median([249.050708, 238.275291, 242.172084, 250.093166, 246.130875]) + expect(originalMedian).toBe(246.130875) + expect(() => assertRequestHistoryBudget(originalMedian)).toThrow() + }) +}) + describe('continuing tool-heavy Sessions with large histories', () => { let scratch: string | undefined const sources = new Map() @@ -124,16 +155,17 @@ describe('continuing tool-heavy Sessions with large histories', () => { finally { await rm(root, { recursive: true, force: true }) } } const totalMs = samples.map(sample => sample.totalMs) - const budgetMs = scenario === 'catalog' ? CATALOG_BUDGET_MS - : scenario === 'tool-continuation' ? TOOL_CONTINUATION_BUDGET_MS - : scenario === 'request-history' ? BASELINE_REQUEST_BUDGET_MS : ciTimeBudget(EXPECTED_MS[scenario]) + const budgetMs = scenario === 'request-history' ? REQUEST_HISTORY_BUDGET_MS + : scenario === 'catalog' ? CATALOG_BUDGET_MS + : scenario === 'tool-continuation' ? TOOL_CONTINUATION_BUDGET_MS : ciTimeBudget(EXPECTED_MS[scenario]) const retainedHeapBudgetMb = EXPECTED_RETAINED_HEAP_MB * PERFORMANCE_BUDGET_HEADROOM console.log(JSON.stringify({ benchmark: 'agent-continuation/' + scenario, workload: WORKLOAD, samples, totalMs: { min: Math.min(...totalMs), median: median(totalMs), max: Math.max(...totalMs) }, budgetMs, ...(scenario === 'tool-continuation' ? { retainedHeapBudgetMb } : {}), })) - expectTotalWithinBudget(median(totalMs), budgetMs) + if (scenario === 'request-history') assertRequestHistoryBudget(median(totalMs)) + else expectTotalWithinBudget(median(totalMs), budgetMs) if (scenario === 'tool-continuation') { expect(median((samples as ContinuationReport[]).map(sample => sample.retainedHeapMb))) .toBeLessThanOrEqual(retainedHeapBudgetMb) From e2b81cf8bfcc75fd1d20447ee4ad1aa5e61f8d29 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 20:52:19 +0800 Subject: [PATCH 09/22] docs(perf): distinguish hosted request and catalog budgets --- benchmarks/agent-continuation/README.i18n.yaml | 4 ++-- benchmarks/agent-continuation/README.md | 2 +- benchmarks/agent-continuation/README.zh.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/agent-continuation/README.i18n.yaml b/benchmarks/agent-continuation/README.i18n.yaml index e6fb5cd78c..1db841dc14 100644 --- a/benchmarks/agent-continuation/README.i18n.yaml +++ b/benchmarks/agent-continuation/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write benchmarks/agent-continuation/README.md -README.md: 9854de19c3465c9ed30fcce8d80e8d7b3ef3d864 -README.zh.md: 91904419b0514b8f47c230e6748109c4be570ef7 +README.md: 5f76b67805a906ea13321ab22eb671d41bfc3190 +README.zh.md: 54d9e1a0e950d14ce5a01568bc9e1cdb61de597a diff --git a/benchmarks/agent-continuation/README.md b/benchmarks/agent-continuation/README.md index 9854de19c3..5f76b67805 100644 --- a/benchmarks/agent-continuation/README.md +++ b/benchmarks/agent-continuation/README.md @@ -18,7 +18,7 @@ Measure long-history request processing, cold tool-heavy continuation, and repea From the repository root, build the libraries and workers with `pnpm run build:bench`, then run `pnpm exec vitest run --config vitest.bench.config.ts benchmarks/agent-continuation/agent-continuation.bench.ts`. Do not overlap timing runs with builds or other benchmarks. -The test reports all five fresh-process samples and enforces reviewed median budgets. Catalog and tool continuation each use a 900 ms standard hosted CI expectation with 1.25× headroom (1,125 ms); baseline request history uses 600 ms with the same headroom (750 ms). The SDK time budget uses reference-machine scaling. A failed worker reports its exit, signal, timeout, and stderr; temporary roots are removed even on failure. The required benchmark lane discovers this file automatically. +The test reports all five fresh-process samples and enforces reviewed median budgets. Catalog and tool continuation each use a 900 ms standard hosted CI expectation with 1.25× headroom (1,125 ms); request history uses a 190 ms hosted expectation with the same headroom (238 ms), and SDK continuation uses reference-machine scaling. A failed worker reports its exit, signal, timeout, and stderr; temporary roots are removed even on failure. The required benchmark lane discovers this file automatically. diff --git a/benchmarks/agent-continuation/README.zh.md b/benchmarks/agent-continuation/README.zh.md index 91904419b0..54d9e1a0e9 100644 --- a/benchmarks/agent-continuation/README.zh.md +++ b/benchmarks/agent-continuation/README.zh.md @@ -18,7 +18,7 @@ 在仓库根目录使用 `pnpm run build:bench` 构建库和 worker，然后运行 `pnpm exec vitest run --config vitest.bench.config.ts benchmarks/agent-continuation/agent-continuation.bench.ts`。不要让计时运行与构建或其他基准重叠。 -测试报告全部五个新进程样本，并约束经审查的中位数预算。目录和工具续聊用例均使用标准托管 CI 的 900 ms 期望值与 1.25× 余量（1,125 ms）；基线请求历史使用 600 ms 与相同余量（750 ms）。SDK 时间预算使用参考机器缩放。worker 失败时报告退出状态、信号、超时和 stderr；失败时也会删除临时根目录。必需基准通道自动发现此文件。 +测试报告全部五个新进程样本，并约束经审查的中位数预算。目录和工具续聊用例均使用标准托管 CI 的 900 ms 期望值与 1.25× 余量（1,125 ms）；请求历史使用 190 ms 托管期望值与相同余量（238 ms），SDK 续聊使用参考机器缩放。worker 失败时报告退出状态、信号、超时和 stderr；失败时也会删除临时根目录。必需基准通道自动发现此文件。 From 843c8723e5694a18c608d67907e4966213990a07 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 20:59:53 +0800 Subject: [PATCH 10/22] docs(perf): link active hosted request calibration --- .../2026-09-06-backend-continuation-performance.i18n.yaml | 4 ++-- .../testing/2026-09-06-backend-continuation-performance.md | 2 +- .../testing/2026-09-06-backend-continuation-performance.zh.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml index 03917f21fd..dfa20e45f0 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md -2026-09-06-backend-continuation-performance.md: f8dddda687185d6b6504cd2423d307df02db9dfe -2026-09-06-backend-continuation-performance.zh.md: 2e87e9157f6a6a834cfcef42c3f35b9f42199e6d +2026-09-06-backend-continuation-performance.md: f4316e790cf62f5027a0f7bfb2d3148cc79e7536 +2026-09-06-backend-continuation-performance.zh.md: 0fb36ba5375c61e907791b31d96f09062b7f62a3 diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md index f8dddda687..f4316e790c 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md @@ -37,7 +37,7 @@ The implementation reference is `925e012340f033f0521e802ba8569ce6dd7ef1ac` on Ap | Tool continuation | 358.953, 324.790, 318.861, 320.119, 322.896 | 324.280, 321.952, 340.409, 325.470, 324.312 | 322.896 / 324.312 | 340 | 850 | | Child catalog | 318.730, 309.006, 311.404, 308.565, 310.105 | 308.670, 310.030, 280.086, 303.084, 284.829 | 310.105 / 303.084 | 320 | 800 | -Continuation retains approximately 22.295 MiB; its source expectation is 23 MiB and its budget is 28.75 MiB. The SDK time expectation uses the existing [calibration helper](../../../../benchmarks/support/calibration.ts): 2× shared CI time scale and 1.25× variance headroom. Memory uses only 1.25× headroom. The scale is inherited from the existing lane's calibration, not a new Linux measurement of these cases; CI evidence remains necessary when runner characteristics change. Baseline budgets protect the measured implementation; tighter budgets belong with a measured behavior-preserving fix. +Continuation retains approximately 22.295 MiB; its source expectation is 23 MiB and its budget is 28.75 MiB. SDK time expectations use the existing [calibration helper](../../../../benchmarks/support/calibration.ts): 2× shared CI time scale and 1.25× variance headroom. Request history uses the direct hosted expectation in the [request-freeze calibration](../simplification/2026-09-06-agent-request-freeze-provenance.md), without the 2× scale. Memory uses only 1.25× headroom. The scale is inherited from the existing lane's calibration, not a new Linux measurement of these cases; CI evidence remains necessary when runner characteristics change. Baseline budgets protect the measured implementation; tighter budgets belong with a measured behavior-preserving fix. A separate plain-Node request-history CPU profile attributes 132.876 ms of sampled self time to deepFreeze called by buildRequest during a 211.300 ms operation. This identifies repeated traversal of already-frozen history as a focused investigation target, not a proven optimization result. Catalog first/repeat timings remain separate because a second listing still reads body-bearing seeded children after observations are released. diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md index 2e87e9157f..0fb36ba537 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md @@ -37,7 +37,7 @@ Status: implemented | 工具续聊 | 358.953, 324.790, 318.861, 320.119, 322.896 | 324.280, 321.952, 340.409, 325.470, 324.312 | 322.896 / 324.312 | 340 | 850 | | 子会话目录 | 318.730, 309.006, 311.404, 308.565, 310.105 | 308.670, 310.030, 280.086, 303.084, 284.829 | 310.105 / 303.084 | 320 | 800 | -续聊保留约 22.295 MiB；源码期望值为 23 MiB，预算为 28.75 MiB。SDK 时间期望值使用现有[校准辅助函数](../../../../benchmarks/support/calibration.ts)：2× 共享 CI 时间比例和 1.25× 波动余量。内存只使用 1.25× 余量。比例继承现有通道的校准，并非这些用例的新 Linux 实测值；runner 特征变化时仍需 CI 证据。基线预算保护实测实现；更紧预算属于有测量依据且保持行为的修复。 +续聊保留约 22.295 MiB；源码期望值为 23 MiB，预算为 28.75 MiB。SDK 时间期望值使用现有[校准辅助函数](../../../../benchmarks/support/calibration.ts)：2× 共享 CI 时间比例和 1.25× 波动余量。请求历史使用[请求冻结校准](../simplification/2026-09-06-agent-request-freeze-provenance.zh.md)中的直接托管期望值，不乘以 2× 比例。内存只使用 1.25× 余量。比例继承现有通道的校准，并非这些用例的新 Linux 实测值；runner 特征变化时仍需 CI 证据。基线预算保护实测实现；更紧预算属于有测量依据且保持行为的修复。 独立的纯 Node 请求历史 CPU profile 在一次 211.300 ms 操作中，将 132.876 ms 采样自身时间归因于 buildRequest 调用的 deepFreeze。这把重复遍历已冻结历史定位为聚焦调查目标，不是已证实的优化结果。目录首次／重复时间分别保留，因为观察释放后第二次列举仍读取带种子子会话的正文。 From 6ad74db2e35a6c18e04cb2ff5dbd6cdb2e1e2e6b Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 14:49:41 +0800 Subject: [PATCH 11/22] test(perf): gate long-session browser and active reconnect workflows --- ...04-session-open-performance-gate.i18n.yaml | 4 +- ...026-09-04-session-open-performance-gate.md | 8 +- ...-09-04-session-open-performance-gate.zh.md | 10 +- ...-06-frontend-performance-budgets.i18n.yaml | 6 + ...2026-09-06-frontend-performance-budgets.md | 55 +++++++ ...6-09-06-frontend-performance-budgets.zh.md | 55 +++++++ .github/workflows/ci.yml | 3 + benchmarks/AGENTS.md | 1 + .../active-stream-reconnect/README.i18n.yaml | 6 + benchmarks/active-stream-reconnect/README.md | 7 + .../active-stream-reconnect/README.zh.md | 7 + .../reconnect.bench.client.ts | 34 +++++ .../reconnect.worker.client.ts | 49 ++++++ .../long-session-browser/README.i18n.yaml | 6 + benchmarks/long-session-browser/README.md | 17 +++ benchmarks/long-session-browser/README.zh.md | 17 +++ .../long-session.bench.ts | 139 ++++++++++++++++++ .../long-session-browser/synthetic-history.ts | 69 +++++++++ benchmarks/package.json | 2 + benchmarks/tsdown.config.ts | 7 + package.json | 2 +- pnpm-lock.yaml | 6 + vitest.bench.config.ts | 5 +- 23 files changed, 501 insertions(+), 14 deletions(-) create mode 100644 .agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml create mode 100644 .agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md create mode 100644 .agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md create mode 100644 benchmarks/active-stream-reconnect/README.i18n.yaml create mode 100644 benchmarks/active-stream-reconnect/README.md create mode 100644 benchmarks/active-stream-reconnect/README.zh.md create mode 100644 benchmarks/active-stream-reconnect/reconnect.bench.client.ts create mode 100644 benchmarks/active-stream-reconnect/reconnect.worker.client.ts create mode 100644 benchmarks/long-session-browser/README.i18n.yaml create mode 100644 benchmarks/long-session-browser/README.md create mode 100644 benchmarks/long-session-browser/README.zh.md create mode 100644 benchmarks/long-session-browser/long-session.bench.ts create mode 100644 benchmarks/long-session-browser/synthetic-history.ts diff --git a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.i18n.yaml b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.i18n.yaml index 22cf5283e3..bfd2d60086 100644 --- a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md -2026-09-04-session-open-performance-gate.md: 2820c9d7d0e5b7d9382c7f8d6540154440175f26 -2026-09-04-session-open-performance-gate.zh.md: 965b9035074504870bcb2f1ca8166962c264d75a +2026-09-04-session-open-performance-gate.md: 2937a2aec1dbddb31fde82d2617d69852a611d90 +2026-09-04-session-open-performance-gate.zh.md: b6608ca79d07c3e9fb00d62801038ecefbdf944c diff --git a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md index 2820c9d7d0..2937a2aec1 100644 --- a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md +++ b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md @@ -12,13 +12,13 @@ Measuring only `SessionPersistence.open()` does not stably describe the result f ## Decision -Linux pull requests run a required `node 24 / benchmarks` job that executes `pnpm run check:ci:bench` → `pnpm run test:bench`. The private `@deepseek-ai/dsh-benchmarks` workspace owns benchmark-only dependencies. The command first builds workspace libraries and dedicated workers under `benchmarks/.dsh-build/`, then invokes `vitest.bench.config.ts`. The [standard hosted runner decision](2026-09-06-standard-hosted-benchmark-runner.md) owns runner selection and the outer job timeout. The job runs the benchmark lane alone; Vitest runs one file at a time and only prepares input, starts measurement children, aggregates results, and enforces budgets. Every timed CPU path executes compiled JavaScript under plain Node with `NODE_OPTIONS` removed and no TypeScript loader; bare workspace imports therefore resolve from `benchmarks/node_modules` through package exports to built `lib/` entries. +Linux pull requests run a required `node 24 / benchmarks` job that executes `pnpm run check:ci:bench` → `pnpm run test:bench`. The private `@deepseek-ai/dsh-benchmarks` workspace owns benchmark-only dependencies. The command first builds workspace libraries and dedicated workers under `benchmarks/.dsh-build/`, then invokes `vitest.bench.config.ts`. The [standard hosted runner decision](2026-09-06-standard-hosted-benchmark-runner.md) owns runner selection and the outer job timeout. The job runs the benchmark lane alone; Vitest runs one file at a time and only prepares input, starts measurement children, aggregates results, and enforces budgets. Every timed Node CPU path executes compiled JavaScript under plain Node with `NODE_OPTIONS` removed and no TypeScript loader; bare workspace imports therefore resolve from `benchmarks/node_modules` through package exports to built `lib/` entries. Required performance gates live under top-level `benchmarks/`, grouped by measured user path rather than package ownership. Host files use `*.bench.ts`, Client-face files use `*.bench.client.ts`, and scenario-specific workers and fixtures stay beside their benchmark without a benchmark suffix. Package-local `.perf.ts` files remain non-gating diagnostics; `scripts/` owns orchestration rather than benchmark cases. The Session benchmarks synthesize a released-v0 input from fixed parameters: 200 turns with 500 text deltas and 125 reasoning deltas per turn, for 127,400 logical events. The input uses Zstandard with fixed logical-row grouping and frame partitioning, so every run processes the same events, bytes, and frame distribution. The fixture constructs the immutable released-v0 physical rows directly instead of depending on a current-runtime historical encoder; compression and every measured read or migration entry point still use production code. Setup writes the input into a private temporary directory for each sample before timing starts; benchmarks never use recorded Sessions. -Every Session endpoint runs at two user-lifecycle points. `first-open` starts with only the released V0 generation and therefore includes migration and successor publication. Setup produces `post-upgrade-reopen` once through that same production migration outside measurement, then copies both the unchanged V0 predecessor and published V2 successor into each sample root. Reopen samples use a fresh process, so they measure an upgraded user's later disk open without migration or process-local caches. +Every Session endpoint runs at two user-lifecycle points. `first-open` starts with only the released V0 generation and includes migration; read-only consumers do not publish a successor, while writable Agent resume does. Setup produces `post-upgrade-reopen` once through that same production migration outside measurement, then copies both the unchanged V0 predecessor and published V2 successor into each sample root. Reopen samples use a fresh process, so they measure an upgraded user's later disk open without migration or process-local caches. Each access-kind and endpoint sample runs in a fresh compiled Node child process. Module imports, Host service initialization, and fixture preparation finish before measurement; the measured process performs no extra parse warm-up. Normal-heap mode runs five independent samples, reports every sample plus minimum, median, and maximum, and enforces access-specific fixed budgets against the median. Another child runs the same path under a fixed 128 MB old-space limit and checks only that it completes; extra GC caused by the constrained heap does not enter the normal timing baseline. @@ -26,7 +26,7 @@ The lane contains three independent Session-opening benchmarks and retains the C | Benchmark | Measured path | Timing metrics | |---|---|---| -| Phase profile | Executes the real persistence open, handle read, Session restore, and projection for both first open and post-upgrade reopen | `openMs`, `readMs`, `sessionRestoreMs`, and `projectionMs` each have a fixed budget; encoding, writes, verification, and publication awaited by migration all belong to first-open `openMs` | +| Phase profile | Executes the real persistence open, handle read, Session restore, and projection for both first open and post-upgrade reopen | `openMs`, `readMs`, `sessionRestoreMs`, and `projectionMs` each have a fixed budget; read-only migration belongs to first-open `openMs`; successor encoding, verification, and publication belong to writable Agent resume | | First history | Reads each access kind through the Host Session history controller until it produces the first paginated snapshot | Separate first-open and reopen end-to-end budgets; each includes source stat, reading, restoration, projection, pagination, and snapshot construction, while first open additionally includes migration; both exclude Gateway network transport, Client fold, and browser paint | | Agent resume | Calls `ctx.agents.resume()` for each access kind until Agent creation, setup, publication, and loop startup finish | Separate first-open and reopen end-to-end budgets; neither path runs after first-history or reuses that benchmark's cache | | Client fold | Folds small and large v2 history windows through the real `ConversationNodeAssembler` and every Chat Definition | The large window's absolute time and scaling relative to the small window each have a fixed budget | @@ -102,4 +102,4 @@ The calibrated source budgets are: Every pull request pays for one required Linux job; its Session portion runs several short-lived child processes in exchange for cold caches, isolated V8 heaps, explicit GC state, and attributable failures. The repository-level benchmark tree accepts deliberate cross-package test dependencies without changing product package manifests. The fixed Zstandard workload covers both event volume and frame topology; first-open measurements protect the one-time upgrade experience, reopen measurements prevent regressions in later opens, phase budgets locate cost, first-history budgets protect user-visible waiting, Agent-resume budgets and post-GC deltas protect complete cold activation and resident memory, and the 128 MB mode protects the transient allocation ceiling. -The gate does not measure network transfer, browser rendering, or recorded Sessions, and it is not a continuous performance-trend system. A Node or runner change requires resampling the same workload and reviewing the budgets; a business-implementation change must not relax a budget without new positive and negative control data. +The Session and Node-fold scenarios do not measure network transfer, browser rendering, or recorded Sessions, and they are not a continuous performance-trend system. [Frontend performance budgets](2026-09-06-frontend-performance-budgets.md) own browser workflow measurements. A Node or runner change requires resampling the same workload and reviewing the budgets; a business-implementation change must not relax a budget without new positive and negative control data. diff --git a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.zh.md b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.zh.md index 965b903507..b6608ca79d 100644 --- a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.zh.md +++ b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.zh.md @@ -12,13 +12,13 @@ Session format v2 的推出改变了两条成本随模型输出增长的路径 ## 决定 -Linux pull request 运行必需的 `node 24 / benchmarks` job，执行 `pnpm run check:ci:bench` → `pnpm run test:bench`。私有 `@deepseek-ai/dsh-benchmarks` workspace 拥有 benchmark 专属依赖。该命令先构建 workspace library 和 `benchmarks/.dsh-build/` 下的专用 worker，再调用 `vitest.bench.config.ts`。[标准托管运行器决策](2026-09-06-standard-hosted-benchmark-runner.zh.md)拥有运行器选择及外层 job 超时。该 job 单独运行 benchmark lane；Vitest 逐文件运行，只负责准备输入、启动测量子进程、汇总结果和执行预算断言。每条被计时的 CPU 路径都以纯 Node 执行编译后的 JavaScript，并移除 `NODE_OPTIONS` 且不加载 TypeScript runtime；workspace 裸导入因此从 `benchmarks/node_modules` 通过 package exports 解析到构建后的 `lib/` 入口。 +Linux pull request 运行必需的 `node 24 / benchmarks` job，执行 `pnpm run check:ci:bench` → `pnpm run test:bench`。私有 `@deepseek-ai/dsh-benchmarks` workspace 拥有 benchmark 专属依赖。该命令先构建 workspace library 和 `benchmarks/.dsh-build/` 下的专用 worker，再调用 `vitest.bench.config.ts`。[标准托管运行器决策](2026-09-06-standard-hosted-benchmark-runner.zh.md)拥有运行器选择及外层 job 超时。该 job 单独运行 benchmark lane；Vitest 逐文件运行，只负责准备输入、启动测量子进程、汇总结果和执行预算断言。每条被计时的 Node CPU 路径都以纯 Node 执行编译后的 JavaScript，并移除 `NODE_OPTIONS` 且不加载 TypeScript runtime；workspace 裸导入因此从 `benchmarks/node_modules` 通过 package exports 解析到构建后的 `lib/` 入口。 必需性能 gate 位于顶层 `benchmarks/`，按被测用户路径而非 package 归属组织。Host 文件使用 `*.bench.ts`，Client 面文件使用 `*.bench.client.ts`，场景专属 worker 与 fixture 留在对应 benchmark 旁且不带 benchmark 后缀。包内 `.perf.ts` 文件仍是非门禁诊断；`scripts/` 负责编排而不承载 benchmark case。 Session benchmark 使用固定参数合成 released-v0 输入：200 轮，每轮 500 个 text delta 与 125 个 reasoning delta，共 127,400 个逻辑事件。输入使用 Zstandard，并固定 logical rows 的分组与 frame 拆分，使每次运行处理相同的事件、字节与 frame 分布。fixture 直接构造不可变的 released-v0 physical rows，不依赖当前 runtime 的历史 encoder；压缩以及所有被测读取和 migration 入口仍使用生产代码。输入在计时前写入每个样本独占的临时目录；benchmark 不使用录制的 Session。 -每个 Session endpoint 都针对用户生命周期中的两个时点运行。`first-open` 最初只有 released V0 generation，因此包含 migration 与后继 generation 发布。测试准备阶段在计时外通过同一套生产 migration 生成一次 `post-upgrade-reopen`，再把未改动的 V0 前代和已发布的 V2 后继一起复制到每个样本目录。Reopen 样本使用全新进程，因此测量用户升级完成后的磁盘再次打开，不包含 migration 或进程内 cache。 +每个 Session endpoint 都针对用户生命周期中的两个时点运行。`first-open` 最初只有 released V0 generation，包含 migration；只读消费者不发布后继文件，可写 Agent resume 才会发布。测试准备阶段在计时外通过同一套生产 migration 生成一次 `post-upgrade-reopen`，再把未改动的 V0 前代和已发布的 V2 后继一起复制到每个样本目录。Reopen 样本使用全新进程，因此测量用户升级完成后的磁盘再次打开，不包含 migration 或进程内 cache。 每个 access kind 与 endpoint 的样本都在全新、已编译的 Node 子进程中运行。模块加载、Host 服务初始化和 fixture 准备在测量开始前完成；测量进程不执行额外的预热解析。正常堆模式运行五个独立样本，报告全部样本及最小值、中位数和最大值，并以中位数执行各访问状态独立的固定预算。另一个子进程使用固定 128 MB old-space 上限运行同一路径，只判断能否完成；低堆限制引起的额外 GC 不进入正常时间基线。 @@ -26,7 +26,7 @@ Session benchmark 使用固定参数合成 released-v0 输入：200 轮，每轮 | Benchmark | 被测路径 | 时间指标 | |---|---|---| -| 阶段剖面 | 分别为 first open 与 post-upgrade reopen 执行真实 persistence open、handle read、Session restore 与 projection | `openMs`、`readMs`、`sessionRestoreMs`、`projectionMs` 各自使用固定预算；migration 所等待的编码、写入、verify 与 publish 全部归入 first-open `openMs` | +| 阶段剖面 | 分别为 first open 与 post-upgrade reopen 执行真实 persistence open、handle read、Session restore 与 projection | `openMs`、`readMs`、`sessionRestoreMs`、`projectionMs` 各自使用固定预算；只读 migration 归入 first-open `openMs`；后继编码、verify 与 publish 属于可写 Agent resume | | 首屏历史 | 两种 access kind 分别经 Host Session history controller 读取到首个分页 snapshot | First open 与 reopen 各有一个端到端预算；均包含 source stat、读取、Session restore、projection、分页与 snapshot 构造，first open 还包含 migration；两者都不包含 Gateway 网络传输、Client fold 或浏览器 paint | | Agent resume | 对两种 access kind 分别调用 `ctx.agents.resume()`，直到 Agent 创建、setup、发布与 loop 启动完成 | First open 与 reopen 各有一个端到端预算；两条路径都不与首屏历史串行，也不依赖它留下的 cache | | Client fold | 大小两个 v2 history window 经真实 `ConversationNodeAssembler` 与全部 Chat Definition fold | 大窗口的绝对时间与相对小窗口的缩放比各自使用固定预算 | @@ -50,7 +50,7 @@ Session benchmark 使用固定参数合成 released-v0 输入：200 轮，每轮 | First open | 栈前参考版本 | 249.0 ms | 253.8 ms | 100.7 ms | 26.1 MB | 完成 | | First open | 重复 snapshot 退化实现 | 4,197.5 ms | 4,284.8 ms | 4,197.9 ms | 4.4 MB | 堆耗尽 | | Post-upgrade reopen | 栈前参考版本 | 251.1 ms | 253.8 ms | 100.7 ms | 26.1 MB | 完成 | -| Post-upgrade reopen | 重复 snapshot 退化实现 | 49.2 ms | 50.4 ms | 43.8 ms | 完成 | +| Post-upgrade reopen | 重复 snapshot 退化实现 | 49.2 ms | 50.4 ms | 43.8 ms | 4.5 MB | 完成 | 栈前实现以 V0 作为当前格式，因此 first open 不改变磁盘表示；它的原生 V0 首屏历史与 Agent resume 测量同时适用于两个生命周期行。 @@ -102,4 +102,4 @@ Session benchmark 使用固定参数合成 released-v0 输入：200 轮，每轮 每个 pull request 多付出一个必需 Linux job；该 job 的 Session 部分运行多个短生命周期子进程，以换取冷 cache、独立 V8 heap、明确 GC 状态和可归因的失败。仓库级 benchmark 目录接受有意的跨包测试依赖，而不修改产品 package manifest。固定 Zstandard workload 同时覆盖事件规模与 frame 拓扑；first-open 测量保护一次性升级体验，reopen 测量防止后续打开退化，四阶段预算定位成本归属，首屏预算保护用户可见等待，Agent resume 预算与 GC 后增量保护完整冷恢复及常驻内存，128 MB 模式保护瞬时分配上限。 -该 gate 不测量网络传输、浏览器渲染或真实录制 Session，也不是持续性能趋势系统。Node 或 runner 变化需要用同一 workload 重新采样并评审预算；修改业务实现时不得顺带放宽预算而不提供新的正反例数据。 +Session 与 Node-fold 场景不测量网络传输、浏览器渲染或真实录制 Session，也不是持续性能趋势系统。[前端性能预算](2026-09-06-frontend-performance-budgets.zh.md)拥有浏览器工作流测量。Node 或 runner 变化需要用同一 workload 重新采样并评审预算；修改业务实现时不得顺带放宽预算而不提供新的正反例数据。 diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml new file mode 100644 index 0000000000..06aa949170 --- /dev/null +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md +2026-09-06-frontend-performance-budgets.md: 0ebe97db9532c4922d2e0e8f2bd41613b9e80b6e +2026-09-06-frontend-performance-budgets.zh.md: f58afd5aae763b145887204a63dd5b90d16566b4 diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md new file mode 100644 index 0000000000..0ebe97db95 --- /dev/null +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md @@ -0,0 +1,55 @@ +# Agent Note: Frontend large-session performance budgets + +Status: implemented + +English | [中文](2026-09-06-frontend-performance-budgets.zh.md) + +## Problem + +A fast Node conversation fold does not prove that a browser paints a long conversation or remains responsive while a response streams. Active reconnect also reconstructs a different representation from settled history: a compact prefix becomes public per-chunk Client entries. The [Session performance policy](2026-09-04-session-open-performance-gate.md) supplies an isolated CI job but does not measure these user paths. + +## Decision + +The existing serial benchmark inventory includes two frontend owners: [active reconnect](../../../../benchmarks/active-stream-reconnect/README.md) and a [browser workflow](../../../../benchmarks/long-session-browser/README.md). The browser workflow combines cold open, older-page navigation, first Trajectory activation, return to Chat, and a paced response with trusted keyboard input into one sequential scenario. These are endpoints of one workflow, not independent cold scenarios. The settled conversation-fold benchmark remains unchanged. + +`build:bench` keeps the Node-only library and worker build. `test:bench` additionally builds the Web shell before running all cases; the required benchmark CI job provisions Chromium. Browser cases reuse the shipped-composition Web scaffold with private temporary roots and an atomically assigned loopback port. Only the nondeterministic model is replaced by synthetic replay. The scaffold Host runs under the existing Vitest source resolver; measured Client rendering runs built bundles in fresh Chromium processes. Browser wall times therefore include this test Host, transport, Playwright actionability, and rendering, and are not claims about a published Host process. + +The browser input contains 240 closed turns, 40 tool results, and 20 code fences, plus mixed-language prose and reasoning. Nine older-page actions exhaust this input from its observed 25-turn initial window; the readiness probe follows mounted turn growth rather than duplicating the pagination algorithm. Each sample uses a fresh scaffold and browser. Setup, seeding, browser launch, initial shell load, and sidebar expansion are excluded from open timing. Open ends at transcript availability and an editable composer; page and navigation timings end at their target DOM state. Two animation frames include a rendering opportunity, not hardware presentation or a guarantee that every offscreen node painted. + +The continuation sends 120 text deltas at 8 ms replay pacing. It records click-to-first-visible-reply, trusted draft typing while the completion marker is absent, complete reply wall time through settled persistence, and Chromium main-thread task duration. The complete wall budget adds the fixed 992 ms scripted pacing to a scaled overhead allowance; input and completion have their own enforced budgets. Post-GC browser heap and DOM counts remain diagnostics because one endpoint does not prove a leak. + +Reconnect uses three fresh compiled plain-Node children. Each creates a 100,000-delta reasoning prefix with distinct timestamps and two compact records before timing `ClientAssistantStream.replace()`. GC precedes the baseline and follows replacement while the result remains reachable; replacement time excludes both collections. The report consumes the result after collection and checks that the next dense live frame remains accepted. This measures reconstruction, not transport, rendering, or an entire reconnect workflow. + +## Calibration + +Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149.0.7827.55, at product revision `925e012340`, establish the baseline below. An isolated repeat follows a complete workflow smoke. Each browser sample reports raw endpoint values and every page; the paging verdict uses the median of the sample maxima. Reconnect reports all child measurements. Source reference constants round above observed values; the shared 2× time scale and 1.25× variance allowance produce CI limits. Memory uses only variance allowance. The existing scale comes from Node CI calibration, not a measured x64 browser comparison; browser-specific runner calibration remains an explicit gap. + +| Endpoint | Measured median | Reference allowance | CI limit | +|---|---:|---:|---:| +| Browser open | 166.77 ms | 200 ms | 500 ms | +| Slowest older page | 245.52 ms | 260 ms | 650 ms | +| First Trajectory | 133.45 ms | 160 ms | 400 ms | +| First reply | 1033.07 ms | 1100 ms | 2750 ms | +| Stream main-thread task | 1668.46 ms | 1800 ms | 4500 ms | +| Draft typing | 228.08 ms | 500 ms | 1250 ms | +| Complete response | 1699.26 ms | 1000 ms overhead + 992 ms pacing | 3492 ms | +| Reconnect replacement | 13.83 ms | 16 ms | 40 ms | +| Reconnect retained heap | 23.03 MiB | 24 MiB | 30 MiB | + +Draft typing spans 138.80–420.28 ms across the three isolated samples; its reference covers that observed spread instead of treating the median as a per-keystroke bound. No budget is an environment override. Temporary zero allowances exercise every rejection path; these negative controls prove enforcement, not an optimization or a historical regression. + +## Alternatives considered + +**Use the Node fold as paint evidence.** Rejected because it never performs DOM mutation, layout, or browser scheduling. The focused reconnect case likewise makes no GUI speed claim. + +**Promote the entire manual browser diagnostic into CI.** Rejected because its 1,000-session sidebar and 100-turn soak cover a much broader workload. The bounded required case reuses its shipped scaffold and measurement approach without importing a test module or changing the manual inventory. + +**Coalesce active reconnect chunks.** Rejected as a benchmark shortcut: Client entries expose per-member ordering and timestamps to conversation definitions. The benchmark retains that production behavior; reducing retained entries requires a separate semantic design, not copied product algorithms or a synthetic approximation. + +**Measure stream CPU alone.** Rejected because transport stalls and final-settlement delays can leave main-thread CPU low. The independent input, first-reply, and complete-wall budgets cover those waits. + +## Consequences + +The benchmark layer changes no product implementation or user-visible behavior. It adds approximately fifteen seconds of local browser/reconnect execution plus Web build and browser provisioning to the existing isolated CI lane. A fresh browser discards previous caches, but each workflow deliberately retains its own loaded history and previously activated Trajectory during continuation. + +The baseline is independently mergeable and protects current performance; optimization layers tighten budgets only with repeated measurements and focused semantic tests. It does not cover sidebar cardinality, an hours-long soak, GPU presentation, real model latency, a published Host launch, or reconnect rendering. The manual Web diagnostic and existing functional browser tests retain those separate responsibilities. The existing Session performance note remains active because it owns Node calibration and persistence rationale; this note extends rather than supersedes it. diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md new file mode 100644 index 0000000000..f58afd5aae --- /dev/null +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md @@ -0,0 +1,55 @@ +# Agent Note: 前端长 Session 性能预算 + +Status: implemented + +[English](2026-09-06-frontend-performance-budgets.md) | 中文 + +## 问题 + +Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式回复期间保持响应。活跃重连还会重建与已结算历史不同的表示：紧凑前缀变成公开的逐 chunk Client 条目。[Session 性能策略](2026-09-04-session-open-performance-gate.zh.md)提供隔离的 CI job，但不测量这些用户路径。 + +## 决策 + +现有串行基准清单包含两个前端所有者：[活跃重连](../../../../benchmarks/active-stream-reconnect/README.zh.md)和[浏览器工作流](../../../../benchmarks/long-session-browser/README.zh.md)。浏览器工作流在一个顺序场景中组合冷打开、更早分页导航、首次激活 Trajectory、返回 Chat，以及伴随真实键盘输入的有节奏回复。这些是同一工作流的测量终点，而不是相互独立的冷场景。已结算对话折叠基准保持不变。 + +`build:bench` 保留仅 Node 的 library 与 worker 构建。`test:bench` 额外构建 Web shell 后再运行所有用例；必需的基准 CI job 安装 Chromium。浏览器用例复用产品组合的 Web scaffold，使用私有临时目录和原子分配的回环端口。只有不确定的模型被合成重放替代。scaffold Host 通过现有 Vitest 源码解析器运行；被测 Client 渲染在全新 Chromium 进程中执行构建后的 bundle。因此浏览器壁钟时间包含测试 Host、传输、Playwright 可交互性等待及渲染，不代表发布版 Host 进程。 + +浏览器输入包含 240 个已关闭轮次、40 个工具结果和 20 个代码块，以及混合语言正文和推理。从观察到的初始 25 轮窗口开始，九次更早分页操作读完该输入；就绪探针跟踪已挂载轮次增长，不复制分页算法。每个样本使用全新 scaffold 和浏览器。环境准备、数据播种、浏览器启动、初始 shell 加载及侧栏展开不计入打开时间。打开测量在对话可用且输入框可编辑时结束；分页与导航测量在目标 DOM 状态出现时结束。两次动画帧包含一次渲染机会，不代表硬件显示或保证每个屏幕外节点都已绘制。 + +续接以 8 ms 重放间隔发送 120 个文本 delta。它记录点击到首段可见回复的时间、完成标记尚未出现时的真实草稿键入、直到持久化结算的完整回复壁钟时间，以及 Chromium 主线程任务时间。完整壁钟预算在缩放后的额外开销额度上加固定的 992 ms 脚本节奏；输入和完成均有独立执行的预算。强制 GC 后的浏览器 heap 和 DOM 数量仍仅供诊断，因为单个终点不能证明泄漏。 + +重连使用三个全新编译后的纯 Node 子进程。各进程在计时 `ClientAssistantStream.replace()` 前创建包含不同时间戳、两条紧凑记录和 100,000 个 delta 的推理前缀。在基线前执行 GC，并在结果仍可达时于替换后再次 GC；替换时间不含两次回收。报告在回收后消费结果，并检查下一个稠密序号的实时 frame 仍被接受。这测量重建，不测量传输、渲染或完整重连工作流。 + +## 校准 + +在 arm64 参考机器、Node 24.19、Chromium 149.0.7827.55 和产品版本 `925e012340` 上，三个样本的中位数建立下表基线。完整工作流 smoke 后执行一次隔离重复测量。每个浏览器样本报告原始终点数据和每一页；分页判定使用各样本最大值的中位数。重连报告全部子进程测量。源码参考常量向上取整覆盖观察值；共享的 2× 时间倍率和 1.25× 方差余量产生 CI 限制。内存仅使用方差余量。现有倍率来自 Node CI 校准，并非实测 x64 浏览器对比；浏览器专用 runner 校准仍是明确缺口。 + +| 终点 | 实测中位数 | 参考额度 | CI 限制 | +|---|---:|---:|---:| +| 浏览器打开 | 166.77 ms | 200 ms | 500 ms | +| 最慢更早分页 | 245.52 ms | 260 ms | 650 ms | +| 首次 Trajectory | 133.45 ms | 160 ms | 400 ms | +| 首段回复 | 1033.07 ms | 1100 ms | 2750 ms | +| 流式主线程任务 | 1668.46 ms | 1800 ms | 4500 ms | +| 草稿键入 | 228.08 ms | 500 ms | 1250 ms | +| 完整回复 | 1699.26 ms | 1000 ms 额外开销 + 992 ms 节奏 | 3492 ms | +| 重连替换 | 13.83 ms | 16 ms | 40 ms | +| 重连保留 heap | 23.03 MiB | 24 MiB | 30 MiB | + +三个隔离样本中的草稿键入时间为 138.80–420.28 ms；参考额度覆盖观察到的波动，而不把中位数作为单次按键上限。预算不能通过环境变量覆盖。临时零额度覆盖每条拒绝路径；这些负向对照证明预算执行，而非优化或历史回归。 + +## 考虑过的替代方案 + +**用 Node 折叠作为绘制证据。** 拒绝，因为它不执行 DOM 修改、布局或浏览器调度。聚焦重连用例同样不声称 GUI 提速。 + +**把整个手动浏览器诊断提升到 CI。** 拒绝，因为其 1,000 Session 侧栏和 100 轮 soak 覆盖更广的工作负载。受限的必需用例复用其产品 scaffold 和测量方式，不导入测试模块，也不改变手动清单。 + +**合并活跃重连 chunk。** 不能作为基准捷径：Client 条目向对话定义公开每个成员的顺序和时间戳。基准保留该生产行为；减少保留条目需要独立的语义设计，而非复制产品算法或使用合成近似。 + +**只测量流式 CPU。** 拒绝，因为传输停顿和最终结算延迟可能不增加主线程 CPU。独立的输入、首段回复和完整壁钟预算覆盖这些等待。 + +## 影响 + +基准层不改变产品实现或用户可见行为。它在现有隔离 CI lane 中增加约十五秒的本地浏览器与重连执行，以及 Web 构建和浏览器安装成本。全新浏览器丢弃此前的缓存，但每个工作流刻意在续接期间保留自身已加载历史和曾激活的 Trajectory。 + +基线可独立合并并保护现有性能；优化层只有在重复测量与聚焦语义测试支持下才收紧预算。它不覆盖侧栏数量级、数小时 soak、GPU 显示、真实模型延迟、发布版 Host 启动或重连渲染。手动 Web 诊断和现有功能浏览器测试继续各负其责。现有 Session 性能记录保持活跃，因为它拥有 Node 校准和持久化理由；本记录扩展而不替代它。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c81db5a69f..03afdec2ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -206,6 +206,9 @@ jobs: - name: Install (immutable) run: pnpm install --frozen-lockfile + - name: Install benchmark browser + run: pnpm --filter @deepseek-ai/dsh-benchmarks exec playwright install --with-deps chromium + - name: Run performance benchmarks env: DSH_GATE_VERBOSE: '1' diff --git a/benchmarks/AGENTS.md b/benchmarks/AGENTS.md index 933d4d25bf..2f40d70afb 100644 --- a/benchmarks/AGENTS.md +++ b/benchmarks/AGENTS.md @@ -5,6 +5,7 @@ This tree owns required, repository-level performance gates whose measured user - Organize benchmarks by measured user path, one directory per path. Do not mirror the package tree. - Host cases use `*.bench.ts`; Client-face cases use `*.bench.client.ts`. Worker, fixture, and support modules do not carry a benchmark suffix. - The private `@deepseek-ai/dsh-benchmarks` workspace owns benchmark-only dependencies. `test:bench` builds workspace libraries and `benchmarks/.dsh-build/` workers before Vitest orchestration. Timed CPU work runs in those workers under plain Node, without a TypeScript loader; runtime package imports must resolve to built `lib/` entries. +- Browser workflow cases drive built Client bundles through the shared shipped-composition Web scaffold. Report its source-resolved test Host separately from published-Host evidence; two animation frames prove a rendering opportunity, not hardware presentation. Use fresh browsers and private scaffold worlds per sample. - Synthesize fixed inputs from reviewed constants. Never use recorded Sessions, user material, ambient repositories, or network services. - Run process-level wall-clock and retained-memory samples in fresh children with private `mkdtemp` roots. Pure synchronous folds create a fresh object graph per sample and must not mutate process-global state. Bound every child, await exit, and remove owned roots after failure as well as success. - Record reference-machine expectations separately from the shared CI time scale and variance headroom. Do not apply the time scale to memory or dimensionless ratios. diff --git a/benchmarks/active-stream-reconnect/README.i18n.yaml b/benchmarks/active-stream-reconnect/README.i18n.yaml new file mode 100644 index 0000000000..b4a785f27e --- /dev/null +++ b/benchmarks/active-stream-reconnect/README.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write benchmarks/active-stream-reconnect/README.md +README.md: 2f10512f144b923df2d89ff2766c4acf1059a652 +README.zh.md: b0c97ea7f06281a71f4e633b9f60f5f5b2ebf4fa diff --git a/benchmarks/active-stream-reconnect/README.md b/benchmarks/active-stream-reconnect/README.md new file mode 100644 index 0000000000..2f10512f14 --- /dev/null +++ b/benchmarks/active-stream-reconnect/README.md @@ -0,0 +1,7 @@ +# Active Assistant reconnect benchmark + +English | [中文](README.zh.md) + +[reconnect.bench.client.ts](reconnect.bench.client.ts) measures the production Client fold when a reconnect carries an unfinished 100,000-delta reasoning prefix. A compiled private adapter reaches `ClientAssistantStream.replace()` without adding product exports. Three fresh plain-Node workers synthesize the compact baseline before timing; replacement time and retained heap after forced GC have separate median budgets. The next dense live frame must still be accepted. + +Build with `pnpm run build:bench`, then select `benchmarks/active-stream-reconnect` in `vitest.bench.config.ts`. This focused Node workload neither builds nor measures browser rendering. [Frontend performance budgets](../../.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md) records calibration and exclusions. diff --git a/benchmarks/active-stream-reconnect/README.zh.md b/benchmarks/active-stream-reconnect/README.zh.md new file mode 100644 index 0000000000..b0c97ea7f0 --- /dev/null +++ b/benchmarks/active-stream-reconnect/README.zh.md @@ -0,0 +1,7 @@ +# 活跃 Assistant 重连基准 + +[English](README.md) | 中文 + +[reconnect.bench.client.ts](reconnect.bench.client.ts) 测量重连携带未完成的 100,000 个 reasoning delta 前缀时，生产 Client 的折叠成本。编译后的私有适配器调用 `ClientAssistantStream.replace()`，不增加产品导出。三个全新纯 Node worker 在计时前合成紧凑 baseline；替换时间与强制 GC 后的保留 heap 分别执行中位数预算检查。下一个稠密序号的实时 frame 仍须被接受。 + +通过 `pnpm run build:bench` 构建，再在 `vitest.bench.config.ts` 中选择 `benchmarks/active-stream-reconnect`。该聚焦 Node workload 既不构建也不测量浏览器渲染。[前端性能预算](../../.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md)记录校准与排除项。 diff --git a/benchmarks/active-stream-reconnect/reconnect.bench.client.ts b/benchmarks/active-stream-reconnect/reconnect.bench.client.ts new file mode 100644 index 0000000000..374173495b --- /dev/null +++ b/benchmarks/active-stream-reconnect/reconnect.bench.client.ts @@ -0,0 +1,34 @@ +/** Required baseline budgets for reconnecting during a large active Assistant stream. */ +import { join } from 'node:path' +import { expect, it } from 'vitest' +import { runBuiltBenchmarkWorker } from '../support/built-worker.ts' +import { ciTimeBudget, PERFORMANCE_BUDGET_HEADROOM } from '../support/calibration.ts' +import type { ReconnectReport } from './reconnect.worker.client.ts' + +const REFERENCE_REPLACE_MS = 16 +const REFERENCE_RETAINED_MB = 24 +const SAMPLES = 3 + +it('reconstructs a 100000-delta live prefix within baseline time and retained-memory budgets', async () => { + const samples: ReconnectReport[] = [] + for (let sample = 0; sample < SAMPLES; sample++) { + const run = await runBuiltBenchmarkWorker({ + worker: join(import.meta.dirname, '../.dsh-build/active-stream-reconnect/reconnect.worker.js'), + exposeGc: true, timeoutMs: 30000, + }) + expect(run.timedOut, run.stderr).toBe(false) + expect(run.signal, run.stderr).toBeNull() + expect(run.exitCode, run.stderr).toBe(0) + if (run.report === undefined) throw new Error('reconnect worker omitted report') + expect(run.report.nextFrame).toBe('transient') + expect(run.report.entries).toBeGreaterThan(0) + samples.push(run.report) + } + const replaceMs = samples.map(sample => sample.replaceMs).toSorted((a, b) => a - b)[1]! + const retainedMb = samples.map(sample => sample.retainedMb).toSorted((a, b) => a - b)[1]! + const budgetMs = ciTimeBudget(REFERENCE_REPLACE_MS) + const budgetMb = REFERENCE_RETAINED_MB * PERFORMANCE_BUDGET_HEADROOM + console.log(JSON.stringify({ benchmark: 'active-stream-reconnect', samples, median: { replaceMs, retainedMb }, referenceMs: REFERENCE_REPLACE_MS, referenceMb: REFERENCE_RETAINED_MB, budgetMs, budgetMb })) + expect.soft(replaceMs).toBeLessThanOrEqual(budgetMs) + expect.soft(retainedMb).toBeLessThanOrEqual(budgetMb) +}) diff --git a/benchmarks/active-stream-reconnect/reconnect.worker.client.ts b/benchmarks/active-stream-reconnect/reconnect.worker.client.ts new file mode 100644 index 0000000000..a1a315a263 --- /dev/null +++ b/benchmarks/active-stream-reconnect/reconnect.worker.client.ts @@ -0,0 +1,49 @@ +/** Compiled production Client fold for a reconnect during a long Assistant attempt. */ +import { performance } from 'node:perf_hooks' +import { AssistantStreamAccumulator } from '@deepseek-ai/dsh-llm/assistant-stream' +import { LlmAttemptId } from '@deepseek-ai/dsh-llm/brand' +import type { SessionAssistantStreamBaseline } from '@deepseek-ai/dsh-api-session-controller/types' +// The Client implementation has no plain-Node export; only this adapter is bundled. +import { ClientAssistantStream } from '../../packages/api/session-controller/src/client/sessions/assistant-stream.ts' +import { assertBuiltBenchmarkRuntime } from '../support/built-worker.ts' + +/** Measurements of replace() only; fixture construction and forced GC are excluded. */ +export interface ReconnectReport { + readonly deltas: number + readonly records: number + readonly entries: number + readonly replaceMs: number + readonly retainedMb: number + readonly nextFrame: string | undefined +} + +assertBuiltBenchmarkRuntime(import.meta.url, { + '@deepseek-ai/dsh-llm/assistant-stream': import.meta.resolve('@deepseek-ai/dsh-llm/assistant-stream'), +}) +const deltas = 100000 +const accumulator = new AssistantStreamAccumulator() +accumulator.push({ time: 1700000000000, chunk: { type: 'block-start', index: 0, blockType: 'reasoning' } }) +for (let index = 0; index < deltas; index++) { + accumulator.push({ time: 1700000000001 + index, chunk: { type: 'reasoning-delta', index: 0, text: 'token ' } }) +} +const attemptId = LlmAttemptId('synthetic-reconnect') +const nextIndex = deltas + 1 +const baseline: SessionAssistantStreamBaseline = { + revision: nextIndex + 1, + activeAttempt: { + attemptId, startedAfterSeq: -1, turn: 1, step: 1, nextIndex, + stream: JSON.parse(JSON.stringify(accumulator.snapshot())) as NonNullable['stream'], + }, +} +if (globalThis.gc === undefined) throw new Error('reconnect benchmark requires --expose-gc') +globalThis.gc() +const before = process.memoryUsage().heapUsed +const client = new ClientAssistantStream() +const start = performance.now() +const visible = client.replace([], baseline) +const replaceMs = performance.now() - start +globalThis.gc() +const retainedMb = (process.memoryUsage().heapUsed - before) / 1048576 +const next = client.acceptFrame({ type: 'chunk', attemptId, revision: nextIndex + 2, index: nextIndex, time: 1700000000001 + deltas, chunk: { type: 'reasoning-delta', index: 0, text: 'suffix' } }) +const report: ReconnectReport = { deltas, records: baseline.activeAttempt!.stream.length, entries: visible.length, replaceMs, retainedMb, nextFrame: next?.type } +process.stdout.write(JSON.stringify(report) + '\n') diff --git a/benchmarks/long-session-browser/README.i18n.yaml b/benchmarks/long-session-browser/README.i18n.yaml new file mode 100644 index 0000000000..e95961a823 --- /dev/null +++ b/benchmarks/long-session-browser/README.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write benchmarks/long-session-browser/README.md +README.md: 53385706703736a76568b0b55141d40731068ed2 +README.zh.md: 5e709ec6bbd53a420fcf8ad4f4f414b4552549b6 diff --git a/benchmarks/long-session-browser/README.md b/benchmarks/long-session-browser/README.md new file mode 100644 index 0000000000..5338570670 --- /dev/null +++ b/benchmarks/long-session-browser/README.md @@ -0,0 +1,17 @@ +# Long-session browser benchmark + +English | [中文](README.zh.md) + +This reference describes the required Chromium workflow in [long-session.bench.ts](long-session.bench.ts). It opens a synthetic 240-turn Session, loads every older page, visits Trajectory, returns to Chat, and submits a paced reply while typing another draft. The shipped Web scaffold owns the isolated home, persistence, replay adapter, and loopback listener; Chromium loads the built Web artifacts, not a replacement development server. + +## Run + +`pnpm run test:bench` builds libraries, workers, and Web artifacts before running the serial benchmark inventory. With artifacts already built, select this directory through `pnpm exec vitest run --config vitest.bench.config.ts benchmarks/long-session-browser`. Install Chromium through the benchmark workspace before the first run. + +## Measurements + +Three fresh browser processes and scaffold worlds produce raw samples and median verdicts. Open and paging end after the expected transcript state and two animation frames; this includes a rendering opportunity, not a hardware presentation timestamp. Paging reports every page and gates the median of each sample’s slowest page. Stream reports first visible reply, trusted draft typing, complete reply wall time, and Chromium main-thread task duration. Heap after forced GC and DOM counts are diagnostics, not leak budgets. + +The fixture contains mixed-language prompts, prose, reasoning, 20 code fences, and 40 synthetic tool results. No model, tool, external network, recorded Session, or private Harness home supplies its content. Streaming uses 120 text deltas at 8 ms replay pacing through the real composer, agent loop, transport, and persistence. + +The [decision record](../../.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md) owns calibration, exclusions, and alternatives. The larger [manual diagnostic](../../apps/web/tests/complex-history.perf.ts) remains separate. diff --git a/benchmarks/long-session-browser/README.zh.md b/benchmarks/long-session-browser/README.zh.md new file mode 100644 index 0000000000..5e709ec6bb --- /dev/null +++ b/benchmarks/long-session-browser/README.zh.md @@ -0,0 +1,17 @@ +# 长 Session 浏览器基准 + +[English](README.md) | 中文 + +本文说明 [long-session.bench.ts](long-session.bench.ts) 中必需的 Chromium 工作流。它打开一个合成的 240 轮 Session，加载所有更早的分页，访问 Trajectory，返回 Chat，并在流式回复期间输入下一条草稿。随产品维护的 Web scaffold 拥有隔离的主目录、持久化、重放适配器和回环监听器；Chromium 加载构建后的 Web 产物，而非替代开发服务器。 + +## 运行 + +`pnpm run test:bench` 先构建 library、worker 和 Web 产物，再串行运行基准清单。产物已构建时，通过 `pnpm exec vitest run --config vitest.bench.config.ts benchmarks/long-session-browser` 选择此目录。首次运行前，通过 benchmark workspace 安装 Chromium。 + +## 测量 + +三个全新浏览器进程与 scaffold 环境产生原始样本及中位数判定。打开和分页在预期对话状态出现且经过两次动画帧后结束；这包含一次渲染机会，而非硬件显示时间戳。分页报告每一页，并对各样本最慢分页时间的中位数执行预算检查。流式报告首段可见回复、真实草稿键入、完整回复壁钟时间和 Chromium 主线程任务时间。强制 GC 后的 heap 与 DOM 数量仅供诊断，不作为泄漏预算。 + +fixture（测试前置数据）包含混合语言提示、正文、推理、20 个代码块和 40 个合成工具结果。其内容不来自模型、工具、外部网络、录制 Session 或私有 Harness 主目录。流式回复以 8 ms 重放间隔发送 120 个文本 delta，经过真实输入框、agent loop（智能体循环）、传输与持久化。 + +[决策记录](../../.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md)拥有校准、排除项与替代方案。更大规模的[手动诊断](../../apps/web/tests/complex-history.perf.ts)保持独立。 diff --git a/benchmarks/long-session-browser/long-session.bench.ts b/benchmarks/long-session-browser/long-session.bench.ts new file mode 100644 index 0000000000..7a4120778b --- /dev/null +++ b/benchmarks/long-session-browser/long-session.bench.ts @@ -0,0 +1,139 @@ +/** Required browser budgets for opening, paging and continuing synthetic long history. */ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { performance } from 'node:perf_hooks' +import { chromium, type Page, type CDPSession } from 'playwright' +import { expect, it } from 'vitest' +import { launchWebScaffold, seedSession, watchConsole, webSnapshotMode } from '../../apps/web/tests/scaffold.ts' +import { newEnglishPage } from '../../apps/web/tests/support.ts' +import { ciTimeBudget } from '../support/calibration.ts' +import { HISTORY_TURNS, SESSION_ID, FIRST, DONE, DELTAS, PACE_MS, syntheticHistory, syntheticReply } from './synthetic-history.ts' + +const SAMPLES = 3 +const TAIL = '[data-chat-flow-key^="9:turn-tail"]' +const REFERENCE = { open: 200, page: 260, trajectory: 160, first: 1100, streamTask: 1800, input: 500, streamWall: 1000 } +const REPLAY_DURATION_MS = (DELTAS + 4) * PACE_MS + +async function painted(page: Page): Promise { + // Two rAF callbacks include a rendering opportunity, not a GPU presentation timestamp. + await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(() => resolve())))) +} + +async function measure(page: Page, action: () => Promise): Promise { + const start = performance.now() + await action() + await painted(page) + return performance.now() - start +} + +async function taskMs(cdp: CDPSession): Promise { + const result = await cdp.send('Performance.getMetrics') + const metric = result.metrics.find(metric => metric.name === 'TaskDuration') + if (metric === undefined) throw new Error('Chromium TaskDuration missing') + return metric.value * 1000 +} + +function median(values: number[]): number { + return values.toSorted((a, b) => a - b)[Math.floor(values.length / 2)]! +} + +it('opens, pages, navigates and streams into a 240-turn browser history', async () => { + if (webSnapshotMode() !== 'replay') throw new Error('browser benchmarks require keyless replay mode') + const samples: { open: number; page: number; trajectory: number; first: number; streamTask: number; streamWall: number; input: number; heapMb: number; nodes: number }[] = [] + for (let sample = 0; sample < SAMPLES; sample++) { + const failures: unknown[] = [] + const root = await mkdtemp(join(tmpdir(), 'dsh-browser-benchmark-')) + try { + const replayOverride = join(root, 'reply.json') + await writeFile(replayOverride, JSON.stringify([{ kind: 'chunks', chunks: syntheticReply() }])) + const scaffold = await launchWebScaffold({ replayFixture: join(root, 'override-only.jsonl'), replayOverride, paceMs: PACE_MS, replayContextWindow: 10000000 }) + try { + await seedSession(scaffold, syntheticHistory(), SESSION_ID) + const browser = await chromium.launch({ headless: true }) + try { + const page = await newEnglishPage(browser) + const consoleWatch = watchConsole(page) + page.setDefaultTimeout(30000) + await page.goto(scaffold.authenticatedUrl, { waitUntil: 'load' }) + expect(new URL(page.url()).origin).toBe(scaffold.baseUrl) + console.log(JSON.stringify({ benchmark: 'long-session-browser/server', url: scaffold.baseUrl, browser: browser.version(), sample })) + await page.waitForSelector('[class*="frame"]') + await page.getByRole('treeitem').first().click() + const result = page.getByRole('treeitem').nth(1) + await result.waitFor() + const open = await measure(page, async () => { + await result.click() + await page.locator(TAIL).last().waitFor() + await page.locator('[data-composer-input][contenteditable="true"]').last().waitFor() + }) + const pages: number[] = [] + const initialTurns = await page.locator(TAIL).count() + expect(initialTurns).toBeGreaterThan(0) + expect(initialTurns).toBeLessThan(HISTORY_TURNS) + let count = initialTurns + while (count < HISTORY_TURNS) { + pages.push(await measure(page, async () => { + await page.getByRole('button', { name: 'Load earlier', exact: true }).click() + await page.waitForFunction(({ selector, previous }) => document.querySelectorAll(selector).length > previous, { selector: TAIL, previous: count }) + })) + count = await page.locator(TAIL).count() + } + const trajectory = await measure(page, async () => { + await page.getByRole('tab', { name: 'Trajectory', exact: true }).click() + await page.getByRole('searchbox', { name: 'Search trajectory', exact: true }).waitFor() + await page.getByRole('row').last().waitFor() + }) + await page.getByRole('tab', { name: 'Chat', exact: true }).click() + await page.waitForFunction(selector => document.querySelectorAll(selector).length === 240, TAIL) + const composer = page.locator('[data-composer-input][contenteditable="true"]').last() + await composer.fill('Continue the synthetic review and summarize the validation. '.repeat(30)) + const cdp = await page.context().newCDPSession(page) + await cdp.send('Performance.enable') + const beforeTask = await taskMs(cdp) + const settled = scaffold.whenTurnSettled(60000).then( + () => ({ ok: true as const }), + (error: unknown) => ({ ok: false as const, error }), + ) + const started = performance.now() + await page.getByRole('button', { name: 'Send message', exact: true }).click() + await page.getByText(FIRST, { exact: false }).last().waitFor() + await painted(page) + const first = performance.now() - started + expect(await page.getByText(DONE, { exact: false }).count()).toBe(0) + // Trusted keyboard input while the response is live, rather than a synthetic heartbeat. + const input = await measure(page, async () => { + await composer.click() + await page.keyboard.type('next synthetic question') + await expect.poll(() => composer.textContent()).toBe('next synthetic question') + }) + await page.getByText(DONE, { exact: false }).last().waitFor() + const settlement = await settled + if (!settlement.ok) throw settlement.error + await painted(page) + const streamWall = performance.now() - started + const streamTask = await taskMs(cdp) - beforeTask + await cdp.send('HeapProfiler.collectGarbage') + const metrics = (await cdp.send('Performance.getMetrics')).metrics + const heap = metrics.find(metric => metric.name === 'JSHeapUsedSize') + if (heap === undefined) throw new Error('Chromium heap metric missing') + samples.push({ open, page: Math.max(...pages), trajectory, first, streamTask, streamWall, input, heapMb: heap.value / 1048576, nodes: await page.locator('*').count() }) + console.log(JSON.stringify({ benchmark: 'long-session-browser/sample', sample, initialTurns, pages, ...samples.at(-1) })) + expect(consoleWatch.pageErrors).toEqual([]) + expect(consoleWatch.warnings).toEqual([]) + } catch (error) { failures.push(error) } finally { + await browser.close().catch((error: unknown) => failures.push(error)) + } + } catch (error) { failures.push(error) } finally { + await scaffold.close().catch((error: unknown) => failures.push(error)) + } + } catch (error) { failures.push(error) } finally { + await rm(root, { recursive: true, force: true }).catch((error: unknown) => failures.push(error)) + } + if (failures.length > 0) throw new AggregateError(failures, 'browser benchmark failed') + } + const aggregate = Object.fromEntries(Object.keys(REFERENCE).map(key => [key, median(samples.map(sample => sample[key as keyof typeof REFERENCE]))])) + const budgets = Object.fromEntries(Object.entries(REFERENCE).map(([key, value]) => [key, ciTimeBudget(value) + (key === 'streamWall' ? REPLAY_DURATION_MS : 0)])) + console.log(JSON.stringify({ benchmark: 'long-session-browser/median', turns: HISTORY_TURNS, deltas: DELTAS, paceMs: PACE_MS, samples, aggregate, referenceMs: REFERENCE, budgets })) + for (const [key, value] of Object.entries(aggregate)) expect.soft(value, key).toBeLessThanOrEqual(budgets[key]!) +}) diff --git a/benchmarks/long-session-browser/synthetic-history.ts b/benchmarks/long-session-browser/synthetic-history.ts new file mode 100644 index 0000000000..1d3b22aedb --- /dev/null +++ b/benchmarks/long-session-browser/synthetic-history.ts @@ -0,0 +1,69 @@ +/** Synthetic current-generation history and paced reply for browser measurements. */ +import { createAssistantMessage, createUserMessage, createToolResultMessage, ToolCallId } from '@deepseek-ai/dsh-llm' +import type { StreamChunk } from '@deepseek-ai/dsh-llm' +import { Session, SessionId, SESSION_FORMAT_VERSION } from '@deepseek-ai/dsh-session' +import type {} from '@deepseek-ai/dsh-session-title' + +/** Closed turns in the browser history workload. */ +export const HISTORY_TURNS = 240 +/** Identity private to each isolated scaffold. */ +export const SESSION_ID = 'benchmark-browser-history' +const TITLE = 'SYNTHETIC_BROWSER_HISTORY' +/** First streamed text marker. */ +export const FIRST = 'SYNTHETIC_REPLY_FIRST' +/** Last streamed text marker. */ +export const DONE = 'SYNTHETIC_REPLY_DONE' +/** Paced text chunks per continuation. */ +export const DELTAS = 120 +/** Replay delay per stream chunk, in milliseconds. */ +export const PACE_MS = 8 + +/** Create mixed prose, code, reasoning and tool history without reading user data. + * @returns Current Session JSONL accepted by the shared Web seeder. + */ +export function syntheticHistory(): string { + const session = Session.create(SessionId(SESSION_ID)) + for (let turn = 1; turn <= HISTORY_TURNS; turn++) { + session.append('turn/start', { turn }) + const user = session.append('user/message', createUserMessage({ + content: [{ type: 'text', text: 'Review synthetic change ' + String(turn) + ': 检查增量渲染。 '.repeat(30) }], + source: { kind: 'user' }, + }), { surfaceOp: 'append' }) + if (turn === 1) session.append('session/title', { title: TITLE, messageSeqs: [user.seq], source: { kind: 'fallback' } }) + session.append('step/start', { turn, step: 1 }) + const callId = ToolCallId('synthetic-tool-' + String(turn)) + const tool = turn % 6 === 0 + const code = turn % 12 === 0 + ? '\n\n```ts\n' + Array.from({ length: 60 }, (_, i) => 'const value' + String(i) + ' = ' + String(i)).join('\n') + '\n```' + : '' + session.append('assistant/message', { + turn, step: 1, stream: [], + message: createAssistantMessage({ + source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' }, + content: [ + { type: 'reasoning', text: 'Compare the synthetic module and test. '.repeat(40) }, + { type: 'text', text: 'Synthetic answer ' + String(turn) + '. ' + 'Preserve ordering and validate the output. '.repeat(30) + code }, + ...tool ? [{ type: 'tool-call' as const, id: callId, name: 'synthetic_tool', arguments: '{"path":"src/example.ts"}' }] : [], + ], + }), + usage: { inputTokens: 4000, outputTokens: 800 }, + }, { surfaceOp: 'append' }) + if (tool) { + const call = session.append('tool/call', { turn, step: 1, callId, name: 'synthetic_tool', arguments: '{"path":"src/example.ts"}' }) + session.append('tool/result', { turn, step: 1, message: createToolResultMessage({ + callId, isError: false, content: [{ type: 'text', text: 'Synthetic tool output line.\n'.repeat(160) }], + }) }, { surfaceOp: 'append', sourceEventSeqs: [call.seq] }) + } + session.append('step/end', { turn, step: 1 }) + session.append('turn/end', { turn, reason: { kind: 'completed' } }) + } + return [JSON.stringify({ type: 'session', version: SESSION_FORMAT_VERSION, id: '{{sessionId}}', createdAt: 1700000000000, cwd: '{{cwd}}', isSeeded: false, delegationDepth: 0 }), ...session.snapshotEvents().map(event => JSON.stringify(event)), ''].join('\n') +} + +/** Create one paced response; replay owns delays outside the browser. + * @returns Stream chunks ending in a visible completion marker. + */ +export function syntheticReply(): StreamChunk[] { + const deltas = Array.from({ length: DELTAS }, (_, i) => i === 0 ? FIRST + ' ' : i === DELTAS - 1 ? DONE : 'Synthetic response ' + String(i) + '. ') + return [{ type: 'block-start', index: 0, blockType: 'text' }, ...deltas.map(text => ({ type: 'text-delta' as const, index: 0, text })), { type: 'block-end', index: 0, block: { type: 'text', text: deltas.join('') } }, { type: 'usage', usage: { inputTokens: 4000, outputTokens: 800 } }, { type: 'finish', reason: { kind: 'stop' } }] +} diff --git a/benchmarks/package.json b/benchmarks/package.json index 2cf0eec369..cc8c60cf71 100644 --- a/benchmarks/package.json +++ b/benchmarks/package.json @@ -5,6 +5,8 @@ "private": true, "type": "module", "devDependencies": { + "playwright": "^1.49.0", + "@deepseek-ai/dsh-llm-replay": "workspace:^", "@deepseek-ai/cordis": "workspace:^", "@deepseek-ai/dsh-agent": "workspace:^", "@deepseek-ai/dsh-agent-loop": "workspace:^", diff --git a/benchmarks/tsdown.config.ts b/benchmarks/tsdown.config.ts index 3c8d6a9ab5..2b7a7d2bb9 100644 --- a/benchmarks/tsdown.config.ts +++ b/benchmarks/tsdown.config.ts @@ -14,6 +14,13 @@ const shared = { /** Compile measured benchmark workers while keeping workspace packages on their built `lib` entries. */ export default defineConfig([ + { + ...shared, + entry: { 'reconnect.worker': 'active-stream-reconnect/reconnect.worker.client.ts' }, + outDir: '.dsh-build/active-stream-reconnect', + clean: true, + tsconfig: 'tsconfig.client.json', + }, { ...shared, entry: { diff --git a/package.json b/package.json index 801b00c765..296b0d20ca 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "test:coverage": "vitest run --coverage", "test:coverage:partitioned": "tsx scripts/run-coverage-partitions.ts", "test:e2e": "vitest run --config vitest.e2e.config.ts", - "test:bench": "npm run build:bench && npm run test:bench:built", + "test:bench": "npm run build:bench && npm run build:web && npm run test:bench:built", "test:bench:built": "vitest run --config vitest.bench.config.ts", "test:expected": "vitest run --config vitest.expected.config.ts", "test:expected:refresh": "DSH_SNAPSHOT=refresh vitest run --config vitest.expected.config.ts", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cf86833ddb..dadcc170fe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -581,6 +581,9 @@ importers: '@deepseek-ai/dsh-llm': specifier: workspace:^ version: link:../packages/llm/llm + '@deepseek-ai/dsh-llm-replay': + specifier: workspace:^ + version: link:../packages/test-support/llm-replay '@deepseek-ai/dsh-sdk-client': specifier: workspace:^ version: link:../packages/sdk/client @@ -620,6 +623,9 @@ importers: '@deepseek-ai/dsh-typert-protocol': specifier: workspace:^ version: link:../packages/typert/protocol + playwright: + specifier: ^1.49.0 + version: 1.61.1 native/landlock-run: devDependencies: diff --git a/vitest.bench.config.ts b/vitest.bench.config.ts index 81a4652718..5e355d85d6 100644 --- a/vitest.bench.config.ts +++ b/vitest.bench.config.ts @@ -3,8 +3,9 @@ import { defineConfig } from 'vitest/config' import { standardDecoratorPlugin, vitestExecArgv } from './vitest.shared.ts' /** - * CI performance gate. Vitest orchestrates compiled plain-Node workers under - * `.dsh-build/benchmarks/`; timed product work never runs through its source transform. + * CI performance gate. Node CPU cases use compiled plain-Node workers under + * `benchmarks/.dsh-build/`; browser cases drive built Client artifacts through + * the shared shipped-composition Web scaffold. * Files run one at a time so a measurement never shares the CPU with another * benchmark. */ From 7ac5e0824679849c06ec7118c88ce5fb5d7afe03 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 15:15:17 +0800 Subject: [PATCH 12/22] test(perf): measure real history streams and live input overlap --- ...-06-frontend-performance-budgets.i18n.yaml | 4 +-- ...2026-09-06-frontend-performance-budgets.md | 22 ++++++------- ...6-09-06-frontend-performance-budgets.zh.md | 22 ++++++------- .../long-session-browser/README.i18n.yaml | 4 +-- benchmarks/long-session-browser/README.md | 4 +-- benchmarks/long-session-browser/README.zh.md | 4 +-- .../long-session.bench.ts | 20 ++++++++--- .../long-session-browser/synthetic-history.ts | 33 ++++++++++++++++--- 8 files changed, 73 insertions(+), 40 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml index 06aa949170..b0c65064a2 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md -2026-09-06-frontend-performance-budgets.md: 0ebe97db9532c4922d2e0e8f2bd41613b9e80b6e -2026-09-06-frontend-performance-budgets.zh.md: f58afd5aae763b145887204a63dd5b90d16566b4 +2026-09-06-frontend-performance-budgets.md: 92cdb27fa64efe304f501916f8f5bb205612e454 +2026-09-06-frontend-performance-budgets.zh.md: 38efa82251f3dcc3886051b434df4018c1665888 diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md index 0ebe97db95..92cdb27fa6 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md @@ -14,29 +14,29 @@ The existing serial benchmark inventory includes two frontend owners: [active re `build:bench` keeps the Node-only library and worker build. `test:bench` additionally builds the Web shell before running all cases; the required benchmark CI job provisions Chromium. Browser cases reuse the shipped-composition Web scaffold with private temporary roots and an atomically assigned loopback port. Only the nondeterministic model is replaced by synthetic replay. The scaffold Host runs under the existing Vitest source resolver; measured Client rendering runs built bundles in fresh Chromium processes. Browser wall times therefore include this test Host, transport, Playwright actionability, and rendering, and are not claims about a published Host process. -The browser input contains 240 closed turns, 40 tool results, and 20 code fences, plus mixed-language prose and reasoning. Nine older-page actions exhaust this input from its observed 25-turn initial window; the readiness probe follows mounted turn growth rather than duplicating the pagination algorithm. Each sample uses a fresh scaffold and browser. Setup, seeding, browser launch, initial shell load, and sidebar expansion are excluded from open timing. Open ends at transcript availability and an editable composer; page and navigation timings end at their target DOM state. Two animation frames include a rendering opportunity, not hardware presentation or a guarantee that every offscreen node painted. +The browser input contains 240 closed turns, 40 tool results, and 20 code fences, plus mixed-language prose and reasoning. Historical Assistant records carry matching compact streams built through the production accumulator with 12-character reasoning/text deltas and 8-character tool-argument deltas; empty streams would omit stored and transferred payload costs. Nine older-page actions exhaust this input from its observed 25-turn initial window; the readiness probe follows mounted turn growth rather than duplicating the pagination algorithm. Each sample uses a fresh scaffold and browser. Setup, seeding, browser launch, initial shell load, and sidebar expansion are excluded from open timing. Open ends at transcript availability and an editable composer; page and navigation timings end at their target DOM state. Two animation frames include a rendering opportunity, not hardware presentation or a guarantee that every offscreen node painted. -The continuation sends 120 text deltas at 8 ms replay pacing. It records click-to-first-visible-reply, trusted draft typing while the completion marker is absent, complete reply wall time through settled persistence, and Chromium main-thread task duration. The complete wall budget adds the fixed 992 ms scripted pacing to a scaled overhead allowance; input and completion have their own enforced budgets. Post-GC browser heap and DOM counts remain diagnostics because one endpoint does not prove a leak. +The continuation sends 120 text deltas at 8 ms replay pacing. It records click-to-first-visible-reply, trusted draft typing whose first actual input event observes the first reply but no completion marker, complete reply wall time through settled persistence and the new rendered turn-tail, and Chromium main-thread task duration. The complete wall budget adds the fixed 992 ms scripted pacing to a scaled overhead allowance; input and completion have their own enforced budgets. Post-GC browser heap and DOM counts remain diagnostics because one endpoint does not prove a leak. Reconnect uses three fresh compiled plain-Node children. Each creates a 100,000-delta reasoning prefix with distinct timestamps and two compact records before timing `ClientAssistantStream.replace()`. GC precedes the baseline and follows replacement while the result remains reachable; replacement time excludes both collections. The report consumes the result after collection and checks that the next dense live frame remains accepted. This measures reconstruction, not transport, rendering, or an entire reconnect workflow. ## Calibration -Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149.0.7827.55, at product revision `925e012340`, establish the baseline below. An isolated repeat follows a complete workflow smoke. Each browser sample reports raw endpoint values and every page; the paging verdict uses the median of the sample maxima. Reconnect reports all child measurements. Source reference constants round above observed values; the shared 2× time scale and 1.25× variance allowance produce CI limits. Memory uses only variance allowance. The existing scale comes from Node CI calibration, not a measured x64 browser comparison; browser-specific runner calibration remains an explicit gap. +Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149.0.7827.55, at product revision `925e012340`, establish the baseline below. An isolated repeat follows a complete workflow smoke. Each browser sample reports raw endpoint values and every page; the paging verdict uses the median of the sample maxima. Reconnect reports all child measurements. Source reference constants retain the original allowances rather than increasing them after the compact-payload correction; the corrected 302.25 ms paging median exceeds its 260 ms reference allowance but remains below its 650 ms CI limit; the shared 2× time scale and 1.25× variance allowance produce CI limits. Memory uses only variance allowance. The existing scale comes from Node CI calibration, not a measured x64 browser comparison; browser-specific runner calibration remains an explicit gap. | Endpoint | Measured median | Reference allowance | CI limit | |---|---:|---:|---:| -| Browser open | 166.77 ms | 200 ms | 500 ms | -| Slowest older page | 245.52 ms | 260 ms | 650 ms | -| First Trajectory | 133.45 ms | 160 ms | 400 ms | -| First reply | 1033.07 ms | 1100 ms | 2750 ms | -| Stream main-thread task | 1668.46 ms | 1800 ms | 4500 ms | -| Draft typing | 228.08 ms | 500 ms | 1250 ms | -| Complete response | 1699.26 ms | 1000 ms overhead + 992 ms pacing | 3492 ms | +| Browser open | 194.04 ms | 200 ms | 500 ms | +| Slowest older page | 302.25 ms | 260 ms | 650 ms | +| First Trajectory | 140.44 ms | 160 ms | 400 ms | +| First reply | 1093.12 ms | 1100 ms | 2750 ms | +| Stream main-thread task | 1712.99 ms | 1800 ms | 4500 ms | +| Draft typing | 126.71 ms | 500 ms | 1250 ms | +| Complete response | 1751.04 ms | 1000 ms overhead + 992 ms pacing | 3492 ms | | Reconnect replacement | 13.83 ms | 16 ms | 40 ms | | Reconnect retained heap | 23.03 MiB | 24 MiB | 30 MiB | -Draft typing spans 138.80–420.28 ms across the three isolated samples; its reference covers that observed spread instead of treating the median as a per-keystroke bound. No budget is an environment override. Temporary zero allowances exercise every rejection path; these negative controls prove enforcement, not an optimization or a historical regression. +Draft typing spans 101.58–415.26 ms across the three isolated samples; its reference covers that observed spread instead of treating the median as a per-keystroke bound. No budget is an environment override. Temporary zero allowances exercise every rejection path; these negative controls prove enforcement, not an optimization or a historical regression. A separate control waits for the final reply marker before typing and fails the actual-input overlap assertion. The compact synthetic JSONL is 3,262,577 bytes; all three corrected samples report an overlapping trusted input event and end after the 241st rendered turn-tail. ## Alternatives considered diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md index f58afd5aae..38efa82251 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md @@ -14,29 +14,29 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 `build:bench` 保留仅 Node 的 library 与 worker 构建。`test:bench` 额外构建 Web shell 后再运行所有用例；必需的基准 CI job 安装 Chromium。浏览器用例复用产品组合的 Web scaffold，使用私有临时目录和原子分配的回环端口。只有不确定的模型被合成重放替代。scaffold Host 通过现有 Vitest 源码解析器运行；被测 Client 渲染在全新 Chromium 进程中执行构建后的 bundle。因此浏览器壁钟时间包含测试 Host、传输、Playwright 可交互性等待及渲染，不代表发布版 Host 进程。 -浏览器输入包含 240 个已关闭轮次、40 个工具结果和 20 个代码块，以及混合语言正文和推理。从观察到的初始 25 轮窗口开始，九次更早分页操作读完该输入；就绪探针跟踪已挂载轮次增长，不复制分页算法。每个样本使用全新 scaffold 和浏览器。环境准备、数据播种、浏览器启动、初始 shell 加载及侧栏展开不计入打开时间。打开测量在对话可用且输入框可编辑时结束；分页与导航测量在目标 DOM 状态出现时结束。两次动画帧包含一次渲染机会，不代表硬件显示或保证每个屏幕外节点都已绘制。 +浏览器输入包含 240 个已关闭轮次、40 个工具结果和 20 个代码块，以及混合语言正文和推理。历史 Assistant 记录携带匹配的紧凑 stream，通过生产 accumulator 按 12 字符推理/文本 delta 和 8 字符工具参数 delta 构建；空 stream 会遗漏存储与传输负载成本。从观察到的初始 25 轮窗口开始，九次更早分页操作读完该输入；就绪探针跟踪已挂载轮次增长，不复制分页算法。每个样本使用全新 scaffold 和浏览器。环境准备、数据播种、浏览器启动、初始 shell 加载及侧栏展开不计入打开时间。打开测量在对话可用且输入框可编辑时结束；分页与导航测量在目标 DOM 状态出现时结束。两次动画帧包含一次渲染机会，不代表硬件显示或保证每个屏幕外节点都已绘制。 -续接以 8 ms 重放间隔发送 120 个文本 delta。它记录点击到首段可见回复的时间、完成标记尚未出现时的真实草稿键入、直到持久化结算的完整回复壁钟时间，以及 Chromium 主线程任务时间。完整壁钟预算在缩放后的额外开销额度上加固定的 992 ms 脚本节奏；输入和完成均有独立执行的预算。强制 GC 后的浏览器 heap 和 DOM 数量仍仅供诊断，因为单个终点不能证明泄漏。 +续接以 8 ms 重放间隔发送 120 个文本 delta。它记录点击到首段可见回复的时间、首个实际输入事件观察到首段回复且完成标记尚未出现时的真实草稿键入、直到持久化结算并渲染新 turn-tail 的完整回复壁钟时间，以及 Chromium 主线程任务时间。完整壁钟预算在缩放后的额外开销额度上加固定的 992 ms 脚本节奏；输入和完成均有独立执行的预算。强制 GC 后的浏览器 heap 和 DOM 数量仍仅供诊断，因为单个终点不能证明泄漏。 重连使用三个全新编译后的纯 Node 子进程。各进程在计时 `ClientAssistantStream.replace()` 前创建包含不同时间戳、两条紧凑记录和 100,000 个 delta 的推理前缀。在基线前执行 GC，并在结果仍可达时于替换后再次 GC；替换时间不含两次回收。报告在回收后消费结果，并检查下一个稠密序号的实时 frame 仍被接受。这测量重建，不测量传输、渲染或完整重连工作流。 ## 校准 -在 arm64 参考机器、Node 24.19、Chromium 149.0.7827.55 和产品版本 `925e012340` 上，三个样本的中位数建立下表基线。完整工作流 smoke 后执行一次隔离重复测量。每个浏览器样本报告原始终点数据和每一页；分页判定使用各样本最大值的中位数。重连报告全部子进程测量。源码参考常量向上取整覆盖观察值；共享的 2× 时间倍率和 1.25× 方差余量产生 CI 限制。内存仅使用方差余量。现有倍率来自 Node CI 校准，并非实测 x64 浏览器对比；浏览器专用 runner 校准仍是明确缺口。 +在 arm64 参考机器、Node 24.19、Chromium 149.0.7827.55 和产品版本 `925e012340` 上，三个样本的中位数建立下表基线。完整工作流 smoke 后执行一次隔离重复测量。每个浏览器样本报告原始终点数据和每一页；分页判定使用各样本最大值的中位数。重连报告全部子进程测量。源码参考常量保留原额度，不因紧凑负载修正而提高；修正后的分页中位数 302.25 ms 超过 260 ms 参考额度，但仍低于 650 ms CI 限制；共享的 2× 时间倍率和 1.25× 方差余量产生 CI 限制。内存仅使用方差余量。现有倍率来自 Node CI 校准，并非实测 x64 浏览器对比；浏览器专用 runner 校准仍是明确缺口。 | 终点 | 实测中位数 | 参考额度 | CI 限制 | |---|---:|---:|---:| -| 浏览器打开 | 166.77 ms | 200 ms | 500 ms | -| 最慢更早分页 | 245.52 ms | 260 ms | 650 ms | -| 首次 Trajectory | 133.45 ms | 160 ms | 400 ms | -| 首段回复 | 1033.07 ms | 1100 ms | 2750 ms | -| 流式主线程任务 | 1668.46 ms | 1800 ms | 4500 ms | -| 草稿键入 | 228.08 ms | 500 ms | 1250 ms | -| 完整回复 | 1699.26 ms | 1000 ms 额外开销 + 992 ms 节奏 | 3492 ms | +| 浏览器打开 | 194.04 ms | 200 ms | 500 ms | +| 最慢更早分页 | 302.25 ms | 260 ms | 650 ms | +| 首次 Trajectory | 140.44 ms | 160 ms | 400 ms | +| 首段回复 | 1093.12 ms | 1100 ms | 2750 ms | +| 流式主线程任务 | 1712.99 ms | 1800 ms | 4500 ms | +| 草稿键入 | 126.71 ms | 500 ms | 1250 ms | +| 完整回复 | 1751.04 ms | 1000 ms 额外开销 + 992 ms 节奏 | 3492 ms | | 重连替换 | 13.83 ms | 16 ms | 40 ms | | 重连保留 heap | 23.03 MiB | 24 MiB | 30 MiB | -三个隔离样本中的草稿键入时间为 138.80–420.28 ms；参考额度覆盖观察到的波动，而不把中位数作为单次按键上限。预算不能通过环境变量覆盖。临时零额度覆盖每条拒绝路径；这些负向对照证明预算执行，而非优化或历史回归。 +三个隔离样本中的草稿键入时间为 101.58–415.26 ms；参考额度覆盖观察到的波动，而不把中位数作为单次按键上限。预算不能通过环境变量覆盖。临时零额度覆盖每条拒绝路径；这些负向对照证明预算执行，而非优化或历史回归。另一项对照在键入前等待最终回复标记，实际输入重叠断言因此失败。紧凑合成 JSONL 为 3,262,577 字节；三个修正样本均报告重叠的真实输入事件，并在第 241 个 turn-tail 渲染后结束。 ## 考虑过的替代方案 diff --git a/benchmarks/long-session-browser/README.i18n.yaml b/benchmarks/long-session-browser/README.i18n.yaml index e95961a823..3d41099765 100644 --- a/benchmarks/long-session-browser/README.i18n.yaml +++ b/benchmarks/long-session-browser/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write benchmarks/long-session-browser/README.md -README.md: 53385706703736a76568b0b55141d40731068ed2 -README.zh.md: 5e709ec6bbd53a420fcf8ad4f4f414b4552549b6 +README.md: 421f2a904e0b40b55b4b5cdeb3a15c66a3e68df9 +README.zh.md: ee26bb28d5c080d202b0285403daff7ab28563b3 diff --git a/benchmarks/long-session-browser/README.md b/benchmarks/long-session-browser/README.md index 5338570670..421f2a904e 100644 --- a/benchmarks/long-session-browser/README.md +++ b/benchmarks/long-session-browser/README.md @@ -10,8 +10,8 @@ This reference describes the required Chromium workflow in [long-session.bench.t ## Measurements -Three fresh browser processes and scaffold worlds produce raw samples and median verdicts. Open and paging end after the expected transcript state and two animation frames; this includes a rendering opportunity, not a hardware presentation timestamp. Paging reports every page and gates the median of each sample’s slowest page. Stream reports first visible reply, trusted draft typing, complete reply wall time, and Chromium main-thread task duration. Heap after forced GC and DOM counts are diagnostics, not leak budgets. +Three fresh browser processes and scaffold worlds produce raw samples and median verdicts. Open and paging end after the expected transcript state and two animation frames; this includes a rendering opportunity, not a hardware presentation timestamp. Paging reports every page and gates the median of each sample’s slowest page. Stream reports first visible reply, trusted draft typing, complete reply wall time, and Chromium main-thread task duration. The actual first input event must observe an unfinished reply; completion waits for the new rendered turn-tail after Host settlement. Heap after forced GC and DOM counts are diagnostics, not leak budgets. -The fixture contains mixed-language prompts, prose, reasoning, 20 code fences, and 40 synthetic tool results. No model, tool, external network, recorded Session, or private Harness home supplies its content. Streaming uses 120 text deltas at 8 ms replay pacing through the real composer, agent loop, transport, and persistence. +The fixture contains mixed-language prompts, prose, reasoning, 20 code fences, and 40 synthetic tool results. Every historical Assistant includes a compact stream built by the production accumulator from matching reasoning, text, tool arguments, usage, and finish chunks. No model, tool, external network, recorded Session, or private Harness home supplies its content. Streaming uses 120 text deltas at 8 ms replay pacing through the real composer, agent loop, transport, and persistence. The [decision record](../../.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md) owns calibration, exclusions, and alternatives. The larger [manual diagnostic](../../apps/web/tests/complex-history.perf.ts) remains separate. diff --git a/benchmarks/long-session-browser/README.zh.md b/benchmarks/long-session-browser/README.zh.md index 5e709ec6bb..ee26bb28d5 100644 --- a/benchmarks/long-session-browser/README.zh.md +++ b/benchmarks/long-session-browser/README.zh.md @@ -10,8 +10,8 @@ ## 测量 -三个全新浏览器进程与 scaffold 环境产生原始样本及中位数判定。打开和分页在预期对话状态出现且经过两次动画帧后结束；这包含一次渲染机会，而非硬件显示时间戳。分页报告每一页，并对各样本最慢分页时间的中位数执行预算检查。流式报告首段可见回复、真实草稿键入、完整回复壁钟时间和 Chromium 主线程任务时间。强制 GC 后的 heap 与 DOM 数量仅供诊断，不作为泄漏预算。 +三个全新浏览器进程与 scaffold 环境产生原始样本及中位数判定。打开和分页在预期对话状态出现且经过两次动画帧后结束；这包含一次渲染机会，而非硬件显示时间戳。分页报告每一页，并对各样本最慢分页时间的中位数执行预算检查。流式报告首段可见回复、真实草稿键入、完整回复壁钟时间和 Chromium 主线程任务时间。实际首个输入事件必须观察到未完成的回复；完成测量在 Host 结算后等待新 turn-tail 渲染。强制 GC 后的 heap 与 DOM 数量仅供诊断，不作为泄漏预算。 -fixture（测试前置数据）包含混合语言提示、正文、推理、20 个代码块和 40 个合成工具结果。其内容不来自模型、工具、外部网络、录制 Session 或私有 Harness 主目录。流式回复以 8 ms 重放间隔发送 120 个文本 delta，经过真实输入框、agent loop（智能体循环）、传输与持久化。 +fixture（测试前置数据）包含混合语言提示、正文、推理、20 个代码块和 40 个合成工具结果。每条历史 Assistant 都含紧凑 stream，由生产 accumulator 从匹配的推理、文本、工具参数、usage 和 finish chunk 构建。其内容不来自模型、工具、外部网络、录制 Session 或私有 Harness 主目录。流式回复以 8 ms 重放间隔发送 120 个文本 delta，经过真实输入框、agent loop（智能体循环）、传输与持久化。 [决策记录](../../.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md)拥有校准、排除项与替代方案。更大规模的[手动诊断](../../apps/web/tests/complex-history.perf.ts)保持独立。 diff --git a/benchmarks/long-session-browser/long-session.bench.ts b/benchmarks/long-session-browser/long-session.bench.ts index 7a4120778b..0d07642d13 100644 --- a/benchmarks/long-session-browser/long-session.bench.ts +++ b/benchmarks/long-session-browser/long-session.bench.ts @@ -40,7 +40,7 @@ function median(values: number[]): number { it('opens, pages, navigates and streams into a 240-turn browser history', async () => { if (webSnapshotMode() !== 'replay') throw new Error('browser benchmarks require keyless replay mode') - const samples: { open: number; page: number; trajectory: number; first: number; streamTask: number; streamWall: number; input: number; heapMb: number; nodes: number }[] = [] + const samples: { open: number; page: number; trajectory: number; first: number; streamTask: number; streamWall: number; input: number; inputOverlapped: boolean; heapMb: number; nodes: number }[] = [] for (let sample = 0; sample < SAMPLES; sample++) { const failures: unknown[] = [] const root = await mkdtemp(join(tmpdir(), 'dsh-browser-benchmark-')) @@ -49,7 +49,9 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async await writeFile(replayOverride, JSON.stringify([{ kind: 'chunks', chunks: syntheticReply() }])) const scaffold = await launchWebScaffold({ replayFixture: join(root, 'override-only.jsonl'), replayOverride, paceMs: PACE_MS, replayContextWindow: 10000000 }) try { - await seedSession(scaffold, syntheticHistory(), SESSION_ID) + const history = syntheticHistory() + await seedSession(scaffold, history, SESSION_ID) + console.log(JSON.stringify({ benchmark: 'long-session-browser/fixture', bytes: Buffer.byteLength(history) })) const browser = await chromium.launch({ headless: true }) try { const page = await newEnglishPage(browser) @@ -100,16 +102,24 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async await page.getByText(FIRST, { exact: false }).last().waitFor() await painted(page) const first = performance.now() - started - expect(await page.getByText(DONE, { exact: false }).count()).toBe(0) - // Trusted keyboard input while the response is live, rather than a synthetic heartbeat. + await composer.evaluate((element, markers) => { + element.addEventListener('input', (event) => { + const transcript = document.querySelector('[data-conversation-scroll]')?.textContent ?? '' + element.setAttribute('data-benchmark-input-overlap', String(event.isTrusted && transcript.includes(markers.first) && !transcript.includes(markers.done))) + }, { once: true }) + }, { first: FIRST, done: DONE }) + // Observe the actual trusted input event, not state before asynchronous click/typing. const input = await measure(page, async () => { await composer.click() await page.keyboard.type('next synthetic question') await expect.poll(() => composer.textContent()).toBe('next synthetic question') }) + const inputOverlapped = await composer.getAttribute('data-benchmark-input-overlap') === 'true' + expect(inputOverlapped).toBe(true) await page.getByText(DONE, { exact: false }).last().waitFor() const settlement = await settled if (!settlement.ok) throw settlement.error + await page.waitForFunction(({ selector, expected }) => document.querySelectorAll(selector).length === expected, { selector: TAIL, expected: HISTORY_TURNS + 1 }) await painted(page) const streamWall = performance.now() - started const streamTask = await taskMs(cdp) - beforeTask @@ -117,7 +127,7 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async const metrics = (await cdp.send('Performance.getMetrics')).metrics const heap = metrics.find(metric => metric.name === 'JSHeapUsedSize') if (heap === undefined) throw new Error('Chromium heap metric missing') - samples.push({ open, page: Math.max(...pages), trajectory, first, streamTask, streamWall, input, heapMb: heap.value / 1048576, nodes: await page.locator('*').count() }) + samples.push({ open, page: Math.max(...pages), trajectory, first, streamTask, streamWall, input, inputOverlapped, heapMb: heap.value / 1048576, nodes: await page.locator('*').count() }) console.log(JSON.stringify({ benchmark: 'long-session-browser/sample', sample, initialTurns, pages, ...samples.at(-1) })) expect(consoleWatch.pageErrors).toEqual([]) expect(consoleWatch.warnings).toEqual([]) diff --git a/benchmarks/long-session-browser/synthetic-history.ts b/benchmarks/long-session-browser/synthetic-history.ts index 1d3b22aedb..f6f20a3ba8 100644 --- a/benchmarks/long-session-browser/synthetic-history.ts +++ b/benchmarks/long-session-browser/synthetic-history.ts @@ -1,6 +1,7 @@ /** Synthetic current-generation history and paced reply for browser measurements. */ import { createAssistantMessage, createUserMessage, createToolResultMessage, ToolCallId } from '@deepseek-ai/dsh-llm' import type { StreamChunk } from '@deepseek-ai/dsh-llm' +import { AssistantStreamAccumulator } from '@deepseek-ai/dsh-llm/assistant-stream' import { Session, SessionId, SESSION_FORMAT_VERSION } from '@deepseek-ai/dsh-session' import type {} from '@deepseek-ai/dsh-session-title' @@ -36,20 +37,42 @@ export function syntheticHistory(): string { const code = turn % 12 === 0 ? '\n\n```ts\n' + Array.from({ length: 60 }, (_, i) => 'const value' + String(i) + ' = ' + String(i)).join('\n') + '\n```' : '' + const reasoning = 'Compare the synthetic module and test. '.repeat(40) + const text = 'Synthetic answer ' + String(turn) + '. ' + 'Preserve ordering and validate the output. '.repeat(30) + code + const args = '{"path":"src/example.ts"}' + const stream = new AssistantStreamAccumulator() + let time = 1700000000000 + turn * 10000 + const push = (chunk: StreamChunk): void => { stream.push({ time: time++, chunk }) } + for (const [index, block] of [{ type: 'reasoning' as const, text: reasoning }, { type: 'text' as const, text }].entries()) { + push({ type: 'block-start', index, blockType: block.type }) + for (let offset = 0; offset < block.text.length; offset += 12) { + push({ type: block.type === 'reasoning' ? 'reasoning-delta' : 'text-delta', index, text: block.text.slice(offset, offset + 12) }) + } + push({ type: 'block-end', index, block }) + } + if (tool) { + push({ type: 'block-start', index: 2, blockType: 'tool-call' }) + for (let offset = 0; offset < args.length; offset += 8) { + push({ type: 'tool-call-delta', index: 2, id: callId, ...offset === 0 ? { name: 'synthetic_tool' } : {}, argumentsDelta: args.slice(offset, offset + 8) }) + } + push({ type: 'block-end', index: 2, block: { type: 'tool-call', id: callId, name: 'synthetic_tool', arguments: args } }) + } + push({ type: 'usage', usage: { inputTokens: 4000, outputTokens: 800 } }) + push({ type: 'finish', reason: { kind: tool ? 'tool-calls' : 'stop' } }) session.append('assistant/message', { - turn, step: 1, stream: [], + turn, step: 1, stream: [...stream.snapshot()], message: createAssistantMessage({ source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' }, content: [ - { type: 'reasoning', text: 'Compare the synthetic module and test. '.repeat(40) }, - { type: 'text', text: 'Synthetic answer ' + String(turn) + '. ' + 'Preserve ordering and validate the output. '.repeat(30) + code }, - ...tool ? [{ type: 'tool-call' as const, id: callId, name: 'synthetic_tool', arguments: '{"path":"src/example.ts"}' }] : [], + { type: 'reasoning', text: reasoning }, + { type: 'text', text }, + ...tool ? [{ type: 'tool-call' as const, id: callId, name: 'synthetic_tool', arguments: args }] : [], ], }), usage: { inputTokens: 4000, outputTokens: 800 }, }, { surfaceOp: 'append' }) if (tool) { - const call = session.append('tool/call', { turn, step: 1, callId, name: 'synthetic_tool', arguments: '{"path":"src/example.ts"}' }) + const call = session.append('tool/call', { turn, step: 1, callId, name: 'synthetic_tool', arguments: args }) session.append('tool/result', { turn, step: 1, message: createToolResultMessage({ callId, isError: false, content: [{ type: 'text', text: 'Synthetic tool output line.\n'.repeat(160) }], }) }, { surfaceOp: 'append', sourceEventSeqs: [call.seq] }) From 2927034ffeb647a6f733e77de10c31a2ef0bcfdd Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 15:31:30 +0800 Subject: [PATCH 13/22] test(perf): retain successful CI measurement output --- .../2026-09-06-frontend-performance-budgets.i18n.yaml | 4 ++-- .../testing/2026-09-06-frontend-performance-budgets.md | 2 +- .../testing/2026-09-06-frontend-performance-budgets.zh.md | 2 +- scripts/ci-workflow.spec.ts | 5 +++++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml index b0c65064a2..89f7d54c18 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md -2026-09-06-frontend-performance-budgets.md: 92cdb27fa64efe304f501916f8f5bb205612e454 -2026-09-06-frontend-performance-budgets.zh.md: 38efa82251f3dcc3886051b434df4018c1665888 +2026-09-06-frontend-performance-budgets.md: 6a5b77bfc7e5518599ad59e625e5aa7fab7546f9 +2026-09-06-frontend-performance-budgets.zh.md: 6c1571fc4a1d833b0b1a392eeb859e80a0938e97 diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md index 92cdb27fa6..6a5b77bfc7 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md @@ -12,7 +12,7 @@ A fast Node conversation fold does not prove that a browser paints a long conver The existing serial benchmark inventory includes two frontend owners: [active reconnect](../../../../benchmarks/active-stream-reconnect/README.md) and a [browser workflow](../../../../benchmarks/long-session-browser/README.md). The browser workflow combines cold open, older-page navigation, first Trajectory activation, return to Chat, and a paced response with trusted keyboard input into one sequential scenario. These are endpoints of one workflow, not independent cold scenarios. The settled conversation-fold benchmark remains unchanged. -`build:bench` keeps the Node-only library and worker build. `test:bench` additionally builds the Web shell before running all cases; the required benchmark CI job provisions Chromium. Browser cases reuse the shipped-composition Web scaffold with private temporary roots and an atomically assigned loopback port. Only the nondeterministic model is replaced by synthetic replay. The scaffold Host runs under the existing Vitest source resolver; measured Client rendering runs built bundles in fresh Chromium processes. Browser wall times therefore include this test Host, transport, Playwright actionability, and rendering, and are not claims about a published Host process. +`build:bench` keeps the Node-only library and worker build. `test:bench` additionally builds the Web shell before running all cases; the required benchmark CI job provisions Chromium and enables the existing verbose gate output so successful raw samples remain available for calibration. Browser cases reuse the shipped-composition Web scaffold with private temporary roots and an atomically assigned loopback port. Only the nondeterministic model is replaced by synthetic replay. The scaffold Host runs under the existing Vitest source resolver; measured Client rendering runs built bundles in fresh Chromium processes. Browser wall times therefore include this test Host, transport, Playwright actionability, and rendering, and are not claims about a published Host process. The browser input contains 240 closed turns, 40 tool results, and 20 code fences, plus mixed-language prose and reasoning. Historical Assistant records carry matching compact streams built through the production accumulator with 12-character reasoning/text deltas and 8-character tool-argument deltas; empty streams would omit stored and transferred payload costs. Nine older-page actions exhaust this input from its observed 25-turn initial window; the readiness probe follows mounted turn growth rather than duplicating the pagination algorithm. Each sample uses a fresh scaffold and browser. Setup, seeding, browser launch, initial shell load, and sidebar expansion are excluded from open timing. Open ends at transcript availability and an editable composer; page and navigation timings end at their target DOM state. Two animation frames include a rendering opportunity, not hardware presentation or a guarantee that every offscreen node painted. diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md index 38efa82251..6c1571fc4a 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md @@ -12,7 +12,7 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 现有串行基准清单包含两个前端所有者：[活跃重连](../../../../benchmarks/active-stream-reconnect/README.zh.md)和[浏览器工作流](../../../../benchmarks/long-session-browser/README.zh.md)。浏览器工作流在一个顺序场景中组合冷打开、更早分页导航、首次激活 Trajectory、返回 Chat，以及伴随真实键盘输入的有节奏回复。这些是同一工作流的测量终点，而不是相互独立的冷场景。已结算对话折叠基准保持不变。 -`build:bench` 保留仅 Node 的 library 与 worker 构建。`test:bench` 额外构建 Web shell 后再运行所有用例；必需的基准 CI job 安装 Chromium。浏览器用例复用产品组合的 Web scaffold，使用私有临时目录和原子分配的回环端口。只有不确定的模型被合成重放替代。scaffold Host 通过现有 Vitest 源码解析器运行；被测 Client 渲染在全新 Chromium 进程中执行构建后的 bundle。因此浏览器壁钟时间包含测试 Host、传输、Playwright 可交互性等待及渲染，不代表发布版 Host 进程。 +`build:bench` 保留仅 Node 的 library 与 worker 构建。`test:bench` 额外构建 Web shell 后再运行所有用例；必需的基准 CI job 安装 Chromium，并启用现有门禁详细输出，使成功用例的原始样本可用于校准。浏览器用例复用产品组合的 Web scaffold，使用私有临时目录和原子分配的回环端口。只有不确定的模型被合成重放替代。scaffold Host 通过现有 Vitest 源码解析器运行；被测 Client 渲染在全新 Chromium 进程中执行构建后的 bundle。因此浏览器壁钟时间包含测试 Host、传输、Playwright 可交互性等待及渲染，不代表发布版 Host 进程。 浏览器输入包含 240 个已关闭轮次、40 个工具结果和 20 个代码块，以及混合语言正文和推理。历史 Assistant 记录携带匹配的紧凑 stream，通过生产 accumulator 按 12 字符推理/文本 delta 和 8 字符工具参数 delta 构建；空 stream 会遗漏存储与传输负载成本。从观察到的初始 25 轮窗口开始，九次更早分页操作读完该输入；就绪探针跟踪已挂载轮次增长，不复制分页算法。每个样本使用全新 scaffold 和浏览器。环境准备、数据播种、浏览器启动、初始 shell 加载及侧栏展开不计入打开时间。打开测量在对话可用且输入框可编辑时结束；分页与导航测量在目标 DOM 状态出现时结束。两次动画帧包含一次渲染机会，不代表硬件显示或保证每个屏幕外节点都已绘制。 diff --git a/scripts/ci-workflow.spec.ts b/scripts/ci-workflow.spec.ts index 20c0b01907..be2eb8865c 100644 --- a/scripts/ci-workflow.spec.ts +++ b/scripts/ci-workflow.spec.ts @@ -211,6 +211,11 @@ describe('CI workflow', () => { expect(aggregate.needs).toContain('node-24-bench') expect(node24Bench.name).toBe('node 24 / benchmarks') expect(node24Bench.env).toBeUndefined() + expect(node24Bench.steps).toContainEqual({ + name: 'Run performance benchmarks', + env: { DSH_GATE_VERBOSE: '1' }, + run: 'pnpm run check:ci:bench', + }) expect(aggregate.needs).not.toContain('windows-coverage') expect(aggregate.needs).toContain('windows-native-tests') expect(aggregate.needs).not.toContain('windows-observational') From 936958587216231974bba57087df60129602db22 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 15:44:56 +0800 Subject: [PATCH 14/22] test(perf): bound browser observers to active response --- ...-06-frontend-performance-budgets.i18n.yaml | 4 ++-- ...2026-09-06-frontend-performance-budgets.md | 22 ++++++++++--------- ...6-09-06-frontend-performance-budgets.zh.md | 22 ++++++++++--------- .../long-session-browser/README.i18n.yaml | 4 ++-- benchmarks/long-session-browser/README.md | 2 +- benchmarks/long-session-browser/README.zh.md | 2 +- .../long-session.bench.ts | 9 ++++---- 7 files changed, 35 insertions(+), 30 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml index 89f7d54c18..ead0adef28 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md -2026-09-06-frontend-performance-budgets.md: 6a5b77bfc7e5518599ad59e625e5aa7fab7546f9 -2026-09-06-frontend-performance-budgets.zh.md: 6c1571fc4a1d833b0b1a392eeb859e80a0938e97 +2026-09-06-frontend-performance-budgets.md: 0df2e9c90b6192a7ededd445c24e35c5b9b6e7dc +2026-09-06-frontend-performance-budgets.zh.md: e1cc7314df90009cce7f994b8d5dc0ca0f9e5807 diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md index 6a5b77bfc7..0df2e9c90b 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md @@ -16,27 +16,27 @@ The existing serial benchmark inventory includes two frontend owners: [active re The browser input contains 240 closed turns, 40 tool results, and 20 code fences, plus mixed-language prose and reasoning. Historical Assistant records carry matching compact streams built through the production accumulator with 12-character reasoning/text deltas and 8-character tool-argument deltas; empty streams would omit stored and transferred payload costs. Nine older-page actions exhaust this input from its observed 25-turn initial window; the readiness probe follows mounted turn growth rather than duplicating the pagination algorithm. Each sample uses a fresh scaffold and browser. Setup, seeding, browser launch, initial shell load, and sidebar expansion are excluded from open timing. Open ends at transcript availability and an editable composer; page and navigation timings end at their target DOM state. Two animation frames include a rendering opportunity, not hardware presentation or a guarantee that every offscreen node painted. -The continuation sends 120 text deltas at 8 ms replay pacing. It records click-to-first-visible-reply, trusted draft typing whose first actual input event observes the first reply but no completion marker, complete reply wall time through settled persistence and the new rendered turn-tail, and Chromium main-thread task duration. The complete wall budget adds the fixed 992 ms scripted pacing to a scaled overhead allowance; input and completion have their own enforced budgets. Post-GC browser heap and DOM counts remain diagnostics because one endpoint does not prove a leak. +The continuation sends 120 text deltas at 8 ms replay pacing. Send lookup stays inside the composer seat; first/final marker lookups stay inside the latest Assistant step and retain visible-state waits. The synchronous input witness reads that same bounded reply. Whole-history text and accessibility queries add observer CPU and garbage collection to the measured interval, so reducing that observer work is benchmark repair, not product optimization. It records click-to-first-visible-reply, trusted draft typing whose first actual input event observes the first reply but no completion marker, complete reply wall time through settled persistence and the new rendered turn-tail, and Chromium main-thread task duration. The complete wall budget adds the fixed 992 ms scripted pacing to a scaled overhead allowance; input and completion have their own enforced budgets. Post-GC browser heap and DOM counts remain diagnostics because one endpoint does not prove a leak. Reconnect uses three fresh compiled plain-Node children. Each creates a 100,000-delta reasoning prefix with distinct timestamps and two compact records before timing `ClientAssistantStream.replace()`. GC precedes the baseline and follows replacement while the result remains reachable; replacement time excludes both collections. The report consumes the result after collection and checks that the next dense live frame remains accepted. This measures reconstruction, not transport, rendering, or an entire reconnect workflow. ## Calibration -Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149.0.7827.55, at product revision `925e012340`, establish the baseline below. An isolated repeat follows a complete workflow smoke. Each browser sample reports raw endpoint values and every page; the paging verdict uses the median of the sample maxima. Reconnect reports all child measurements. Source reference constants retain the original allowances rather than increasing them after the compact-payload correction; the corrected 302.25 ms paging median exceeds its 260 ms reference allowance but remains below its 650 ms CI limit; the shared 2× time scale and 1.25× variance allowance produce CI limits. Memory uses only variance allowance. The existing scale comes from Node CI calibration, not a measured x64 browser comparison; browser-specific runner calibration remains an explicit gap. +Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149.0.7827.55, at product revision `925e012340`, establish the baseline below. An isolated repeat follows a complete workflow smoke. Each browser sample reports raw endpoint values and every page; the paging verdict uses the median of the sample maxima. Reconnect reports all child measurements. Source reference constants retain the original allowances while CI calibration is pending; the bounded-observer 261.60 ms paging median exceeds its 260 ms reference allowance but remains below its 650 ms CI limit; the shared 2× time scale and 1.25× variance allowance produce CI limits. Memory uses only variance allowance. The existing scale comes from Node CI calibration, not a measured x64 browser comparison; browser-specific runner calibration remains an explicit gap. | Endpoint | Measured median | Reference allowance | CI limit | |---|---:|---:|---:| -| Browser open | 194.04 ms | 200 ms | 500 ms | -| Slowest older page | 302.25 ms | 260 ms | 650 ms | -| First Trajectory | 140.44 ms | 160 ms | 400 ms | -| First reply | 1093.12 ms | 1100 ms | 2750 ms | -| Stream main-thread task | 1712.99 ms | 1800 ms | 4500 ms | -| Draft typing | 126.71 ms | 500 ms | 1250 ms | -| Complete response | 1751.04 ms | 1000 ms overhead + 992 ms pacing | 3492 ms | +| Browser open | 184.62 ms | 200 ms | 500 ms | +| Slowest older page | 261.60 ms | 260 ms | 650 ms | +| First Trajectory | 136.46 ms | 160 ms | 400 ms | +| First reply | 373.72 ms | 1100 ms | 2750 ms | +| Stream main-thread task | 1053.87 ms | 1800 ms | 4500 ms | +| Draft typing | 487.35 ms | 500 ms | 1250 ms | +| Complete response | 1366.36 ms | 1000 ms overhead + 992 ms pacing | 3492 ms | | Reconnect replacement | 13.83 ms | 16 ms | 40 ms | | Reconnect retained heap | 23.03 MiB | 24 MiB | 30 MiB | -Draft typing spans 101.58–415.26 ms across the three isolated samples; its reference covers that observed spread instead of treating the median as a per-keystroke bound. No budget is an environment override. Temporary zero allowances exercise every rejection path; these negative controls prove enforcement, not an optimization or a historical regression. A separate control waits for the final reply marker before typing and fails the actual-input overlap assertion. The compact synthetic JSONL is 3,262,577 bytes; all three corrected samples report an overlapping trusted input event and end after the 241st rendered turn-tail. +Draft typing spans 124.97–504.96 ms across the three isolated samples; the reference remains 500 ms and the scaled CI limit covers that observed spread; the median is not a per-keystroke bound. No budget is an environment override. Temporary zero allowances exercise every rejection path; these negative controls prove enforcement, not an optimization or a historical regression. A separate control waits for the final reply marker before typing and fails the actual-input overlap assertion. The compact synthetic JSONL is 3,262,577 bytes; all three corrected samples report an overlapping trusted input event and end after the 241st rendered turn-tail. ## Alternatives considered @@ -46,6 +46,8 @@ Draft typing spans 101.58–415.26 ms across the three isolated samples; its ref **Coalesce active reconnect chunks.** Rejected as a benchmark shortcut: Client entries expose per-member ordering and timestamps to conversation definitions. The benchmark retains that production behavior; reducing retained entries requires a separate semantic design, not copied product algorithms or a synthetic approximation. +**Search the entire loaded history for every stream marker.** Rejected because Playwright injects text and accessibility scans into the same renderer whose CPU the benchmark measures. Scoping queries to the composer and latest Assistant preserves visible completion checks without making observer cost proportional to loaded history. + **Measure stream CPU alone.** Rejected because transport stalls and final-settlement delays can leave main-thread CPU low. The independent input, first-reply, and complete-wall budgets cover those waits. ## Consequences diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md index 6c1571fc4a..e1cc7314df 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md @@ -16,27 +16,27 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 浏览器输入包含 240 个已关闭轮次、40 个工具结果和 20 个代码块，以及混合语言正文和推理。历史 Assistant 记录携带匹配的紧凑 stream，通过生产 accumulator 按 12 字符推理/文本 delta 和 8 字符工具参数 delta 构建；空 stream 会遗漏存储与传输负载成本。从观察到的初始 25 轮窗口开始，九次更早分页操作读完该输入；就绪探针跟踪已挂载轮次增长，不复制分页算法。每个样本使用全新 scaffold 和浏览器。环境准备、数据播种、浏览器启动、初始 shell 加载及侧栏展开不计入打开时间。打开测量在对话可用且输入框可编辑时结束；分页与导航测量在目标 DOM 状态出现时结束。两次动画帧包含一次渲染机会，不代表硬件显示或保证每个屏幕外节点都已绘制。 -续接以 8 ms 重放间隔发送 120 个文本 delta。它记录点击到首段可见回复的时间、首个实际输入事件观察到首段回复且完成标记尚未出现时的真实草稿键入、直到持久化结算并渲染新 turn-tail 的完整回复壁钟时间，以及 Chromium 主线程任务时间。完整壁钟预算在缩放后的额外开销额度上加固定的 992 ms 脚本节奏；输入和完成均有独立执行的预算。强制 GC 后的浏览器 heap 和 DOM 数量仍仅供诊断，因为单个终点不能证明泄漏。 +续接以 8 ms 重放间隔发送 120 个文本 delta。发送控件查找限制在 composer seat；首段/最终标记查找限制在最新 Assistant step，并保留可见状态等待。同步输入证据读取同一个受限回复。全历史文本与无障碍查询会向测量区间加入观察器 CPU 和垃圾回收成本，因此减少此类观察工作属于基准修正，而非产品优化。它记录点击到首段可见回复的时间、首个实际输入事件观察到首段回复且完成标记尚未出现时的真实草稿键入、直到持久化结算并渲染新 turn-tail 的完整回复壁钟时间，以及 Chromium 主线程任务时间。完整壁钟预算在缩放后的额外开销额度上加固定的 992 ms 脚本节奏；输入和完成均有独立执行的预算。强制 GC 后的浏览器 heap 和 DOM 数量仍仅供诊断，因为单个终点不能证明泄漏。 重连使用三个全新编译后的纯 Node 子进程。各进程在计时 `ClientAssistantStream.replace()` 前创建包含不同时间戳、两条紧凑记录和 100,000 个 delta 的推理前缀。在基线前执行 GC，并在结果仍可达时于替换后再次 GC；替换时间不含两次回收。报告在回收后消费结果，并检查下一个稠密序号的实时 frame 仍被接受。这测量重建，不测量传输、渲染或完整重连工作流。 ## 校准 -在 arm64 参考机器、Node 24.19、Chromium 149.0.7827.55 和产品版本 `925e012340` 上，三个样本的中位数建立下表基线。完整工作流 smoke 后执行一次隔离重复测量。每个浏览器样本报告原始终点数据和每一页；分页判定使用各样本最大值的中位数。重连报告全部子进程测量。源码参考常量保留原额度，不因紧凑负载修正而提高；修正后的分页中位数 302.25 ms 超过 260 ms 参考额度，但仍低于 650 ms CI 限制；共享的 2× 时间倍率和 1.25× 方差余量产生 CI 限制。内存仅使用方差余量。现有倍率来自 Node CI 校准，并非实测 x64 浏览器对比；浏览器专用 runner 校准仍是明确缺口。 +在 arm64 参考机器、Node 24.19、Chromium 149.0.7827.55 和产品版本 `925e012340` 上，三个样本的中位数建立下表基线。完整工作流 smoke 后执行一次隔离重复测量。每个浏览器样本报告原始终点数据和每一页；分页判定使用各样本最大值的中位数。重连报告全部子进程测量。源码参考常量在 CI 校准待完成期间保留原额度；受限观察器的分页中位数 261.60 ms 超过 260 ms 参考额度，但仍低于 650 ms CI 限制；共享的 2× 时间倍率和 1.25× 方差余量产生 CI 限制。内存仅使用方差余量。现有倍率来自 Node CI 校准，并非实测 x64 浏览器对比；浏览器专用 runner 校准仍是明确缺口。 | 终点 | 实测中位数 | 参考额度 | CI 限制 | |---|---:|---:|---:| -| 浏览器打开 | 194.04 ms | 200 ms | 500 ms | -| 最慢更早分页 | 302.25 ms | 260 ms | 650 ms | -| 首次 Trajectory | 140.44 ms | 160 ms | 400 ms | -| 首段回复 | 1093.12 ms | 1100 ms | 2750 ms | -| 流式主线程任务 | 1712.99 ms | 1800 ms | 4500 ms | -| 草稿键入 | 126.71 ms | 500 ms | 1250 ms | -| 完整回复 | 1751.04 ms | 1000 ms 额外开销 + 992 ms 节奏 | 3492 ms | +| 浏览器打开 | 184.62 ms | 200 ms | 500 ms | +| 最慢更早分页 | 261.60 ms | 260 ms | 650 ms | +| 首次 Trajectory | 136.46 ms | 160 ms | 400 ms | +| 首段回复 | 373.72 ms | 1100 ms | 2750 ms | +| 流式主线程任务 | 1053.87 ms | 1800 ms | 4500 ms | +| 草稿键入 | 487.35 ms | 500 ms | 1250 ms | +| 完整回复 | 1366.36 ms | 1000 ms 额外开销 + 992 ms 节奏 | 3492 ms | | 重连替换 | 13.83 ms | 16 ms | 40 ms | | 重连保留 heap | 23.03 MiB | 24 MiB | 30 MiB | -三个隔离样本中的草稿键入时间为 101.58–415.26 ms；参考额度覆盖观察到的波动，而不把中位数作为单次按键上限。预算不能通过环境变量覆盖。临时零额度覆盖每条拒绝路径；这些负向对照证明预算执行，而非优化或历史回归。另一项对照在键入前等待最终回复标记，实际输入重叠断言因此失败。紧凑合成 JSONL 为 3,262,577 字节；三个修正样本均报告重叠的真实输入事件，并在第 241 个 turn-tail 渲染后结束。 +三个隔离样本中的草稿键入时间为 124.97–504.96 ms；参考额度保持 500 ms，缩放后的 CI 限制覆盖观察到的波动；中位数不是单次按键上限。预算不能通过环境变量覆盖。临时零额度覆盖每条拒绝路径；这些负向对照证明预算执行，而非优化或历史回归。另一项对照在键入前等待最终回复标记，实际输入重叠断言因此失败。紧凑合成 JSONL 为 3,262,577 字节；三个修正样本均报告重叠的真实输入事件，并在第 241 个 turn-tail 渲染后结束。 ## 考虑过的替代方案 @@ -46,6 +46,8 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 **合并活跃重连 chunk。** 不能作为基准捷径：Client 条目向对话定义公开每个成员的顺序和时间戳。基准保留该生产行为；减少保留条目需要独立的语义设计，而非复制产品算法或使用合成近似。 +**每次流式标记查找都搜索全部已加载历史。** 拒绝，因为 Playwright 把文本与无障碍扫描注入同一个被测 CPU 的渲染进程。将查询限定在输入框和最新 Assistant 中，可保留可见完成检查，同时避免观察器成本随已加载历史增长。 + **只测量流式 CPU。** 拒绝，因为传输停顿和最终结算延迟可能不增加主线程 CPU。独立的输入、首段回复和完整壁钟预算覆盖这些等待。 ## 影响 diff --git a/benchmarks/long-session-browser/README.i18n.yaml b/benchmarks/long-session-browser/README.i18n.yaml index 3d41099765..f51684c58f 100644 --- a/benchmarks/long-session-browser/README.i18n.yaml +++ b/benchmarks/long-session-browser/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write benchmarks/long-session-browser/README.md -README.md: 421f2a904e0b40b55b4b5cdeb3a15c66a3e68df9 -README.zh.md: ee26bb28d5c080d202b0285403daff7ab28563b3 +README.md: a926ef0ff7c48caf45ed77e760a4d764f778c097 +README.zh.md: 6b35fd570f1ba53f114f2924607effc84dacb4a5 diff --git a/benchmarks/long-session-browser/README.md b/benchmarks/long-session-browser/README.md index 421f2a904e..a926ef0ff7 100644 --- a/benchmarks/long-session-browser/README.md +++ b/benchmarks/long-session-browser/README.md @@ -10,7 +10,7 @@ This reference describes the required Chromium workflow in [long-session.bench.t ## Measurements -Three fresh browser processes and scaffold worlds produce raw samples and median verdicts. Open and paging end after the expected transcript state and two animation frames; this includes a rendering opportunity, not a hardware presentation timestamp. Paging reports every page and gates the median of each sample’s slowest page. Stream reports first visible reply, trusted draft typing, complete reply wall time, and Chromium main-thread task duration. The actual first input event must observe an unfinished reply; completion waits for the new rendered turn-tail after Host settlement. Heap after forced GC and DOM counts are diagnostics, not leak budgets. +Three fresh browser processes and scaffold worlds produce raw samples and median verdicts. Open and paging end after the expected transcript state and two animation frames; this includes a rendering opportunity, not a hardware presentation timestamp. Paging reports every page and gates the median of each sample’s slowest page. Stream reports first visible reply, trusted draft typing, complete reply wall time, and Chromium main-thread task duration. Send lookup is scoped to the composer seat; reply-marker lookups and the input-event text witness read only the latest Assistant step, avoiding repeated whole-history text and accessibility scans. The actual first input event must observe an unfinished reply; completion waits for the new rendered turn-tail after Host settlement. Heap after forced GC and DOM counts are diagnostics, not leak budgets. The fixture contains mixed-language prompts, prose, reasoning, 20 code fences, and 40 synthetic tool results. Every historical Assistant includes a compact stream built by the production accumulator from matching reasoning, text, tool arguments, usage, and finish chunks. No model, tool, external network, recorded Session, or private Harness home supplies its content. Streaming uses 120 text deltas at 8 ms replay pacing through the real composer, agent loop, transport, and persistence. diff --git a/benchmarks/long-session-browser/README.zh.md b/benchmarks/long-session-browser/README.zh.md index ee26bb28d5..6b35fd570f 100644 --- a/benchmarks/long-session-browser/README.zh.md +++ b/benchmarks/long-session-browser/README.zh.md @@ -10,7 +10,7 @@ ## 测量 -三个全新浏览器进程与 scaffold 环境产生原始样本及中位数判定。打开和分页在预期对话状态出现且经过两次动画帧后结束；这包含一次渲染机会，而非硬件显示时间戳。分页报告每一页，并对各样本最慢分页时间的中位数执行预算检查。流式报告首段可见回复、真实草稿键入、完整回复壁钟时间和 Chromium 主线程任务时间。实际首个输入事件必须观察到未完成的回复；完成测量在 Host 结算后等待新 turn-tail 渲染。强制 GC 后的 heap 与 DOM 数量仅供诊断，不作为泄漏预算。 +三个全新浏览器进程与 scaffold 环境产生原始样本及中位数判定。打开和分页在预期对话状态出现且经过两次动画帧后结束；这包含一次渲染机会，而非硬件显示时间戳。分页报告每一页，并对各样本最慢分页时间的中位数执行预算检查。流式报告首段可见回复、真实草稿键入、完整回复壁钟时间和 Chromium 主线程任务时间。发送控件查找限定在 composer seat；回复标记查找与输入事件文本证据仅读取最新 Assistant step，避免重复扫描全部历史文本与无障碍属性。实际首个输入事件必须观察到未完成的回复；完成测量在 Host 结算后等待新 turn-tail 渲染。强制 GC 后的 heap 与 DOM 数量仅供诊断，不作为泄漏预算。 fixture（测试前置数据）包含混合语言提示、正文、推理、20 个代码块和 40 个合成工具结果。每条历史 Assistant 都含紧凑 stream，由生产 accumulator 从匹配的推理、文本、工具参数、usage 和 finish chunk 构建。其内容不来自模型、工具、外部网络、录制 Session 或私有 Harness 主目录。流式回复以 8 ms 重放间隔发送 120 个文本 delta，经过真实输入框、agent loop（智能体循环）、传输与持久化。 diff --git a/benchmarks/long-session-browser/long-session.bench.ts b/benchmarks/long-session-browser/long-session.bench.ts index 0d07642d13..3302e73fa4 100644 --- a/benchmarks/long-session-browser/long-session.bench.ts +++ b/benchmarks/long-session-browser/long-session.bench.ts @@ -98,13 +98,14 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async (error: unknown) => ({ ok: false as const, error }), ) const started = performance.now() - await page.getByRole('button', { name: 'Send message', exact: true }).click() - await page.getByText(FIRST, { exact: false }).last().waitFor() + await page.locator('[data-composer-seat]').getByRole('button', { name: 'Send message', exact: true }).click() + const reply = page.locator('[data-chat-flow-kind="assistant-step"]').last() + await reply.getByText(FIRST, { exact: false }).last().waitFor() await painted(page) const first = performance.now() - started await composer.evaluate((element, markers) => { element.addEventListener('input', (event) => { - const transcript = document.querySelector('[data-conversation-scroll]')?.textContent ?? '' + const transcript = Array.from(document.querySelectorAll('[data-chat-flow-kind="assistant-step"]')).at(-1)?.textContent ?? '' element.setAttribute('data-benchmark-input-overlap', String(event.isTrusted && transcript.includes(markers.first) && !transcript.includes(markers.done))) }, { once: true }) }, { first: FIRST, done: DONE }) @@ -116,7 +117,7 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async }) const inputOverlapped = await composer.getAttribute('data-benchmark-input-overlap') === 'true' expect(inputOverlapped).toBe(true) - await page.getByText(DONE, { exact: false }).last().waitFor() + await reply.getByText(DONE, { exact: false }).last().waitFor() const settlement = await settled if (!settlement.ok) throw settlement.error await page.waitForFunction(({ selector, expected }) => document.querySelectorAll(selector).length === expected, { selector: TAIL, expected: HISTORY_TURNS + 1 }) From 82cd35467ab55a89d02f1d59f4f5d5073242b58b Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 16:06:56 +0800 Subject: [PATCH 15/22] docs(perf): record first frontend CI calibration --- ...-06-frontend-performance-budgets.i18n.yaml | 4 ++-- ...2026-09-06-frontend-performance-budgets.md | 20 ++++++++++++++++++- ...6-09-06-frontend-performance-budgets.zh.md | 20 ++++++++++++++++++- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml index ead0adef28..1541c2b226 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md -2026-09-06-frontend-performance-budgets.md: 0df2e9c90b6192a7ededd445c24e35c5b9b6e7dc -2026-09-06-frontend-performance-budgets.zh.md: e1cc7314df90009cce7f994b8d5dc0ca0f9e5807 +2026-09-06-frontend-performance-budgets.md: 7136ff614dcbe47511acff9619e4529d30ae0d87 +2026-09-06-frontend-performance-budgets.zh.md: 60b4071ce7f6712931d7bdc7b17f7bcbe78dca9d diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md index 0df2e9c90b..7136ff614d 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md @@ -22,7 +22,7 @@ Reconnect uses three fresh compiled plain-Node children. Each creates a 100,000- ## Calibration -Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149.0.7827.55, at product revision `925e012340`, establish the baseline below. An isolated repeat follows a complete workflow smoke. Each browser sample reports raw endpoint values and every page; the paging verdict uses the median of the sample maxima. Reconnect reports all child measurements. Source reference constants retain the original allowances while CI calibration is pending; the bounded-observer 261.60 ms paging median exceeds its 260 ms reference allowance but remains below its 650 ms CI limit; the shared 2× time scale and 1.25× variance allowance produce CI limits. Memory uses only variance allowance. The existing scale comes from Node CI calibration, not a measured x64 browser comparison; browser-specific runner calibration remains an explicit gap. +Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149.0.7827.55, at product revision `925e012340`, establish the baseline below. An isolated repeat follows a complete workflow smoke. Each browser sample reports raw endpoint values and every page; the paging verdict uses the median of the sample maxima. Reconnect reports all child measurements. Source reference constants retain the original allowances while CI calibration is pending; the bounded-observer 261.60 ms paging median exceeds its 260 ms reference allowance but remains below its 650 ms CI limit; the shared 2× time scale and 1.25× variance allowance produce CI limits. Memory uses only variance allowance. The shared scale originates in Node CI calibration. The first actual x64 browser run below passes the fixed budgets; a second independent CI run remains pending, so repeated-run browser calibration is not complete. | Endpoint | Measured median | Reference allowance | CI limit | |---|---:|---:|---:| @@ -38,6 +38,24 @@ Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149 Draft typing spans 124.97–504.96 ms across the three isolated samples; the reference remains 500 ms and the scaled CI limit covers that observed spread; the median is not a per-keystroke bound. No budget is an environment override. Temporary zero allowances exercise every rejection path; these negative controls prove enforcement, not an optimization or a historical regression. A separate control waits for the final reply marker before typing and fails the actual-input overlap assertion. The compact synthetic JSONL is 3,262,577 bytes; all three corrected samples report an overlapping trusted input event and end after the 241st rendered turn-tail. +### First actual CI run + +[Run 34020120425, benchmark job 101451135853](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34020120425/job/101451135853) passes the complete benchmark inventory at `6d1ba089e5052680961825c08aa4de19b4fe137a`. The runner is `VM-7-113-ubuntu-ci-19` in `dsh-selfhosted-ci`, using x64 Node 24.19.0 and Chromium 149.0.7827.55. The following medians use three fresh samples per scenario and leave the local reference table and source budgets unchanged. + +| Endpoint | First CI median | +|---|---:| +| Browser open | 303.066 ms | +| Slowest older page | 432.979 ms | +| First Trajectory | 298.608 ms | +| First reply | 740.265 ms | +| Stream main-thread task | 1614.594 ms | +| Draft typing | 932.746 ms | +| Complete response | 1677.882 ms | +| Reconnect replacement | 29.232 ms | +| Reconnect retained heap | 23.028 MiB | + +All three browser samples report `inputOverlapped: true` and finish after the 241st rendered turn-tail. Post-GC browser heap is approximately 52.94 MiB with 17,064 DOM elements; both remain diagnostic endpoints. This run supports the existing budgets on this runner, not a universal 2× browser speed ratio. The required second independent CI run is pending; no budget is relaxed and no product optimization is claimed. + ## Alternatives considered **Use the Node fold as paint evidence.** Rejected because it never performs DOM mutation, layout, or browser scheduling. The focused reconnect case likewise makes no GUI speed claim. diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md index e1cc7314df..60b4071ce7 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md @@ -22,7 +22,7 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 ## 校准 -在 arm64 参考机器、Node 24.19、Chromium 149.0.7827.55 和产品版本 `925e012340` 上，三个样本的中位数建立下表基线。完整工作流 smoke 后执行一次隔离重复测量。每个浏览器样本报告原始终点数据和每一页；分页判定使用各样本最大值的中位数。重连报告全部子进程测量。源码参考常量在 CI 校准待完成期间保留原额度；受限观察器的分页中位数 261.60 ms 超过 260 ms 参考额度，但仍低于 650 ms CI 限制；共享的 2× 时间倍率和 1.25× 方差余量产生 CI 限制。内存仅使用方差余量。现有倍率来自 Node CI 校准，并非实测 x64 浏览器对比；浏览器专用 runner 校准仍是明确缺口。 +在 arm64 参考机器、Node 24.19、Chromium 149.0.7827.55 和产品版本 `925e012340` 上，三个样本的中位数建立下表基线。完整工作流 smoke 后执行一次隔离重复测量。每个浏览器样本报告原始终点数据和每一页；分页判定使用各样本最大值的中位数。重连报告全部子进程测量。源码参考常量在 CI 校准待完成期间保留原额度；受限观察器的分页中位数 261.60 ms 超过 260 ms 参考额度，但仍低于 650 ms CI 限制；共享的 2× 时间倍率和 1.25× 方差余量产生 CI 限制。内存仅使用方差余量。共享倍率源自 Node CI 校准。下述首次实际 x64 浏览器运行通过固定预算；第二次独立 CI 运行仍待完成，因此浏览器重复运行校准尚未完成。 | 终点 | 实测中位数 | 参考额度 | CI 限制 | |---|---:|---:|---:| @@ -38,6 +38,24 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 三个隔离样本中的草稿键入时间为 124.97–504.96 ms；参考额度保持 500 ms，缩放后的 CI 限制覆盖观察到的波动；中位数不是单次按键上限。预算不能通过环境变量覆盖。临时零额度覆盖每条拒绝路径；这些负向对照证明预算执行，而非优化或历史回归。另一项对照在键入前等待最终回复标记，实际输入重叠断言因此失败。紧凑合成 JSONL 为 3,262,577 字节；三个修正样本均报告重叠的真实输入事件，并在第 241 个 turn-tail 渲染后结束。 +### 首次实际 CI 运行 + +[运行 34020120425，基准 job 101451135853](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34020120425/job/101451135853) 在 `6d1ba089e5052680961825c08aa4de19b4fe137a` 上通过完整基准清单。runner 为 `dsh-selfhosted-ci` 中的 `VM-7-113-ubuntu-ci-19`，使用 x64 Node 24.19.0 和 Chromium 149.0.7827.55。下列中位数来自每个场景的三个全新样本，本地参考表和源码预算保持不变。 + +| 终点 | 首次 CI 中位数 | +|---|---:| +| 浏览器打开 | 303.066 ms | +| 最慢更早分页 | 432.979 ms | +| 首次 Trajectory | 298.608 ms | +| 首段回复 | 740.265 ms | +| 流式主线程任务 | 1614.594 ms | +| 草稿键入 | 932.746 ms | +| 完整回复 | 1677.882 ms | +| 重连替换 | 29.232 ms | +| 重连保留 heap | 23.028 MiB | + +三个浏览器样本均报告 `inputOverlapped: true`，并在第 241 个 turn-tail 渲染后结束。强制 GC 后浏览器 heap 约为 52.94 MiB，DOM 元素为 17,064 个；两者仍为诊断终点。该运行支持此 runner 上的现有预算，而不证明普遍适用的 2× 浏览器速度比。必需的第二次独立 CI 运行仍待完成；没有放宽预算，也不声称产品优化。 + ## 考虑过的替代方案 **用 Node 折叠作为绘制证据。** 拒绝，因为它不执行 DOM 修改、布局或浏览器调度。聚焦重连用例同样不声称 GUI 提速。 From 00e452aa03d7ee6e5e2351e0cfd614f50eefdca4 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 16:16:00 +0800 Subject: [PATCH 16/22] docs(perf): confirm repeated frontend CI calibration --- ...-06-frontend-performance-budgets.i18n.yaml | 4 +-- ...2026-09-06-frontend-performance-budgets.md | 30 +++++++++---------- ...6-09-06-frontend-performance-budgets.zh.md | 30 +++++++++---------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml index 1541c2b226..528630364f 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md -2026-09-06-frontend-performance-budgets.md: 7136ff614dcbe47511acff9619e4529d30ae0d87 -2026-09-06-frontend-performance-budgets.zh.md: 60b4071ce7f6712931d7bdc7b17f7bcbe78dca9d +2026-09-06-frontend-performance-budgets.md: 9cbe5980336ad4b0f2a3fb2f7cbe70757885f82e +2026-09-06-frontend-performance-budgets.zh.md: 6df0d00285d60ce1ea0ec9d7284e7a73924459ab diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md index 7136ff614d..9cbe598033 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md @@ -22,7 +22,7 @@ Reconnect uses three fresh compiled plain-Node children. Each creates a 100,000- ## Calibration -Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149.0.7827.55, at product revision `925e012340`, establish the baseline below. An isolated repeat follows a complete workflow smoke. Each browser sample reports raw endpoint values and every page; the paging verdict uses the median of the sample maxima. Reconnect reports all child measurements. Source reference constants retain the original allowances while CI calibration is pending; the bounded-observer 261.60 ms paging median exceeds its 260 ms reference allowance but remains below its 650 ms CI limit; the shared 2× time scale and 1.25× variance allowance produce CI limits. Memory uses only variance allowance. The shared scale originates in Node CI calibration. The first actual x64 browser run below passes the fixed budgets; a second independent CI run remains pending, so repeated-run browser calibration is not complete. +Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149.0.7827.55, at product revision `925e012340`, establish the baseline below. An isolated repeat follows a complete workflow smoke. Each browser sample reports raw endpoint values and every page; the paging verdict uses the median of the sample maxima. Reconnect reports all child measurements. Source reference constants retain the original allowances after two passing CI runs; the bounded-observer 261.60 ms paging median exceeds its 260 ms reference allowance but remains below its 650 ms CI limit; the shared 2× time scale and 1.25× variance allowance produce CI limits. Memory uses only variance allowance. The shared scale originates in Node CI calibration. Both actual x64 browser runs below pass the fixed budgets on unchanged benchmark code; this supplies repeated-run evidence for these runners, not a universal browser speed ratio. | Endpoint | Measured median | Reference allowance | CI limit | |---|---:|---:|---:| @@ -38,23 +38,23 @@ Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149 Draft typing spans 124.97–504.96 ms across the three isolated samples; the reference remains 500 ms and the scaled CI limit covers that observed spread; the median is not a per-keystroke bound. No budget is an environment override. Temporary zero allowances exercise every rejection path; these negative controls prove enforcement, not an optimization or a historical regression. A separate control waits for the final reply marker before typing and fails the actual-input overlap assertion. The compact synthetic JSONL is 3,262,577 bytes; all three corrected samples report an overlapping trusted input event and end after the 241st rendered turn-tail. -### First actual CI run +### Actual CI runs -[Run 34020120425, benchmark job 101451135853](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34020120425/job/101451135853) passes the complete benchmark inventory at `6d1ba089e5052680961825c08aa4de19b4fe137a`. The runner is `VM-7-113-ubuntu-ci-19` in `dsh-selfhosted-ci`, using x64 Node 24.19.0 and Chromium 149.0.7827.55. The following medians use three fresh samples per scenario and leave the local reference table and source budgets unchanged. +[Run 34020120425, benchmark job 101451135853](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34020120425/job/101451135853) passes the complete benchmark inventory at `6d1ba089e5052680961825c08aa4de19b4fe137a`. The runner is `VM-7-113-ubuntu-ci-19` in `dsh-selfhosted-ci`, using x64 Node 24.19.0 and Chromium 149.0.7827.55. [Attempt 2, benchmark job 101453296071](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34020120425/job/101453296071) also passes the complete inventory at the same commit, on `VM-7-113-ubuntu-ci-25` with the same Node and Chromium versions. The following medians use three fresh samples per scenario in each run and leave the local reference table and source budgets unchanged. -| Endpoint | First CI median | -|---|---:| -| Browser open | 303.066 ms | -| Slowest older page | 432.979 ms | -| First Trajectory | 298.608 ms | -| First reply | 740.265 ms | -| Stream main-thread task | 1614.594 ms | -| Draft typing | 932.746 ms | -| Complete response | 1677.882 ms | -| Reconnect replacement | 29.232 ms | -| Reconnect retained heap | 23.028 MiB | +| Endpoint | First CI median | Second CI median | +|---|---:|---:| +| Browser open | 303.066 ms | 284.726 ms | +| Slowest older page | 432.979 ms | 413.798 ms | +| First Trajectory | 298.608 ms | 267.372 ms | +| First reply | 740.265 ms | 658.906 ms | +| Stream main-thread task | 1614.594 ms | 1035.385 ms | +| Draft typing | 932.746 ms | 142.148 ms | +| Complete response | 1677.882 ms | 1641.702 ms | +| Reconnect replacement | 29.232 ms | 31.674 ms | +| Reconnect retained heap | 23.028 MiB | 23.028 MiB | -All three browser samples report `inputOverlapped: true` and finish after the 241st rendered turn-tail. Post-GC browser heap is approximately 52.94 MiB with 17,064 DOM elements; both remain diagnostic endpoints. This run supports the existing budgets on this runner, not a universal 2× browser speed ratio. The required second independent CI run is pending; no budget is relaxed and no product optimization is claimed. +All six browser samples report `inputOverlapped: true` and finish after the 241st rendered turn-tail. Post-GC browser heap is approximately 52.94 MiB in the first run and 53.00 MiB in the second, with 17,064 DOM elements in both; these remain diagnostic endpoints. Both runs support the existing budgets on these runners, not a universal 2× browser speed ratio. Draft-typing medians vary from 932.746 ms to 142.148 ms because the endpoint measures the entire typed draft, including scheduling and Playwright actionability, rather than a per-key latency guarantee. No budget is relaxed and no product optimization is claimed. ## Alternatives considered diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md index 60b4071ce7..6df0d00285 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md @@ -22,7 +22,7 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 ## 校准 -在 arm64 参考机器、Node 24.19、Chromium 149.0.7827.55 和产品版本 `925e012340` 上，三个样本的中位数建立下表基线。完整工作流 smoke 后执行一次隔离重复测量。每个浏览器样本报告原始终点数据和每一页；分页判定使用各样本最大值的中位数。重连报告全部子进程测量。源码参考常量在 CI 校准待完成期间保留原额度；受限观察器的分页中位数 261.60 ms 超过 260 ms 参考额度，但仍低于 650 ms CI 限制；共享的 2× 时间倍率和 1.25× 方差余量产生 CI 限制。内存仅使用方差余量。共享倍率源自 Node CI 校准。下述首次实际 x64 浏览器运行通过固定预算；第二次独立 CI 运行仍待完成，因此浏览器重复运行校准尚未完成。 +在 arm64 参考机器、Node 24.19、Chromium 149.0.7827.55 和产品版本 `925e012340` 上，三个样本的中位数建立下表基线。完整工作流 smoke 后执行一次隔离重复测量。每个浏览器样本报告原始终点数据和每一页；分页判定使用各样本最大值的中位数。重连报告全部子进程测量。源码参考常量在两次 CI 运行通过后保留原额度；受限观察器的分页中位数 261.60 ms 超过 260 ms 参考额度，但仍低于 650 ms CI 限制；共享的 2× 时间倍率和 1.25× 方差余量产生 CI 限制。内存仅使用方差余量。共享倍率源自 Node CI 校准。下述两次实际 x64 浏览器运行在基准代码不变的情况下均通过固定预算；这提供这些 runner 的重复运行证据，而非普遍适用的浏览器速度比。 | 终点 | 实测中位数 | 参考额度 | CI 限制 | |---|---:|---:|---:| @@ -38,23 +38,23 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 三个隔离样本中的草稿键入时间为 124.97–504.96 ms；参考额度保持 500 ms，缩放后的 CI 限制覆盖观察到的波动；中位数不是单次按键上限。预算不能通过环境变量覆盖。临时零额度覆盖每条拒绝路径；这些负向对照证明预算执行，而非优化或历史回归。另一项对照在键入前等待最终回复标记，实际输入重叠断言因此失败。紧凑合成 JSONL 为 3,262,577 字节；三个修正样本均报告重叠的真实输入事件，并在第 241 个 turn-tail 渲染后结束。 -### 首次实际 CI 运行 +### 实际 CI 运行 -[运行 34020120425，基准 job 101451135853](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34020120425/job/101451135853) 在 `6d1ba089e5052680961825c08aa4de19b4fe137a` 上通过完整基准清单。runner 为 `dsh-selfhosted-ci` 中的 `VM-7-113-ubuntu-ci-19`，使用 x64 Node 24.19.0 和 Chromium 149.0.7827.55。下列中位数来自每个场景的三个全新样本，本地参考表和源码预算保持不变。 +[运行 34020120425，基准 job 101451135853](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34020120425/job/101451135853) 在 `6d1ba089e5052680961825c08aa4de19b4fe137a` 上通过完整基准清单。runner 为 `dsh-selfhosted-ci` 中的 `VM-7-113-ubuntu-ci-19`，使用 x64 Node 24.19.0 和 Chromium 149.0.7827.55。[第 2 次执行，基准 job 101453296071](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34020120425/job/101453296071) 在相同 commit 上也通过完整清单，runner 为 `VM-7-113-ubuntu-ci-25`，Node 和 Chromium 版本相同。下列中位数来自每次运行中每个场景的三个全新样本，本地参考表和源码预算保持不变。 -| 终点 | 首次 CI 中位数 | -|---|---:| -| 浏览器打开 | 303.066 ms | -| 最慢更早分页 | 432.979 ms | -| 首次 Trajectory | 298.608 ms | -| 首段回复 | 740.265 ms | -| 流式主线程任务 | 1614.594 ms | -| 草稿键入 | 932.746 ms | -| 完整回复 | 1677.882 ms | -| 重连替换 | 29.232 ms | -| 重连保留 heap | 23.028 MiB | +| 终点 | 首次 CI 中位数 | 第二次 CI 中位数 | +|---|---:|---:| +| 浏览器打开 | 303.066 ms | 284.726 ms | +| 最慢更早分页 | 432.979 ms | 413.798 ms | +| 首次 Trajectory | 298.608 ms | 267.372 ms | +| 首段回复 | 740.265 ms | 658.906 ms | +| 流式主线程任务 | 1614.594 ms | 1035.385 ms | +| 草稿键入 | 932.746 ms | 142.148 ms | +| 完整回复 | 1677.882 ms | 1641.702 ms | +| 重连替换 | 29.232 ms | 31.674 ms | +| 重连保留 heap | 23.028 MiB | 23.028 MiB | -三个浏览器样本均报告 `inputOverlapped: true`，并在第 241 个 turn-tail 渲染后结束。强制 GC 后浏览器 heap 约为 52.94 MiB，DOM 元素为 17,064 个；两者仍为诊断终点。该运行支持此 runner 上的现有预算，而不证明普遍适用的 2× 浏览器速度比。必需的第二次独立 CI 运行仍待完成；没有放宽预算，也不声称产品优化。 +六个浏览器样本均报告 `inputOverlapped: true`，并在第 241 个 turn-tail 渲染后结束。强制 GC 后浏览器 heap 首次运行约为 52.94 MiB，第二次约为 53.00 MiB，两次 DOM 元素均为 17,064 个；这些仍为诊断终点。两次运行支持这些 runner 上的现有预算，而不证明普遍适用的 2× 浏览器速度比。草稿键入中位数从 932.746 ms 变化到 142.148 ms，因为该终点测量整个草稿键入，包含调度和 Playwright 可交互性等待，而非单次按键延迟保证。没有放宽预算，也不声称产品优化。 ## 考虑过的替代方案 From c51c16cdaca831389a90e82f550c8086435d995d Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 16:41:48 +0800 Subject: [PATCH 17/22] test(perf): respect browser failover provisioning policy --- .../2026-07-24-web-gui-browser-e2e-lane.i18n.yaml | 4 ++-- .../testing/2026-07-24-web-gui-browser-e2e-lane.md | 2 +- .../testing/2026-07-24-web-gui-browser-e2e-lane.zh.md | 2 +- .../2026-09-06-frontend-performance-budgets.i18n.yaml | 4 ++-- .../testing/2026-09-06-frontend-performance-budgets.md | 2 +- .../2026-09-06-frontend-performance-budgets.zh.md | 2 +- .github/workflows/ci.yml | 8 +++++++- benchmarks/long-session-browser/long-session.bench.ts | 2 +- scripts/ci-workflow.spec.ts | 10 ++++++++++ 9 files changed, 26 insertions(+), 10 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.i18n.yaml b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.i18n.yaml index 7bbae78b83..48e9139ebb 100644 --- a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md -2026-07-24-web-gui-browser-e2e-lane.md: a996b380a55fc33f44cfdc2e31e179bc11f40be3 -2026-07-24-web-gui-browser-e2e-lane.zh.md: 1a413b6b8277ac696c6b728de740494bbf056695 +2026-07-24-web-gui-browser-e2e-lane.md: 07a37ada9c2a43f04612048f9bff6b22d022ec40 +2026-07-24-web-gui-browser-e2e-lane.zh.md: 668e612712175821d6ad123ce364a7cb96e01272 diff --git a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md index a996b380a5..07a37ada9c 100644 --- a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md +++ b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md @@ -93,4 +93,4 @@ Surveyed AI-chat/agent web UIs and mocking layers (LibreChat, vercel/ai-chatbot ## Consequences -The web surface gains its record-once/replay-forever tier: the real chromium → SSE → apiproxy → loop → tools → persistence chain runs keylessly in ~10-30s, deterministic across repeat runs, with fixtures owned and re-recordable by the lane itself. Costs accepted: every intentional conversation-UI change ends with a keyless `DSH_SNAPSHOT=refresh` (golden churn is reviewed diff, anchors keep semantic green); the aria format is Playwright-owned — the one committed snapshot format the repo does not control — so playwright version bumps must be deliberate bump-and-refresh commits (the dependency floats `^1.49.0` in `apps/web/package.json`; pin exactly if churn bites); replay's first-call-order binding constrains scenarios to one prompting session each, with the consumption assertion as the tripwire; `compaction-basic` shares the session's replay cursor and stays inert only under the published 128k catalog window; and the required consumer job pays for Chromium provisioning and one browser run so the PR that changes the assembled UI owns its expected-output diff. The opt-in performance lane preserves a repeatable diagnostic workload without adding host-sensitive duration or memory expectations to CI; performance regressions remain a manually interpreted signal until the repository owns a calibrated benchmark environment. +The web surface gains its record-once/replay-forever tier: the real chromium → SSE → apiproxy → loop → tools → persistence chain runs keylessly in ~10-30s, deterministic across repeat runs, with fixtures owned and re-recordable by the lane itself. Costs accepted: every intentional conversation-UI change ends with a keyless `DSH_SNAPSHOT=refresh` (golden churn is reviewed diff, anchors keep semantic green); the aria format is Playwright-owned — the one committed snapshot format the repo does not control — so playwright version bumps must be deliberate bump-and-refresh commits (the dependency floats `^1.49.0` in `apps/web/package.json`; pin exactly if churn bites); replay's first-call-order binding constrains scenarios to one prompting session each, with the consumption assertion as the tripwire; `compaction-basic` shares the session's replay cursor and stays inert only under the published 128k catalog window; and the required consumer job pays for Chromium provisioning and one browser run so the PR that changes the assembled UI owns its expected-output diff. The opt-in performance lane preserves a repeatable, threshold-free diagnostic workload whose measurements require manual interpretation. The separate required [frontend performance benchmarks](2026-09-06-frontend-performance-budgets.md) enforce calibrated budgets in the isolated benchmark CI job; they do not add thresholds to the manual inventory. diff --git a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.zh.md b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.zh.md index 1a413b6b82..668e612712 100644 --- a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.zh.md +++ b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.zh.md @@ -93,4 +93,4 @@ Web GUI 以一条真实组装链交付——chromium 页面 → client 插件 bu ## 后果 -Web 表面获得了录制一次/永久回放的层级：真实 chromium → SSE → apiproxy → 循环 → 工具 → 持久化的链路以约 10-30 秒无密钥运行，重复运行结果确定，fixture 由车道自身持有并可重录。接受的成本：每次有意的会话 UI 变更都以一次无密钥 `DSH_SNAPSHOT=refresh` 收尾（预期输出变动是受评审的 diff，锚断言保住语义绿色）；aria 格式归 Playwright 所有——仓库唯一不受自己控制的提交快照格式——因此 playwright 版本升级必须是刻意的升级加刷新提交（依赖在 `apps/web/package.json` 中浮动为 `^1.49.0`；若变动伤人则改为精确锁定）；回放的首次调用顺序绑定把每个场景限制为至多一个发起提示的会话，消费断言是绊线；`compaction-basic` 与会话共享回放游标，仅在目录中发布的 128k 上下文窗口下保持闲置；必需的消费方任务承担 Chromium 供给与一次浏览器运行的成本，使改动组装后 UI 的 PR（Pull Request）持有相应的预期输出 diff。按需启用的性能车道保留了可重复的诊断工作负载，又不会向 CI 添加受 host 差异影响的时长或内存预期；在仓库拥有经校准的基准测试环境之前，性能回归仍是需要人工解读的信号。 +Web 表面获得了录制一次/永久回放的层级：真实 chromium → SSE → apiproxy → 循环 → 工具 → 持久化的链路以约 10-30 秒无密钥运行，重复运行结果确定，fixture 由车道自身持有并可重录。接受的成本：每次有意的会话 UI 变更都以一次无密钥 `DSH_SNAPSHOT=refresh` 收尾（预期输出变动是受评审的 diff，锚断言保住语义绿色）；aria 格式归 Playwright 所有——仓库唯一不受自己控制的提交快照格式——因此 playwright 版本升级必须是刻意的升级加刷新提交（依赖在 `apps/web/package.json` 中浮动为 `^1.49.0`；若变动伤人则改为精确锁定）；回放的首次调用顺序绑定把每个场景限制为至多一个发起提示的会话，消费断言是绊线；`compaction-basic` 与会话共享回放游标，仅在目录中发布的 128k 上下文窗口下保持闲置；必需的消费方任务承担 Chromium 供给与一次浏览器运行的成本，使改动组装后 UI 的 PR（Pull Request）持有相应的预期输出 diff。按需启用的性能车道保留可重复、无阈值的诊断工作负载，其测量需要人工解读。独立的必需[前端性能基准](2026-09-06-frontend-performance-budgets.zh.md)在隔离的基准 CI job 中执行经校准的预算；它们不向手动清单添加阈值。 diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml index 528630364f..85fb328bb5 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md -2026-09-06-frontend-performance-budgets.md: 9cbe5980336ad4b0f2a3fb2f7cbe70757885f82e -2026-09-06-frontend-performance-budgets.zh.md: 6df0d00285d60ce1ea0ec9d7284e7a73924459ab +2026-09-06-frontend-performance-budgets.md: 0474025b42c1f08bd964972081f2b612f3cbd8d8 +2026-09-06-frontend-performance-budgets.zh.md: 533c5784260674ed5f6834c83d172ccd4c01a96e diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md index 9cbe598033..0474025b42 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md @@ -72,4 +72,4 @@ All six browser samples report `inputOverlapped: true` and finish after the 241s The benchmark layer changes no product implementation or user-visible behavior. It adds approximately fifteen seconds of local browser/reconnect execution plus Web build and browser provisioning to the existing isolated CI lane. A fresh browser discards previous caches, but each workflow deliberately retains its own loaded history and previously activated Trajectory during continuation. -The baseline is independently mergeable and protects current performance; optimization layers tighten budgets only with repeated measurements and focused semantic tests. It does not cover sidebar cardinality, an hours-long soak, GPU presentation, real model latency, a published Host launch, or reconnect rendering. The manual Web diagnostic and existing functional browser tests retain those separate responsibilities. The existing Session performance note remains active because it owns Node calibration and persistence rationale; this note extends rather than supersedes it. +The baseline is independently mergeable and protects current performance; optimization layers tighten budgets only with repeated measurements and focused semantic tests. It does not cover sidebar cardinality, an hours-long soak, GPU presentation, real model latency, a published Host launch, or reconnect rendering. The [Web browser lane](2026-07-24-web-gui-browser-e2e-lane.md) retains its separate threshold-free manual diagnostics and functional browser tests; calibrated required measurements belong to this benchmark lane. The existing Session performance note remains active because it owns Node calibration and persistence rationale; this note extends rather than supersedes it. diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md index 6df0d00285..533c578426 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md @@ -72,4 +72,4 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 基准层不改变产品实现或用户可见行为。它在现有隔离 CI lane 中增加约十五秒的本地浏览器与重连执行，以及 Web 构建和浏览器安装成本。全新浏览器丢弃此前的缓存，但每个工作流刻意在续接期间保留自身已加载历史和曾激活的 Trajectory。 -基线可独立合并并保护现有性能；优化层只有在重复测量与聚焦语义测试支持下才收紧预算。它不覆盖侧栏数量级、数小时 soak、GPU 显示、真实模型延迟、发布版 Host 启动或重连渲染。手动 Web 诊断和现有功能浏览器测试继续各负其责。现有 Session 性能记录保持活跃，因为它拥有 Node 校准和持久化理由；本记录扩展而不替代它。 +基线可独立合并并保护现有性能；优化层只有在重复测量与聚焦语义测试支持下才收紧预算。它不覆盖侧栏数量级、数小时 soak、GPU 显示、真实模型延迟、发布版 Host 启动或重连渲染。[Web 浏览器车道](2026-07-24-web-gui-browser-e2e-lane.zh.md)保留独立的无阈值手动诊断与功能浏览器测试；经校准的必需测量由本基准车道负责。现有 Session 性能记录保持活跃，因为它拥有 Node 校准和持久化理由；本记录扩展而不替代它。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03afdec2ca..e6188bcc1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -206,9 +206,15 @@ jobs: - name: Install (immutable) run: pnpm install --frozen-lockfile - - name: Install benchmark browser + - name: Install benchmark browser and hosted dependencies + if: vars.DSH_CI_FAILOVER_LINUX != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]' run: pnpm --filter @deepseek-ai/dsh-benchmarks exec playwright install --with-deps chromium + # The persistent VM image owns Linux system packages; do not run apt here. + - name: Install benchmark browser on the failover VM + if: vars.DSH_CI_FAILOVER_LINUX == 'selfhosted' && github.event.pull_request.user.login != 'dependabot[bot]' + run: pnpm --filter @deepseek-ai/dsh-benchmarks exec playwright install chromium + - name: Run performance benchmarks env: DSH_GATE_VERBOSE: '1' diff --git a/benchmarks/long-session-browser/long-session.bench.ts b/benchmarks/long-session-browser/long-session.bench.ts index 3302e73fa4..783ceb5725 100644 --- a/benchmarks/long-session-browser/long-session.bench.ts +++ b/benchmarks/long-session-browser/long-session.bench.ts @@ -87,7 +87,7 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async await page.getByRole('row').last().waitFor() }) await page.getByRole('tab', { name: 'Chat', exact: true }).click() - await page.waitForFunction(selector => document.querySelectorAll(selector).length === 240, TAIL) + await page.waitForFunction(({ selector, expected }) => document.querySelectorAll(selector).length === expected, { selector: TAIL, expected: HISTORY_TURNS }) const composer = page.locator('[data-composer-input][contenteditable="true"]').last() await composer.fill('Continue the synthetic review and summarize the validation. '.repeat(30)) const cdp = await page.context().newCDPSession(page) diff --git a/scripts/ci-workflow.spec.ts b/scripts/ci-workflow.spec.ts index be2eb8865c..d83446a8c8 100644 --- a/scripts/ci-workflow.spec.ts +++ b/scripts/ci-workflow.spec.ts @@ -211,6 +211,16 @@ describe('CI workflow', () => { expect(aggregate.needs).toContain('node-24-bench') expect(node24Bench.name).toBe('node 24 / benchmarks') expect(node24Bench.env).toBeUndefined() + expect(node24Bench.steps).toContainEqual({ + name: 'Install benchmark browser and hosted dependencies', + if: "vars.DSH_CI_FAILOVER_LINUX != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]'", + run: 'pnpm --filter @deepseek-ai/dsh-benchmarks exec playwright install --with-deps chromium', + }) + expect(node24Bench.steps).toContainEqual({ + name: 'Install benchmark browser on the failover VM', + if: "vars.DSH_CI_FAILOVER_LINUX == 'selfhosted' && github.event.pull_request.user.login != 'dependabot[bot]'", + run: 'pnpm --filter @deepseek-ai/dsh-benchmarks exec playwright install chromium', + }) expect(node24Bench.steps).toContainEqual({ name: 'Run performance benchmarks', env: { DSH_GATE_VERBOSE: '1' }, From 0ac1d4d865f7d74ddd3f3a49d2e1e11e320aad60 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 20:30:38 +0800 Subject: [PATCH 18/22] test(perf): align browser provisioning with hosted benchmark runner --- .../2026-09-06-frontend-performance-budgets.i18n.yaml | 4 ++-- .../testing/2026-09-06-frontend-performance-budgets.md | 2 +- .../testing/2026-09-06-frontend-performance-budgets.zh.md | 2 +- .github/workflows/ci.yml | 6 ------ scripts/ci-workflow.spec.ts | 7 +------ 5 files changed, 5 insertions(+), 16 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml index 85fb328bb5..5994688400 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md -2026-09-06-frontend-performance-budgets.md: 0474025b42c1f08bd964972081f2b612f3cbd8d8 -2026-09-06-frontend-performance-budgets.zh.md: 533c5784260674ed5f6834c83d172ccd4c01a96e +2026-09-06-frontend-performance-budgets.md: fd657efcdc18f5879e8a48ff8991e87d466b1fe1 +2026-09-06-frontend-performance-budgets.zh.md: 67b62a2699c37d11b54dea0c5f8cce19e6ea0022 diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md index 0474025b42..fd657efcdc 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md @@ -12,7 +12,7 @@ A fast Node conversation fold does not prove that a browser paints a long conver The existing serial benchmark inventory includes two frontend owners: [active reconnect](../../../../benchmarks/active-stream-reconnect/README.md) and a [browser workflow](../../../../benchmarks/long-session-browser/README.md). The browser workflow combines cold open, older-page navigation, first Trajectory activation, return to Chat, and a paced response with trusted keyboard input into one sequential scenario. These are endpoints of one workflow, not independent cold scenarios. The settled conversation-fold benchmark remains unchanged. -`build:bench` keeps the Node-only library and worker build. `test:bench` additionally builds the Web shell before running all cases; the required benchmark CI job provisions Chromium and enables the existing verbose gate output so successful raw samples remain available for calibration. Browser cases reuse the shipped-composition Web scaffold with private temporary roots and an atomically assigned loopback port. Only the nondeterministic model is replaced by synthetic replay. The scaffold Host runs under the existing Vitest source resolver; measured Client rendering runs built bundles in fresh Chromium processes. Browser wall times therefore include this test Host, transport, Playwright actionability, and rendering, and are not claims about a published Host process. +`build:bench` keeps the Node-only library and worker build. `test:bench` additionally builds the Web shell before running all cases; the required benchmark CI job follows the [standard hosted runner decision](2026-09-06-standard-hosted-benchmark-runner.md), unconditionally provisions Chromium and its Linux dependencies on that hosted runner, and enables the existing verbose gate output so successful raw samples remain available for calibration. Browser cases reuse the shipped-composition Web scaffold with private temporary roots and an atomically assigned loopback port. Only the nondeterministic model is replaced by synthetic replay. The scaffold Host runs under the existing Vitest source resolver; measured Client rendering runs built bundles in fresh Chromium processes. Browser wall times therefore include this test Host, transport, Playwright actionability, and rendering, and are not claims about a published Host process. The browser input contains 240 closed turns, 40 tool results, and 20 code fences, plus mixed-language prose and reasoning. Historical Assistant records carry matching compact streams built through the production accumulator with 12-character reasoning/text deltas and 8-character tool-argument deltas; empty streams would omit stored and transferred payload costs. Nine older-page actions exhaust this input from its observed 25-turn initial window; the readiness probe follows mounted turn growth rather than duplicating the pagination algorithm. Each sample uses a fresh scaffold and browser. Setup, seeding, browser launch, initial shell load, and sidebar expansion are excluded from open timing. Open ends at transcript availability and an editable composer; page and navigation timings end at their target DOM state. Two animation frames include a rendering opportunity, not hardware presentation or a guarantee that every offscreen node painted. diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md index 533c578426..67b62a2699 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md @@ -12,7 +12,7 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 现有串行基准清单包含两个前端所有者：[活跃重连](../../../../benchmarks/active-stream-reconnect/README.zh.md)和[浏览器工作流](../../../../benchmarks/long-session-browser/README.zh.md)。浏览器工作流在一个顺序场景中组合冷打开、更早分页导航、首次激活 Trajectory、返回 Chat，以及伴随真实键盘输入的有节奏回复。这些是同一工作流的测量终点，而不是相互独立的冷场景。已结算对话折叠基准保持不变。 -`build:bench` 保留仅 Node 的 library 与 worker 构建。`test:bench` 额外构建 Web shell 后再运行所有用例；必需的基准 CI job 安装 Chromium，并启用现有门禁详细输出，使成功用例的原始样本可用于校准。浏览器用例复用产品组合的 Web scaffold，使用私有临时目录和原子分配的回环端口。只有不确定的模型被合成重放替代。scaffold Host 通过现有 Vitest 源码解析器运行；被测 Client 渲染在全新 Chromium 进程中执行构建后的 bundle。因此浏览器壁钟时间包含测试 Host、传输、Playwright 可交互性等待及渲染，不代表发布版 Host 进程。 +`build:bench` 保留仅 Node 的 library 与 worker 构建。`test:bench` 额外构建 Web shell 后再运行所有用例；必需的基准 CI job 遵循[标准托管运行器决策](2026-09-06-standard-hosted-benchmark-runner.zh.md)，在该托管运行器上无条件安装 Chromium 及其 Linux 依赖，并启用现有门禁详细输出，使成功用例的原始样本可用于校准。浏览器用例复用产品组合的 Web scaffold，使用私有临时目录和原子分配的回环端口。只有不确定的模型被合成重放替代。scaffold Host 通过现有 Vitest 源码解析器运行；被测 Client 渲染在全新 Chromium 进程中执行构建后的 bundle。因此浏览器壁钟时间包含测试 Host、传输、Playwright 可交互性等待及渲染，不代表发布版 Host 进程。 浏览器输入包含 240 个已关闭轮次、40 个工具结果和 20 个代码块，以及混合语言正文和推理。历史 Assistant 记录携带匹配的紧凑 stream，通过生产 accumulator 按 12 字符推理/文本 delta 和 8 字符工具参数 delta 构建；空 stream 会遗漏存储与传输负载成本。从观察到的初始 25 轮窗口开始，九次更早分页操作读完该输入；就绪探针跟踪已挂载轮次增长，不复制分页算法。每个样本使用全新 scaffold 和浏览器。环境准备、数据播种、浏览器启动、初始 shell 加载及侧栏展开不计入打开时间。打开测量在对话可用且输入框可编辑时结束；分页与导航测量在目标 DOM 状态出现时结束。两次动画帧包含一次渲染机会，不代表硬件显示或保证每个屏幕外节点都已绘制。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6188bcc1a..0eaa6f6335 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -207,14 +207,8 @@ jobs: run: pnpm install --frozen-lockfile - name: Install benchmark browser and hosted dependencies - if: vars.DSH_CI_FAILOVER_LINUX != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]' run: pnpm --filter @deepseek-ai/dsh-benchmarks exec playwright install --with-deps chromium - # The persistent VM image owns Linux system packages; do not run apt here. - - name: Install benchmark browser on the failover VM - if: vars.DSH_CI_FAILOVER_LINUX == 'selfhosted' && github.event.pull_request.user.login != 'dependabot[bot]' - run: pnpm --filter @deepseek-ai/dsh-benchmarks exec playwright install chromium - - name: Run performance benchmarks env: DSH_GATE_VERBOSE: '1' diff --git a/scripts/ci-workflow.spec.ts b/scripts/ci-workflow.spec.ts index d83446a8c8..02ffac7aba 100644 --- a/scripts/ci-workflow.spec.ts +++ b/scripts/ci-workflow.spec.ts @@ -213,14 +213,9 @@ describe('CI workflow', () => { expect(node24Bench.env).toBeUndefined() expect(node24Bench.steps).toContainEqual({ name: 'Install benchmark browser and hosted dependencies', - if: "vars.DSH_CI_FAILOVER_LINUX != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]'", run: 'pnpm --filter @deepseek-ai/dsh-benchmarks exec playwright install --with-deps chromium', }) - expect(node24Bench.steps).toContainEqual({ - name: 'Install benchmark browser on the failover VM', - if: "vars.DSH_CI_FAILOVER_LINUX == 'selfhosted' && github.event.pull_request.user.login != 'dependabot[bot]'", - run: 'pnpm --filter @deepseek-ai/dsh-benchmarks exec playwright install chromium', - }) + expect(JSON.stringify(node24Bench.steps)).not.toContain('DSH_CI_FAILOVER_LINUX') expect(node24Bench.steps).toContainEqual({ name: 'Run performance benchmarks', env: { DSH_GATE_VERBOSE: '1' }, From 620c2b0b271ab317a5b91e24bbb537408778618f Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 20:48:52 +0800 Subject: [PATCH 19/22] fix(benchmarks): calibrate hosted frontend endpoints and preserve input overlap --- ...-06-frontend-performance-budgets.i18n.yaml | 4 +- ...2026-09-06-frontend-performance-budgets.md | 14 +++-- ...6-09-06-frontend-performance-budgets.zh.md | 14 +++-- .../active-stream-reconnect/README.i18n.yaml | 4 +- benchmarks/active-stream-reconnect/README.md | 2 +- .../active-stream-reconnect/README.zh.md | 2 +- .../reconnect.bench.client.ts | 22 +++++-- .../long-session-browser/README.i18n.yaml | 4 +- benchmarks/long-session-browser/README.md | 4 +- benchmarks/long-session-browser/README.zh.md | 4 +- .../long-session.bench.ts | 59 +++++++++++++++---- .../long-session-browser/synthetic-history.ts | 2 +- 12 files changed, 97 insertions(+), 38 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml index 5994688400..7a0d2ca2e6 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md -2026-09-06-frontend-performance-budgets.md: fd657efcdc18f5879e8a48ff8991e87d466b1fe1 -2026-09-06-frontend-performance-budgets.zh.md: 67b62a2699c37d11b54dea0c5f8cce19e6ea0022 +2026-09-06-frontend-performance-budgets.md: de0cfd5bf03a2b2e2b69fac9dba451e090804fe5 +2026-09-06-frontend-performance-budgets.zh.md: 9df105acdbd28f6cdd4aaa14ad98d847027f4cf4 diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md index fd657efcdc..de0cfd5bf0 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md @@ -16,15 +16,15 @@ The existing serial benchmark inventory includes two frontend owners: [active re The browser input contains 240 closed turns, 40 tool results, and 20 code fences, plus mixed-language prose and reasoning. Historical Assistant records carry matching compact streams built through the production accumulator with 12-character reasoning/text deltas and 8-character tool-argument deltas; empty streams would omit stored and transferred payload costs. Nine older-page actions exhaust this input from its observed 25-turn initial window; the readiness probe follows mounted turn growth rather than duplicating the pagination algorithm. Each sample uses a fresh scaffold and browser. Setup, seeding, browser launch, initial shell load, and sidebar expansion are excluded from open timing. Open ends at transcript availability and an editable composer; page and navigation timings end at their target DOM state. Two animation frames include a rendering opportunity, not hardware presentation or a guarantee that every offscreen node painted. -The continuation sends 120 text deltas at 8 ms replay pacing. Send lookup stays inside the composer seat; first/final marker lookups stay inside the latest Assistant step and retain visible-state waits. The synchronous input witness reads that same bounded reply. Whole-history text and accessibility queries add observer CPU and garbage collection to the measured interval, so reducing that observer work is benchmark repair, not product optimization. It records click-to-first-visible-reply, trusted draft typing whose first actual input event observes the first reply but no completion marker, complete reply wall time through settled persistence and the new rendered turn-tail, and Chromium main-thread task duration. The complete wall budget adds the fixed 992 ms scripted pacing to a scaled overhead allowance; input and completion have their own enforced budgets. Post-GC browser heap and DOM counts remain diagnostics because one endpoint does not prove a leak. +The continuation sends 120 text deltas at 16 ms replay pacing. The input witness is installed before Send; typing starts immediately after the first visible marker, without a separate pre-input animation-frame wait. Send lookup stays inside the composer seat; first/final marker lookups stay inside the latest Assistant step and retain visible-state waits. The synchronous input witness reads that same bounded reply. Whole-history text and accessibility queries add observer CPU and garbage collection to the measured interval, so reducing that observer work is benchmark repair, not product optimization. It records click-to-first-visible-reply, trusted draft typing whose first actual input event observes the first reply but no completion marker, complete reply wall time through settled persistence and the new rendered turn-tail, and Chromium main-thread task duration. The complete wall budget adds the fixed 1984 ms scripted pacing to a scaled overhead allowance; input and completion have their own enforced budgets. Post-GC browser heap and DOM counts remain diagnostics because one endpoint does not prove a leak. Reconnect uses three fresh compiled plain-Node children. Each creates a 100,000-delta reasoning prefix with distinct timestamps and two compact records before timing `ClientAssistantStream.replace()`. GC precedes the baseline and follows replacement while the result remains reachable; replacement time excludes both collections. The report consumes the result after collection and checks that the next dense live frame remains accepted. This measures reconstruction, not transport, rendering, or an entire reconnect workflow. ## Calibration -Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149.0.7827.55, at product revision `925e012340`, establish the baseline below. An isolated repeat follows a complete workflow smoke. Each browser sample reports raw endpoint values and every page; the paging verdict uses the median of the sample maxima. Reconnect reports all child measurements. Source reference constants retain the original allowances after two passing CI runs; the bounded-observer 261.60 ms paging median exceeds its 260 ms reference allowance but remains below its 650 ms CI limit; the shared 2× time scale and 1.25× variance allowance produce CI limits. Memory uses only variance allowance. The shared scale originates in Node CI calibration. Both actual x64 browser runs below pass the fixed budgets on unchanged benchmark code; this supplies repeated-run evidence for these runners, not a universal browser speed ratio. +Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149.0.7827.55, at product revision `925e012340`, establish the baseline below. An isolated repeat follows a complete workflow smoke. Each browser sample reports raw endpoint values and every page; the paging verdict uses the median of the sample maxima. Reconnect reports all child measurements. The following historical reference table uses 8 ms replay pacing and includes a two-frame wait in first-reply timing. Standard-hosted open and reconnect expectations are recorded separately below; other source reference constants retain these allowances. The bounded-observer 261.60 ms paging median exceeds its 260 ms reference allowance but remains below its 650 ms CI limit; the shared 2× time scale and 1.25× variance allowance produce CI limits. Memory uses only variance allowance. The shared scale originates in Node CI calibration. Both actual x64 browser runs below pass the fixed budgets on unchanged benchmark code; this supplies repeated-run evidence for these runners, not a universal browser speed ratio. -| Endpoint | Measured median | Reference allowance | CI limit | +| Endpoint | Measured median | Reference allowance | Historical CI limit | |---|---:|---:|---:| | Browser open | 184.62 ms | 200 ms | 500 ms | | Slowest older page | 261.60 ms | 260 ms | 650 ms | @@ -54,7 +54,13 @@ Draft typing spans 124.97–504.96 ms across the three isolated samples; the ref | Reconnect replacement | 29.232 ms | 31.674 ms | | Reconnect retained heap | 23.028 MiB | 23.028 MiB | -All six browser samples report `inputOverlapped: true` and finish after the 241st rendered turn-tail. Post-GC browser heap is approximately 52.94 MiB in the first run and 53.00 MiB in the second, with 17,064 DOM elements in both; these remain diagnostic endpoints. Both runs support the existing budgets on these runners, not a universal 2× browser speed ratio. Draft-typing medians vary from 932.746 ms to 142.148 ms because the endpoint measures the entire typed draft, including scheduling and Playwright actionability, rather than a per-key latency guarantee. No budget is relaxed and no product optimization is claimed. +All six browser samples report `inputOverlapped: true` and finish after the 241st rendered turn-tail. Post-GC browser heap is approximately 52.94 MiB in the first run and 53.00 MiB in the second, with 17,064 DOM elements in both; these remain diagnostic endpoints. Both runs support the existing budgets on these runners, not a universal 2× browser speed ratio. Draft-typing medians vary from 932.746 ms to 142.148 ms because the endpoint measures the entire typed draft, including scheduling and Playwright actionability, rather than a per-key latency guarantee. These are self-hosted measurements, not standard-hosted calibration; no product optimization is claimed. + +### Standard hosted expectations and input scheduling + +[Run 34033336246, job 101487216170](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336246/job/101487216170) on standard hosted Ubuntu with two CPUs records reconnect replacements of 46.574411, 46.067910, and 44.193704 ms, with 23.028 MiB retained heap. The endpoint-specific expectation is 50 ms; the existing 1.25× headroom gives a 63 ms integer ceiling. The 30 MiB memory budget and shared machine factor remain unchanged. Browser open records 681.276514 and 541.051233 ms before the third sample fails input overlap; both exceed the historical 500 ms limit. Its hosted expectation is 700 ms, giving an 875 ms ceiling. Deterministic controls pass these recorded values and reject values above the new ceilings through the same assertions as the measured verdicts. Complete repeated hosted verdicts remain required; the two open values are not a three-sample median. + +A local diagnostic with temporary 3× Chromium CPU throttling reproduces the overlap failure: the first marker becomes visible at 1321 ms, two animation frames finish at 1370 ms, and the composer click finishes at 1660 ms; the actual input is trusted but already sees DONE. Removing the frame wait and installing the witness before Send still leaves a run with first visibility at 1415 ms and click completion at 1726 ms, after the original 992 ms scripted stream. The fixed 16 ms cadence keeps the same 120 deltas and payload, providing 1984 ms of scripted pacing for this workload. Only that pacing term changes in the complete-wall allowance (4484 ms); input, first-reply, and main-thread overhead allowances remain unchanged. With the same diagnostic slowdown, three 16 ms samples reach first visibility at 1307/1479/1599 ms and accept trusted input before DONE; their post-DONE controls reject it. The diagnostic is not a CPU-ratio calibration. Each measured sample still requires trusted input while FIRST is present and DONE absent; a post-measurement trusted key after DONE must fail that same assertion. Host settlement and the 241st rendered turn-tail remain completion witnesses. ## Alternatives considered diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md index 67b62a2699..9df105acdb 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md @@ -16,15 +16,15 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 浏览器输入包含 240 个已关闭轮次、40 个工具结果和 20 个代码块，以及混合语言正文和推理。历史 Assistant 记录携带匹配的紧凑 stream，通过生产 accumulator 按 12 字符推理/文本 delta 和 8 字符工具参数 delta 构建；空 stream 会遗漏存储与传输负载成本。从观察到的初始 25 轮窗口开始，九次更早分页操作读完该输入；就绪探针跟踪已挂载轮次增长，不复制分页算法。每个样本使用全新 scaffold 和浏览器。环境准备、数据播种、浏览器启动、初始 shell 加载及侧栏展开不计入打开时间。打开测量在对话可用且输入框可编辑时结束；分页与导航测量在目标 DOM 状态出现时结束。两次动画帧包含一次渲染机会，不代表硬件显示或保证每个屏幕外节点都已绘制。 -续接以 8 ms 重放间隔发送 120 个文本 delta。发送控件查找限制在 composer seat；首段/最终标记查找限制在最新 Assistant step，并保留可见状态等待。同步输入证据读取同一个受限回复。全历史文本与无障碍查询会向测量区间加入观察器 CPU 和垃圾回收成本，因此减少此类观察工作属于基准修正，而非产品优化。它记录点击到首段可见回复的时间、首个实际输入事件观察到首段回复且完成标记尚未出现时的真实草稿键入、直到持久化结算并渲染新 turn-tail 的完整回复壁钟时间，以及 Chromium 主线程任务时间。完整壁钟预算在缩放后的额外开销额度上加固定的 992 ms 脚本节奏；输入和完成均有独立执行的预算。强制 GC 后的浏览器 heap 和 DOM 数量仍仅供诊断，因为单个终点不能证明泄漏。 +续接以 16 ms 重放间隔发送 120 个文本 delta。输入观察器在发送前安装；首个标记可见后立即开始键入，不单独等待输入前动画帧。发送控件查找限制在 composer seat；首段/最终标记查找限制在最新 Assistant step，并保留可见状态等待。同步输入证据读取同一个受限回复。全历史文本与无障碍查询会向测量区间加入观察器 CPU 和垃圾回收成本，因此减少此类观察工作属于基准修正，而非产品优化。它记录点击到首段可见回复的时间、首个实际输入事件观察到首段回复且完成标记尚未出现时的真实草稿键入、直到持久化结算并渲染新 turn-tail 的完整回复壁钟时间，以及 Chromium 主线程任务时间。完整壁钟预算在缩放后的额外开销额度上加固定的 1984 ms 脚本节奏；输入和完成均有独立执行的预算。强制 GC 后的浏览器 heap 和 DOM 数量仍仅供诊断，因为单个终点不能证明泄漏。 重连使用三个全新编译后的纯 Node 子进程。各进程在计时 `ClientAssistantStream.replace()` 前创建包含不同时间戳、两条紧凑记录和 100,000 个 delta 的推理前缀。在基线前执行 GC，并在结果仍可达时于替换后再次 GC；替换时间不含两次回收。报告在回收后消费结果，并检查下一个稠密序号的实时 frame 仍被接受。这测量重建，不测量传输、渲染或完整重连工作流。 ## 校准 -在 arm64 参考机器、Node 24.19、Chromium 149.0.7827.55 和产品版本 `925e012340` 上，三个样本的中位数建立下表基线。完整工作流 smoke 后执行一次隔离重复测量。每个浏览器样本报告原始终点数据和每一页；分页判定使用各样本最大值的中位数。重连报告全部子进程测量。源码参考常量在两次 CI 运行通过后保留原额度；受限观察器的分页中位数 261.60 ms 超过 260 ms 参考额度，但仍低于 650 ms CI 限制；共享的 2× 时间倍率和 1.25× 方差余量产生 CI 限制。内存仅使用方差余量。共享倍率源自 Node CI 校准。下述两次实际 x64 浏览器运行在基准代码不变的情况下均通过固定预算；这提供这些 runner 的重复运行证据，而非普遍适用的浏览器速度比。 +在 arm64 参考机器、Node 24.19、Chromium 149.0.7827.55 和产品版本 `925e012340` 上，三个样本的中位数建立下表基线。完整工作流 smoke 后执行一次隔离重复测量。每个浏览器样本报告原始终点数据和每一页；分页判定使用各样本最大值的中位数。重连报告全部子进程测量。下列历史参考表使用 8 ms 重放节奏，首段回复计时包含两帧等待。标准托管打开和重连预期在下文单独记录；其他源码参考常量保留这些额度。受限观察器的分页中位数 261.60 ms 超过 260 ms 参考额度，但仍低于 650 ms CI 限制；共享的 2× 时间倍率和 1.25× 方差余量产生 CI 限制。内存仅使用方差余量。共享倍率源自 Node CI 校准。下述两次实际 x64 浏览器运行在基准代码不变的情况下均通过固定预算；这提供这些 runner 的重复运行证据，而非普遍适用的浏览器速度比。 -| 终点 | 实测中位数 | 参考额度 | CI 限制 | +| 终点 | 实测中位数 | 参考额度 | 历史 CI 限制 | |---|---:|---:|---:| | 浏览器打开 | 184.62 ms | 200 ms | 500 ms | | 最慢更早分页 | 261.60 ms | 260 ms | 650 ms | @@ -54,7 +54,13 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 | 重连替换 | 29.232 ms | 31.674 ms | | 重连保留 heap | 23.028 MiB | 23.028 MiB | -六个浏览器样本均报告 `inputOverlapped: true`，并在第 241 个 turn-tail 渲染后结束。强制 GC 后浏览器 heap 首次运行约为 52.94 MiB，第二次约为 53.00 MiB，两次 DOM 元素均为 17,064 个；这些仍为诊断终点。两次运行支持这些 runner 上的现有预算，而不证明普遍适用的 2× 浏览器速度比。草稿键入中位数从 932.746 ms 变化到 142.148 ms，因为该终点测量整个草稿键入，包含调度和 Playwright 可交互性等待，而非单次按键延迟保证。没有放宽预算，也不声称产品优化。 +六个浏览器样本均报告 `inputOverlapped: true`，并在第 241 个 turn-tail 渲染后结束。强制 GC 后浏览器 heap 首次运行约为 52.94 MiB，第二次约为 53.00 MiB，两次 DOM 元素均为 17,064 个；这些仍为诊断终点。两次运行支持这些 runner 上的现有预算，而不证明普遍适用的 2× 浏览器速度比。草稿键入中位数从 932.746 ms 变化到 142.148 ms，因为该终点测量整个草稿键入，包含调度和 Playwright 可交互性等待，而非单次按键延迟保证。这些是自托管测量，而非标准托管校准；不声称产品优化。 + +### 标准托管预期与输入调度 + +[运行 34033336246，job 101487216170](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336246/job/101487216170) 在双 CPU 标准托管 Ubuntu 上记录重连替换时间 46.574411、46.067910 和 44.193704 ms，保留 heap 为 23.028 MiB。该终点的预期为 50 ms；现有 1.25× 余量产生向上取整后的 63 ms 上限。30 MiB 内存预算及共享机器倍率不变。浏览器打开记录 681.276514 和 541.051233 ms，第三个样本因输入重叠失败而中止；两个值均超过历史 500 ms 上限。其托管预期为 700 ms，上限为 875 ms。确定性对照通过这些记录值，并使用与测量判定相同的断言拒绝超过新上限的值。仍需完整的托管重复运行判定；这两个打开值不是三样本中位数。 + +临时使用 3× Chromium CPU 降速的本地诊断复现重叠失败：首个标记在 1321 ms 可见，两次动画帧在 1370 ms 结束，输入框点击在 1660 ms 完成；实际输入是真实事件，但已看到 DONE。移除帧等待并在发送前安装观察器后，一次运行仍在 1415 ms 才看到首个标记，点击在 1726 ms 完成，晚于原先 992 ms 的脚本流。固定 16 ms 节奏保留相同的 120 个 delta 和负载，为该工作负载提供 1984 ms 脚本节奏。完整壁钟额度仅改变该节奏项（4484 ms）；输入、首段回复及主线程额外开销额度不变。在相同诊断降速下，三个 16 ms 样本在 1307/1479/1599 ms 达到首段可见状态，并接受 DONE 之前的真实输入；其 DONE 之后的对照拒绝该输入。该诊断不是 CPU 比率校准。每个测量样本仍要求真实输入发生时 FIRST 存在且 DONE 不存在；测量后在 DONE 之后发送的真实按键必须无法通过同一个断言。Host 结算和第 241 个已渲染 turn-tail 仍是完成证据。 ## 考虑过的替代方案 diff --git a/benchmarks/active-stream-reconnect/README.i18n.yaml b/benchmarks/active-stream-reconnect/README.i18n.yaml index b4a785f27e..107b140b68 100644 --- a/benchmarks/active-stream-reconnect/README.i18n.yaml +++ b/benchmarks/active-stream-reconnect/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write benchmarks/active-stream-reconnect/README.md -README.md: 2f10512f144b923df2d89ff2766c4acf1059a652 -README.zh.md: b0c97ea7f06281a71f4e633b9f60f5f5b2ebf4fa +README.md: e75a41eba3952bb4db343c2018e217e6f5e88f96 +README.zh.md: acf0260f53855f9f4e643b2e72c67aa7a8afd6bb diff --git a/benchmarks/active-stream-reconnect/README.md b/benchmarks/active-stream-reconnect/README.md index 2f10512f14..e75a41eba3 100644 --- a/benchmarks/active-stream-reconnect/README.md +++ b/benchmarks/active-stream-reconnect/README.md @@ -2,6 +2,6 @@ English | [中文](README.zh.md) -[reconnect.bench.client.ts](reconnect.bench.client.ts) measures the production Client fold when a reconnect carries an unfinished 100,000-delta reasoning prefix. A compiled private adapter reaches `ClientAssistantStream.replace()` without adding product exports. Three fresh plain-Node workers synthesize the compact baseline before timing; replacement time and retained heap after forced GC have separate median budgets. The next dense live frame must still be accepted. +[reconnect.bench.client.ts](reconnect.bench.client.ts) measures the production Client fold when a reconnect carries an unfinished 100,000-delta reasoning prefix. A compiled private adapter reaches `ClientAssistantStream.replace()` without adding product exports. Three fresh plain-Node workers synthesize the compact baseline before timing; replacement time and retained heap after forced GC have separate median budgets. The next dense live frame must still be accepted. Standard hosted CI uses a 50 ms replacement expectation with the shared 1.25× headroom (63 ms ceiling); the retained-heap budget remains 30 MiB. Recorded-sample and synthetic-regression controls exercise the same time assertion as the worker verdict. Build with `pnpm run build:bench`, then select `benchmarks/active-stream-reconnect` in `vitest.bench.config.ts`. This focused Node workload neither builds nor measures browser rendering. [Frontend performance budgets](../../.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md) records calibration and exclusions. diff --git a/benchmarks/active-stream-reconnect/README.zh.md b/benchmarks/active-stream-reconnect/README.zh.md index b0c97ea7f0..acf0260f53 100644 --- a/benchmarks/active-stream-reconnect/README.zh.md +++ b/benchmarks/active-stream-reconnect/README.zh.md @@ -2,6 +2,6 @@ [English](README.md) | 中文 -[reconnect.bench.client.ts](reconnect.bench.client.ts) 测量重连携带未完成的 100,000 个 reasoning delta 前缀时，生产 Client 的折叠成本。编译后的私有适配器调用 `ClientAssistantStream.replace()`，不增加产品导出。三个全新纯 Node worker 在计时前合成紧凑 baseline；替换时间与强制 GC 后的保留 heap 分别执行中位数预算检查。下一个稠密序号的实时 frame 仍须被接受。 +[reconnect.bench.client.ts](reconnect.bench.client.ts) 测量重连携带未完成的 100,000 个 reasoning delta 前缀时，生产 Client 的折叠成本。编译后的私有适配器调用 `ClientAssistantStream.replace()`，不增加产品导出。三个全新纯 Node worker 在计时前合成紧凑 baseline；替换时间与强制 GC 后的保留 heap 分别执行中位数预算检查。下一个稠密序号的实时 frame 仍须被接受。标准托管 CI 使用 50 ms 替换预期及共享的 1.25× 余量（向上取整为 63 ms）；保留 heap 预算仍为 30 MiB。记录样本和合成回归对照使用与 worker 判定相同的时间断言。 通过 `pnpm run build:bench` 构建，再在 `vitest.bench.config.ts` 中选择 `benchmarks/active-stream-reconnect`。该聚焦 Node workload 既不构建也不测量浏览器渲染。[前端性能预算](../../.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md)记录校准与排除项。 diff --git a/benchmarks/active-stream-reconnect/reconnect.bench.client.ts b/benchmarks/active-stream-reconnect/reconnect.bench.client.ts index 374173495b..3388f9e3b9 100644 --- a/benchmarks/active-stream-reconnect/reconnect.bench.client.ts +++ b/benchmarks/active-stream-reconnect/reconnect.bench.client.ts @@ -5,10 +5,24 @@ import { runBuiltBenchmarkWorker } from '../support/built-worker.ts' import { ciTimeBudget, PERFORMANCE_BUDGET_HEADROOM } from '../support/calibration.ts' import type { ReconnectReport } from './reconnect.worker.client.ts' -const REFERENCE_REPLACE_MS = 16 +const EXPECTED_REPLACE_CI_MS = 50 +const REPLACE_BUDGET_MS = Math.ceil(EXPECTED_REPLACE_CI_MS * PERFORMANCE_BUDGET_HEADROOM) const REFERENCE_RETAINED_MB = 24 const SAMPLES = 3 +function expectReplacementWithinBudget(value: number, budget: number): void { + expect(value).toBeLessThanOrEqual(budget) +} + +it('accepts recorded hosted reconnect samples and rejects replacement regressions', () => { + const recordedMedian = [46.574411, 46.067910, 44.193704].toSorted((a, b) => a - b)[1]! + expect(() => expectReplacementWithinBudget(recordedMedian, ciTimeBudget(16))).toThrow() + expectReplacementWithinBudget(recordedMedian, REPLACE_BUDGET_MS) + expect(REPLACE_BUDGET_MS).toBe(63) + expect(() => expectReplacementWithinBudget(75, REPLACE_BUDGET_MS)).toThrow() + expect(() => expectReplacementWithinBudget(REPLACE_BUDGET_MS + 1, REPLACE_BUDGET_MS)).toThrow() +}) + it('reconstructs a 100000-delta live prefix within baseline time and retained-memory budgets', async () => { const samples: ReconnectReport[] = [] for (let sample = 0; sample < SAMPLES; sample++) { @@ -26,9 +40,9 @@ it('reconstructs a 100000-delta live prefix within baseline time and retained-me } const replaceMs = samples.map(sample => sample.replaceMs).toSorted((a, b) => a - b)[1]! const retainedMb = samples.map(sample => sample.retainedMb).toSorted((a, b) => a - b)[1]! - const budgetMs = ciTimeBudget(REFERENCE_REPLACE_MS) + const budgetMs = REPLACE_BUDGET_MS const budgetMb = REFERENCE_RETAINED_MB * PERFORMANCE_BUDGET_HEADROOM - console.log(JSON.stringify({ benchmark: 'active-stream-reconnect', samples, median: { replaceMs, retainedMb }, referenceMs: REFERENCE_REPLACE_MS, referenceMb: REFERENCE_RETAINED_MB, budgetMs, budgetMb })) - expect.soft(replaceMs).toBeLessThanOrEqual(budgetMs) + console.log(JSON.stringify({ benchmark: 'active-stream-reconnect', samples, median: { replaceMs, retainedMb }, expectedCiMs: EXPECTED_REPLACE_CI_MS, referenceMb: REFERENCE_RETAINED_MB, budgetMs, budgetMb })) + expectReplacementWithinBudget(replaceMs, budgetMs) expect.soft(retainedMb).toBeLessThanOrEqual(budgetMb) }) diff --git a/benchmarks/long-session-browser/README.i18n.yaml b/benchmarks/long-session-browser/README.i18n.yaml index f51684c58f..0154487dc7 100644 --- a/benchmarks/long-session-browser/README.i18n.yaml +++ b/benchmarks/long-session-browser/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write benchmarks/long-session-browser/README.md -README.md: a926ef0ff7c48caf45ed77e760a4d764f778c097 -README.zh.md: 6b35fd570f1ba53f114f2924607effc84dacb4a5 +README.md: 02d5555853ebf5bf9cc583c6e245e099caef9b30 +README.zh.md: dee8b9bad08a3d1f7a09d65058cad20fd5d60c33 diff --git a/benchmarks/long-session-browser/README.md b/benchmarks/long-session-browser/README.md index a926ef0ff7..02d5555853 100644 --- a/benchmarks/long-session-browser/README.md +++ b/benchmarks/long-session-browser/README.md @@ -10,8 +10,8 @@ This reference describes the required Chromium workflow in [long-session.bench.t ## Measurements -Three fresh browser processes and scaffold worlds produce raw samples and median verdicts. Open and paging end after the expected transcript state and two animation frames; this includes a rendering opportunity, not a hardware presentation timestamp. Paging reports every page and gates the median of each sample’s slowest page. Stream reports first visible reply, trusted draft typing, complete reply wall time, and Chromium main-thread task duration. Send lookup is scoped to the composer seat; reply-marker lookups and the input-event text witness read only the latest Assistant step, avoiding repeated whole-history text and accessibility scans. The actual first input event must observe an unfinished reply; completion waits for the new rendered turn-tail after Host settlement. Heap after forced GC and DOM counts are diagnostics, not leak budgets. +Three fresh browser processes and scaffold worlds produce raw samples and median verdicts. Open and paging end after the expected transcript state and two animation frames; this includes a rendering opportunity, not a hardware presentation timestamp. Paging reports every page and gates the median of each sample’s slowest page. Stream reports first visible reply, trusted draft typing, complete reply wall time, and Chromium main-thread task duration. Send lookup is scoped to the composer seat; reply-marker lookups and the input-event text witness read only the latest Assistant step, avoiding repeated whole-history text and accessibility scans. The input witness is installed before Send, and draft typing starts as soon as the first marker is visible, without an extra pre-input animation-frame wait. The actual first input event must observe an unfinished reply; completion waits for the new rendered turn-tail after Host settlement. After measurement, a trusted keystroke after DONE must fail the same overlap assertion. Open uses a standard-hosted expectation of 700 ms with 1.25× headroom (875 ms); other endpoint overhead budgets are unchanged. Heap after forced GC and DOM counts are diagnostics, not leak budgets. -The fixture contains mixed-language prompts, prose, reasoning, 20 code fences, and 40 synthetic tool results. Every historical Assistant includes a compact stream built by the production accumulator from matching reasoning, text, tool arguments, usage, and finish chunks. No model, tool, external network, recorded Session, or private Harness home supplies its content. Streaming uses 120 text deltas at 8 ms replay pacing through the real composer, agent loop, transport, and persistence. +The fixture contains mixed-language prompts, prose, reasoning, 20 code fences, and 40 synthetic tool results. Every historical Assistant includes a compact stream built by the production accumulator from matching reasoning, text, tool arguments, usage, and finish chunks. No model, tool, external network, recorded Session, or private Harness home supplies its content. Streaming uses 120 text deltas at 16 ms replay pacing through the real composer, agent loop, transport, and persistence. The [decision record](../../.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md) owns calibration, exclusions, and alternatives. The larger [manual diagnostic](../../apps/web/tests/complex-history.perf.ts) remains separate. diff --git a/benchmarks/long-session-browser/README.zh.md b/benchmarks/long-session-browser/README.zh.md index 6b35fd570f..dee8b9bad0 100644 --- a/benchmarks/long-session-browser/README.zh.md +++ b/benchmarks/long-session-browser/README.zh.md @@ -10,8 +10,8 @@ ## 测量 -三个全新浏览器进程与 scaffold 环境产生原始样本及中位数判定。打开和分页在预期对话状态出现且经过两次动画帧后结束；这包含一次渲染机会，而非硬件显示时间戳。分页报告每一页，并对各样本最慢分页时间的中位数执行预算检查。流式报告首段可见回复、真实草稿键入、完整回复壁钟时间和 Chromium 主线程任务时间。发送控件查找限定在 composer seat；回复标记查找与输入事件文本证据仅读取最新 Assistant step，避免重复扫描全部历史文本与无障碍属性。实际首个输入事件必须观察到未完成的回复；完成测量在 Host 结算后等待新 turn-tail 渲染。强制 GC 后的 heap 与 DOM 数量仅供诊断，不作为泄漏预算。 +三个全新浏览器进程与 scaffold 环境产生原始样本及中位数判定。打开和分页在预期对话状态出现且经过两次动画帧后结束；这包含一次渲染机会，而非硬件显示时间戳。分页报告每一页，并对各样本最慢分页时间的中位数执行预算检查。流式报告首段可见回复、真实草稿键入、完整回复壁钟时间和 Chromium 主线程任务时间。发送控件查找限定在 composer seat；回复标记查找与输入事件文本证据仅读取最新 Assistant step，避免重复扫描全部历史文本与无障碍属性。输入观察器在发送前安装，首个标记可见后立即开始草稿键入，不额外等待输入前动画帧。实际首个输入事件必须观察到未完成的回复；完成测量在 Host 结算后等待新 turn-tail 渲染。测量后，在 DONE 之后发送的真实按键必须无法通过同一个重叠断言。打开使用标准托管预期 700 ms 及 1.25× 余量（875 ms）；其他终点的额外开销预算不变。强制 GC 后的 heap 与 DOM 数量仅供诊断，不作为泄漏预算。 -fixture（测试前置数据）包含混合语言提示、正文、推理、20 个代码块和 40 个合成工具结果。每条历史 Assistant 都含紧凑 stream，由生产 accumulator 从匹配的推理、文本、工具参数、usage 和 finish chunk 构建。其内容不来自模型、工具、外部网络、录制 Session 或私有 Harness 主目录。流式回复以 8 ms 重放间隔发送 120 个文本 delta，经过真实输入框、agent loop（智能体循环）、传输与持久化。 +fixture（测试前置数据）包含混合语言提示、正文、推理、20 个代码块和 40 个合成工具结果。每条历史 Assistant 都含紧凑 stream，由生产 accumulator 从匹配的推理、文本、工具参数、usage 和 finish chunk 构建。其内容不来自模型、工具、外部网络、录制 Session 或私有 Harness 主目录。流式回复以 16 ms 重放间隔发送 120 个文本 delta，经过真实输入框、agent loop（智能体循环）、传输与持久化。 [决策记录](../../.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md)拥有校准、排除项与替代方案。更大规模的[手动诊断](../../apps/web/tests/complex-history.perf.ts)保持独立。 diff --git a/benchmarks/long-session-browser/long-session.bench.ts b/benchmarks/long-session-browser/long-session.bench.ts index 783ceb5725..ffaf73a056 100644 --- a/benchmarks/long-session-browser/long-session.bench.ts +++ b/benchmarks/long-session-browser/long-session.bench.ts @@ -3,16 +3,18 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { performance } from 'node:perf_hooks' -import { chromium, type Page, type CDPSession } from 'playwright' +import { chromium, type Page, type CDPSession, type Locator } from 'playwright' import { expect, it } from 'vitest' import { launchWebScaffold, seedSession, watchConsole, webSnapshotMode } from '../../apps/web/tests/scaffold.ts' import { newEnglishPage } from '../../apps/web/tests/support.ts' -import { ciTimeBudget } from '../support/calibration.ts' +import { ciTimeBudget, PERFORMANCE_BUDGET_HEADROOM } from '../support/calibration.ts' import { HISTORY_TURNS, SESSION_ID, FIRST, DONE, DELTAS, PACE_MS, syntheticHistory, syntheticReply } from './synthetic-history.ts' const SAMPLES = 3 const TAIL = '[data-chat-flow-key^="9:turn-tail"]' const REFERENCE = { open: 200, page: 260, trajectory: 160, first: 1100, streamTask: 1800, input: 500, streamWall: 1000 } +const EXPECTED_OPEN_CI_MS = 700 +const OPEN_BUDGET_MS = Math.ceil(EXPECTED_OPEN_CI_MS * PERFORMANCE_BUDGET_HEADROOM) const REPLAY_DURATION_MS = (DELTAS + 4) * PACE_MS async function painted(page: Page): Promise { @@ -38,6 +40,36 @@ function median(values: number[]): number { return values.toSorted((a, b) => a - b)[Math.floor(values.length / 2)]! } +function expectEndpointWithinBudget(value: number, budget: number): void { + expect(value).toBeLessThanOrEqual(budget) +} + +function expectInputOverlap(value: boolean): void { + expect(value).toBe(true) +} + +async function watchInputOverlap(composer: Locator): Promise { + await composer.evaluate((element, markers) => { + element.removeAttribute('data-benchmark-input-witness') + element.removeAttribute('data-benchmark-input-overlap') + element.addEventListener('input', (event) => { + const transcript = Array.from(document.querySelectorAll('[data-chat-flow-kind="assistant-step"]')).at(-1)?.textContent ?? '' + element.setAttribute('data-benchmark-input-overlap', String(event.isTrusted && transcript.includes(markers.first) && !transcript.includes(markers.done))) + element.setAttribute('data-benchmark-input-witness', JSON.stringify({ trusted: event.isTrusted, first: transcript.includes(markers.first), done: transcript.includes(markers.done) })) + }, { once: true }) + }, { first: FIRST, done: DONE }) +} + +it('accepts recorded hosted open samples and rejects slower endpoints', () => { + for (const value of [681.276514, 541.051233]) { + expect(() => expectEndpointWithinBudget(value, ciTimeBudget(REFERENCE.open))).toThrow() + expectEndpointWithinBudget(value, OPEN_BUDGET_MS) + } + expect(OPEN_BUDGET_MS).toBe(875) + expect(() => expectEndpointWithinBudget(OPEN_BUDGET_MS + 1, OPEN_BUDGET_MS)).toThrow() + expect(() => expectEndpointWithinBudget(2000, OPEN_BUDGET_MS)).toThrow() +}) + it('opens, pages, navigates and streams into a 240-turn browser history', async () => { if (webSnapshotMode() !== 'replay') throw new Error('browser benchmarks require keyless replay mode') const samples: { open: number; page: number; trajectory: number; first: number; streamTask: number; streamWall: number; input: number; inputOverlapped: boolean; heapMb: number; nodes: number }[] = [] @@ -97,18 +129,12 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async () => ({ ok: true as const }), (error: unknown) => ({ ok: false as const, error }), ) + await watchInputOverlap(composer) const started = performance.now() await page.locator('[data-composer-seat]').getByRole('button', { name: 'Send message', exact: true }).click() const reply = page.locator('[data-chat-flow-kind="assistant-step"]').last() await reply.getByText(FIRST, { exact: false }).last().waitFor() - await painted(page) const first = performance.now() - started - await composer.evaluate((element, markers) => { - element.addEventListener('input', (event) => { - const transcript = Array.from(document.querySelectorAll('[data-chat-flow-kind="assistant-step"]')).at(-1)?.textContent ?? '' - element.setAttribute('data-benchmark-input-overlap', String(event.isTrusted && transcript.includes(markers.first) && !transcript.includes(markers.done))) - }, { once: true }) - }, { first: FIRST, done: DONE }) // Observe the actual trusted input event, not state before asynchronous click/typing. const input = await measure(page, async () => { await composer.click() @@ -116,7 +142,8 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async await expect.poll(() => composer.textContent()).toBe('next synthetic question') }) const inputOverlapped = await composer.getAttribute('data-benchmark-input-overlap') === 'true' - expect(inputOverlapped).toBe(true) + console.log(JSON.stringify({ benchmark: 'long-session-browser/input', sample, first, input, witness: await composer.getAttribute('data-benchmark-input-witness') })) + expectInputOverlap(inputOverlapped) await reply.getByText(DONE, { exact: false }).last().waitFor() const settlement = await settled if (!settlement.ok) throw settlement.error @@ -130,6 +157,12 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async if (heap === undefined) throw new Error('Chromium heap metric missing') samples.push({ open, page: Math.max(...pages), trajectory, first, streamTask, streamWall, input, inputOverlapped, heapMb: heap.value / 1048576, nodes: await page.locator('*').count() }) console.log(JSON.stringify({ benchmark: 'long-session-browser/sample', sample, initialTurns, pages, ...samples.at(-1) })) + await watchInputOverlap(composer) + await composer.click() + await page.keyboard.type('!') + const lateInputOverlapped = await composer.getAttribute('data-benchmark-input-overlap') === 'true' + expect(await composer.getAttribute('data-benchmark-input-witness')).toBe(JSON.stringify({ trusted: true, first: true, done: true })) + expect(() => expectInputOverlap(lateInputOverlapped)).toThrow() expect(consoleWatch.pageErrors).toEqual([]) expect(consoleWatch.warnings).toEqual([]) } catch (error) { failures.push(error) } finally { @@ -144,7 +177,7 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async if (failures.length > 0) throw new AggregateError(failures, 'browser benchmark failed') } const aggregate = Object.fromEntries(Object.keys(REFERENCE).map(key => [key, median(samples.map(sample => sample[key as keyof typeof REFERENCE]))])) - const budgets = Object.fromEntries(Object.entries(REFERENCE).map(([key, value]) => [key, ciTimeBudget(value) + (key === 'streamWall' ? REPLAY_DURATION_MS : 0)])) - console.log(JSON.stringify({ benchmark: 'long-session-browser/median', turns: HISTORY_TURNS, deltas: DELTAS, paceMs: PACE_MS, samples, aggregate, referenceMs: REFERENCE, budgets })) - for (const [key, value] of Object.entries(aggregate)) expect.soft(value, key).toBeLessThanOrEqual(budgets[key]!) + const budgets = Object.fromEntries(Object.entries(REFERENCE).map(([key, value]) => [key, key === 'open' ? OPEN_BUDGET_MS : ciTimeBudget(value) + (key === 'streamWall' ? REPLAY_DURATION_MS : 0)])) + console.log(JSON.stringify({ benchmark: 'long-session-browser/median', turns: HISTORY_TURNS, deltas: DELTAS, paceMs: PACE_MS, samples, aggregate, referenceMs: REFERENCE, expectedOpenCiMs: EXPECTED_OPEN_CI_MS, budgets })) + for (const [key, value] of Object.entries(aggregate)) expectEndpointWithinBudget(value, budgets[key]!) }) diff --git a/benchmarks/long-session-browser/synthetic-history.ts b/benchmarks/long-session-browser/synthetic-history.ts index f6f20a3ba8..106c9950a3 100644 --- a/benchmarks/long-session-browser/synthetic-history.ts +++ b/benchmarks/long-session-browser/synthetic-history.ts @@ -17,7 +17,7 @@ export const DONE = 'SYNTHETIC_REPLY_DONE' /** Paced text chunks per continuation. */ export const DELTAS = 120 /** Replay delay per stream chunk, in milliseconds. */ -export const PACE_MS = 8 +export const PACE_MS = 16 /** Create mixed prose, code, reasoning and tool history without reading user data. * @returns Current Session JSONL accepted by the shared Web seeder. From 1a3af89a8d0acf555559db3142cb8f7b253e0585 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 21:05:14 +0800 Subject: [PATCH 20/22] fix(benchmarks): calibrate hosted paging and trajectory endpoints --- ...-06-frontend-performance-budgets.i18n.yaml | 4 +-- ...2026-09-06-frontend-performance-budgets.md | 4 ++- ...6-09-06-frontend-performance-budgets.zh.md | 4 ++- .../long-session-browser/README.i18n.yaml | 4 +-- benchmarks/long-session-browser/README.md | 2 +- benchmarks/long-session-browser/README.zh.md | 2 +- .../long-session.bench.ts | 25 +++++++++++++++++-- 7 files changed, 35 insertions(+), 10 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml index 7a0d2ca2e6..ada28c249b 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md -2026-09-06-frontend-performance-budgets.md: de0cfd5bf03a2b2e2b69fac9dba451e090804fe5 -2026-09-06-frontend-performance-budgets.zh.md: 9df105acdbd28f6cdd4aaa14ad98d847027f4cf4 +2026-09-06-frontend-performance-budgets.md: de97e902ed7d51837fbb387081206ad5eb1bcce2 +2026-09-06-frontend-performance-budgets.zh.md: 15ba01813cf76252a5aaa1bfe509b7b34fcc49c7 diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md index de0cfd5bf0..de97e902ed 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md @@ -22,7 +22,7 @@ Reconnect uses three fresh compiled plain-Node children. Each creates a 100,000- ## Calibration -Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149.0.7827.55, at product revision `925e012340`, establish the baseline below. An isolated repeat follows a complete workflow smoke. Each browser sample reports raw endpoint values and every page; the paging verdict uses the median of the sample maxima. Reconnect reports all child measurements. The following historical reference table uses 8 ms replay pacing and includes a two-frame wait in first-reply timing. Standard-hosted open and reconnect expectations are recorded separately below; other source reference constants retain these allowances. The bounded-observer 261.60 ms paging median exceeds its 260 ms reference allowance but remains below its 650 ms CI limit; the shared 2× time scale and 1.25× variance allowance produce CI limits. Memory uses only variance allowance. The shared scale originates in Node CI calibration. Both actual x64 browser runs below pass the fixed budgets on unchanged benchmark code; this supplies repeated-run evidence for these runners, not a universal browser speed ratio. +Three-sample medians on the arm64 reference machine, Node 24.19 and Chromium 149.0.7827.55, at product revision `925e012340`, establish the baseline below. An isolated repeat follows a complete workflow smoke. Each browser sample reports raw endpoint values and every page; the paging verdict uses the median of the sample maxima. Reconnect reports all child measurements. The following historical reference table uses 8 ms replay pacing and includes a two-frame wait in first-reply timing. Standard-hosted open, paging, Trajectory, and reconnect expectations are recorded separately below; other source reference constants retain these allowances. The bounded-observer 261.60 ms paging median exceeds its 260 ms reference allowance but remains below its 650 ms CI limit; the shared 2× time scale and 1.25× variance allowance produce CI limits. Memory uses only variance allowance. The shared scale originates in Node CI calibration. Both actual x64 browser runs below pass the fixed budgets on unchanged benchmark code; this supplies repeated-run evidence for these runners, not a universal browser speed ratio. | Endpoint | Measured median | Reference allowance | Historical CI limit | |---|---:|---:|---:| @@ -62,6 +62,8 @@ All six browser samples report `inputOverlapped: true` and finish after the 241s A local diagnostic with temporary 3× Chromium CPU throttling reproduces the overlap failure: the first marker becomes visible at 1321 ms, two animation frames finish at 1370 ms, and the composer click finishes at 1660 ms; the actual input is trusted but already sees DONE. Removing the frame wait and installing the witness before Send still leaves a run with first visibility at 1415 ms and click completion at 1726 ms, after the original 992 ms scripted stream. The fixed 16 ms cadence keeps the same 120 deltas and payload, providing 1984 ms of scripted pacing for this workload. Only that pacing term changes in the complete-wall allowance (4484 ms); input, first-reply, and main-thread overhead allowances remain unchanged. With the same diagnostic slowdown, three 16 ms samples reach first visibility at 1307/1479/1599 ms and accept trusted input before DONE; their post-DONE controls reject it. The diagnostic is not a CPU-ratio calibration. Each measured sample still requires trusted input while FIRST is present and DONE absent; a post-measurement trusted key after DONE must fail that same assertion. Host settlement and the 241st rendered turn-tail remain completion witnesses. +[Run 34034524861, job 101490135303](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34034524861/job/101490135303) records three complete browser samples with trusted input overlap and passing post-DONE rejection controls. Slowest-page samples are 843.941625/672.834329/684.461818 ms (median 684.461818); first-Trajectory samples are 605.788061/367.754027/485.931656 ms (median 485.931656). Their endpoint-specific hosted expectations are 700 and 500 ms, with the same 1.25× headroom producing 875 and 625 ms limits. Recorded-median controls reject the historical 650/400 ms limits, accept these hosted limits, and reject one millisecond above each limit through the measured verdict's assertion. Open, first reply, main-thread task, input, and complete-wall medians are 713.910/1486.206/2806.415/947.398/2986.983 ms; their limits remain unchanged. This run supplies calibration data, not a passing benchmark verdict; a complete hosted repeat remains required. + ## Alternatives considered **Use the Node fold as paint evidence.** Rejected because it never performs DOM mutation, layout, or browser scheduling. The focused reconnect case likewise makes no GUI speed claim. diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md index 9df105acdb..15ba01813c 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md @@ -22,7 +22,7 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 ## 校准 -在 arm64 参考机器、Node 24.19、Chromium 149.0.7827.55 和产品版本 `925e012340` 上，三个样本的中位数建立下表基线。完整工作流 smoke 后执行一次隔离重复测量。每个浏览器样本报告原始终点数据和每一页；分页判定使用各样本最大值的中位数。重连报告全部子进程测量。下列历史参考表使用 8 ms 重放节奏，首段回复计时包含两帧等待。标准托管打开和重连预期在下文单独记录；其他源码参考常量保留这些额度。受限观察器的分页中位数 261.60 ms 超过 260 ms 参考额度，但仍低于 650 ms CI 限制；共享的 2× 时间倍率和 1.25× 方差余量产生 CI 限制。内存仅使用方差余量。共享倍率源自 Node CI 校准。下述两次实际 x64 浏览器运行在基准代码不变的情况下均通过固定预算；这提供这些 runner 的重复运行证据，而非普遍适用的浏览器速度比。 +在 arm64 参考机器、Node 24.19、Chromium 149.0.7827.55 和产品版本 `925e012340` 上，三个样本的中位数建立下表基线。完整工作流 smoke 后执行一次隔离重复测量。每个浏览器样本报告原始终点数据和每一页；分页判定使用各样本最大值的中位数。重连报告全部子进程测量。下列历史参考表使用 8 ms 重放节奏，首段回复计时包含两帧等待。标准托管打开、分页、Trajectory 和重连预期在下文单独记录；其他源码参考常量保留这些额度。受限观察器的分页中位数 261.60 ms 超过 260 ms 参考额度，但仍低于 650 ms CI 限制；共享的 2× 时间倍率和 1.25× 方差余量产生 CI 限制。内存仅使用方差余量。共享倍率源自 Node CI 校准。下述两次实际 x64 浏览器运行在基准代码不变的情况下均通过固定预算；这提供这些 runner 的重复运行证据，而非普遍适用的浏览器速度比。 | 终点 | 实测中位数 | 参考额度 | 历史 CI 限制 | |---|---:|---:|---:| @@ -62,6 +62,8 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 临时使用 3× Chromium CPU 降速的本地诊断复现重叠失败：首个标记在 1321 ms 可见，两次动画帧在 1370 ms 结束，输入框点击在 1660 ms 完成；实际输入是真实事件，但已看到 DONE。移除帧等待并在发送前安装观察器后，一次运行仍在 1415 ms 才看到首个标记，点击在 1726 ms 完成，晚于原先 992 ms 的脚本流。固定 16 ms 节奏保留相同的 120 个 delta 和负载，为该工作负载提供 1984 ms 脚本节奏。完整壁钟额度仅改变该节奏项（4484 ms）；输入、首段回复及主线程额外开销额度不变。在相同诊断降速下，三个 16 ms 样本在 1307/1479/1599 ms 达到首段可见状态，并接受 DONE 之前的真实输入；其 DONE 之后的对照拒绝该输入。该诊断不是 CPU 比率校准。每个测量样本仍要求真实输入发生时 FIRST 存在且 DONE 不存在；测量后在 DONE 之后发送的真实按键必须无法通过同一个断言。Host 结算和第 241 个已渲染 turn-tail 仍是完成证据。 +[运行 34034524861，job 101490135303](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34034524861/job/101490135303) 记录三个完整浏览器样本，均具有真实输入重叠，并通过 DONE 之后的拒绝对照。最慢分页样本为 843.941625/672.834329/684.461818 ms（中位数 684.461818）；首次 Trajectory 样本为 605.788061/367.754027/485.931656 ms（中位数 485.931656）。两者的终点专属托管预期分别为 700 和 500 ms，相同的 1.25× 余量产生 875 和 625 ms 上限。记录中位数对照拒绝历史 650/400 ms 上限，接受这些托管上限，并通过测量判定所用断言拒绝超过各上限一毫秒的值。打开、首段回复、主线程任务、输入及完整壁钟的中位数为 713.910/1486.206/2806.415/947.398/2986.983 ms；其上限不变。该运行提供校准数据，不代表基准判定通过；仍需完整的托管重复运行。 + ## 考虑过的替代方案 **用 Node 折叠作为绘制证据。** 拒绝，因为它不执行 DOM 修改、布局或浏览器调度。聚焦重连用例同样不声称 GUI 提速。 diff --git a/benchmarks/long-session-browser/README.i18n.yaml b/benchmarks/long-session-browser/README.i18n.yaml index 0154487dc7..449255bb5c 100644 --- a/benchmarks/long-session-browser/README.i18n.yaml +++ b/benchmarks/long-session-browser/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write benchmarks/long-session-browser/README.md -README.md: 02d5555853ebf5bf9cc583c6e245e099caef9b30 -README.zh.md: dee8b9bad08a3d1f7a09d65058cad20fd5d60c33 +README.md: 0ae9478dbcfa85fbc31d82e76b9e26cc1975a33e +README.zh.md: e04d6e6bd5684180d943c766056dde98f56331e9 diff --git a/benchmarks/long-session-browser/README.md b/benchmarks/long-session-browser/README.md index 02d5555853..0ae9478dbc 100644 --- a/benchmarks/long-session-browser/README.md +++ b/benchmarks/long-session-browser/README.md @@ -10,7 +10,7 @@ This reference describes the required Chromium workflow in [long-session.bench.t ## Measurements -Three fresh browser processes and scaffold worlds produce raw samples and median verdicts. Open and paging end after the expected transcript state and two animation frames; this includes a rendering opportunity, not a hardware presentation timestamp. Paging reports every page and gates the median of each sample’s slowest page. Stream reports first visible reply, trusted draft typing, complete reply wall time, and Chromium main-thread task duration. Send lookup is scoped to the composer seat; reply-marker lookups and the input-event text witness read only the latest Assistant step, avoiding repeated whole-history text and accessibility scans. The input witness is installed before Send, and draft typing starts as soon as the first marker is visible, without an extra pre-input animation-frame wait. The actual first input event must observe an unfinished reply; completion waits for the new rendered turn-tail after Host settlement. After measurement, a trusted keystroke after DONE must fail the same overlap assertion. Open uses a standard-hosted expectation of 700 ms with 1.25× headroom (875 ms); other endpoint overhead budgets are unchanged. Heap after forced GC and DOM counts are diagnostics, not leak budgets. +Three fresh browser processes and scaffold worlds produce raw samples and median verdicts. Open and paging end after the expected transcript state and two animation frames; this includes a rendering opportunity, not a hardware presentation timestamp. Paging reports every page and gates the median of each sample’s slowest page. Stream reports first visible reply, trusted draft typing, complete reply wall time, and Chromium main-thread task duration. Send lookup is scoped to the composer seat; reply-marker lookups and the input-event text witness read only the latest Assistant step, avoiding repeated whole-history text and accessibility scans. The input witness is installed before Send, and draft typing starts as soon as the first marker is visible, without an extra pre-input animation-frame wait. The actual first input event must observe an unfinished reply; completion waits for the new rendered turn-tail after Host settlement. After measurement, a trusted keystroke after DONE must fail the same overlap assertion. Open and the slowest older page each use a standard-hosted expectation of 700 ms; first Trajectory uses 500 ms. Shared 1.25× headroom gives limits of 875/875/625 ms respectively; stream endpoint overhead budgets are unchanged. Heap after forced GC and DOM counts are diagnostics, not leak budgets. The fixture contains mixed-language prompts, prose, reasoning, 20 code fences, and 40 synthetic tool results. Every historical Assistant includes a compact stream built by the production accumulator from matching reasoning, text, tool arguments, usage, and finish chunks. No model, tool, external network, recorded Session, or private Harness home supplies its content. Streaming uses 120 text deltas at 16 ms replay pacing through the real composer, agent loop, transport, and persistence. diff --git a/benchmarks/long-session-browser/README.zh.md b/benchmarks/long-session-browser/README.zh.md index dee8b9bad0..e04d6e6bd5 100644 --- a/benchmarks/long-session-browser/README.zh.md +++ b/benchmarks/long-session-browser/README.zh.md @@ -10,7 +10,7 @@ ## 测量 -三个全新浏览器进程与 scaffold 环境产生原始样本及中位数判定。打开和分页在预期对话状态出现且经过两次动画帧后结束；这包含一次渲染机会，而非硬件显示时间戳。分页报告每一页，并对各样本最慢分页时间的中位数执行预算检查。流式报告首段可见回复、真实草稿键入、完整回复壁钟时间和 Chromium 主线程任务时间。发送控件查找限定在 composer seat；回复标记查找与输入事件文本证据仅读取最新 Assistant step，避免重复扫描全部历史文本与无障碍属性。输入观察器在发送前安装，首个标记可见后立即开始草稿键入，不额外等待输入前动画帧。实际首个输入事件必须观察到未完成的回复；完成测量在 Host 结算后等待新 turn-tail 渲染。测量后，在 DONE 之后发送的真实按键必须无法通过同一个重叠断言。打开使用标准托管预期 700 ms 及 1.25× 余量（875 ms）；其他终点的额外开销预算不变。强制 GC 后的 heap 与 DOM 数量仅供诊断，不作为泄漏预算。 +三个全新浏览器进程与 scaffold 环境产生原始样本及中位数判定。打开和分页在预期对话状态出现且经过两次动画帧后结束；这包含一次渲染机会，而非硬件显示时间戳。分页报告每一页，并对各样本最慢分页时间的中位数执行预算检查。流式报告首段可见回复、真实草稿键入、完整回复壁钟时间和 Chromium 主线程任务时间。发送控件查找限定在 composer seat；回复标记查找与输入事件文本证据仅读取最新 Assistant step，避免重复扫描全部历史文本与无障碍属性。输入观察器在发送前安装，首个标记可见后立即开始草稿键入，不额外等待输入前动画帧。实际首个输入事件必须观察到未完成的回复；完成测量在 Host 结算后等待新 turn-tail 渲染。测量后，在 DONE 之后发送的真实按键必须无法通过同一个重叠断言。打开和最慢更早分页各使用标准托管预期 700 ms；首次 Trajectory 使用 500 ms。共享的 1.25× 余量分别产生 875/875/625 ms 上限；流式终点的额外开销预算不变。强制 GC 后的 heap 与 DOM 数量仅供诊断，不作为泄漏预算。 fixture（测试前置数据）包含混合语言提示、正文、推理、20 个代码块和 40 个合成工具结果。每条历史 Assistant 都含紧凑 stream，由生产 accumulator 从匹配的推理、文本、工具参数、usage 和 finish chunk 构建。其内容不来自模型、工具、外部网络、录制 Session 或私有 Harness 主目录。流式回复以 16 ms 重放间隔发送 120 个文本 delta，经过真实输入框、agent loop（智能体循环）、传输与持久化。 diff --git a/benchmarks/long-session-browser/long-session.bench.ts b/benchmarks/long-session-browser/long-session.bench.ts index ffaf73a056..cd1031a8f2 100644 --- a/benchmarks/long-session-browser/long-session.bench.ts +++ b/benchmarks/long-session-browser/long-session.bench.ts @@ -14,7 +14,11 @@ const SAMPLES = 3 const TAIL = '[data-chat-flow-key^="9:turn-tail"]' const REFERENCE = { open: 200, page: 260, trajectory: 160, first: 1100, streamTask: 1800, input: 500, streamWall: 1000 } const EXPECTED_OPEN_CI_MS = 700 +const EXPECTED_PAGE_CI_MS = 700 +const EXPECTED_TRAJECTORY_CI_MS = 500 const OPEN_BUDGET_MS = Math.ceil(EXPECTED_OPEN_CI_MS * PERFORMANCE_BUDGET_HEADROOM) +const PAGE_BUDGET_MS = Math.ceil(EXPECTED_PAGE_CI_MS * PERFORMANCE_BUDGET_HEADROOM) +const TRAJECTORY_BUDGET_MS = Math.ceil(EXPECTED_TRAJECTORY_CI_MS * PERFORMANCE_BUDGET_HEADROOM) const REPLAY_DURATION_MS = (DELTAS + 4) * PACE_MS async function painted(page: Page): Promise { @@ -70,6 +74,20 @@ it('accepts recorded hosted open samples and rejects slower endpoints', () => { expect(() => expectEndpointWithinBudget(2000, OPEN_BUDGET_MS)).toThrow() }) +it('accepts recorded hosted paging and Trajectory medians and rejects slower endpoints', () => { + const endpoints = [ + { samples: [843.941625, 672.834329, 684.461818], reference: REFERENCE.page, budget: PAGE_BUDGET_MS, expectedBudget: 875 }, + { samples: [605.788061, 367.754027, 485.931656], reference: REFERENCE.trajectory, budget: TRAJECTORY_BUDGET_MS, expectedBudget: 625 }, + ] + for (const { samples, reference, budget, expectedBudget } of endpoints) { + const value = median(samples) + expect(() => expectEndpointWithinBudget(value, ciTimeBudget(reference))).toThrow() + expectEndpointWithinBudget(value, budget) + expect(budget).toBe(expectedBudget) + expect(() => expectEndpointWithinBudget(budget + 1, budget)).toThrow() + } +}) + it('opens, pages, navigates and streams into a 240-turn browser history', async () => { if (webSnapshotMode() !== 'replay') throw new Error('browser benchmarks require keyless replay mode') const samples: { open: number; page: number; trajectory: number; first: number; streamTask: number; streamWall: number; input: number; inputOverlapped: boolean; heapMb: number; nodes: number }[] = [] @@ -177,7 +195,10 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async if (failures.length > 0) throw new AggregateError(failures, 'browser benchmark failed') } const aggregate = Object.fromEntries(Object.keys(REFERENCE).map(key => [key, median(samples.map(sample => sample[key as keyof typeof REFERENCE]))])) - const budgets = Object.fromEntries(Object.entries(REFERENCE).map(([key, value]) => [key, key === 'open' ? OPEN_BUDGET_MS : ciTimeBudget(value) + (key === 'streamWall' ? REPLAY_DURATION_MS : 0)])) - console.log(JSON.stringify({ benchmark: 'long-session-browser/median', turns: HISTORY_TURNS, deltas: DELTAS, paceMs: PACE_MS, samples, aggregate, referenceMs: REFERENCE, expectedOpenCiMs: EXPECTED_OPEN_CI_MS, budgets })) + const budgets: Record = { + ...Object.fromEntries(Object.entries(REFERENCE).map(([key, value]) => [key, ciTimeBudget(value) + (key === 'streamWall' ? REPLAY_DURATION_MS : 0)])), + open: OPEN_BUDGET_MS, page: PAGE_BUDGET_MS, trajectory: TRAJECTORY_BUDGET_MS, + } + console.log(JSON.stringify({ benchmark: 'long-session-browser/median', turns: HISTORY_TURNS, deltas: DELTAS, paceMs: PACE_MS, samples, aggregate, referenceMs: REFERENCE, expectedOpenCiMs: EXPECTED_OPEN_CI_MS, expectedPageCiMs: EXPECTED_PAGE_CI_MS, expectedTrajectoryCiMs: EXPECTED_TRAJECTORY_CI_MS, budgets })) for (const [key, value] of Object.entries(aggregate)) expectEndpointWithinBudget(value, budgets[key]!) }) From cbdf32447e8986f41d7db9978bcb71ea086b2a6b Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 6 Sep 2026 21:35:34 +0800 Subject: [PATCH 21/22] fix(benchmarks): calibrate repeated hosted browser open median --- .../2026-09-06-frontend-performance-budgets.i18n.yaml | 4 ++-- .../testing/2026-09-06-frontend-performance-budgets.md | 6 ++++-- .../2026-09-06-frontend-performance-budgets.zh.md | 6 ++++-- benchmarks/long-session-browser/README.i18n.yaml | 4 ++-- benchmarks/long-session-browser/README.md | 2 +- benchmarks/long-session-browser/README.zh.md | 2 +- benchmarks/long-session-browser/long-session.bench.ts | 9 +++++++-- 7 files changed, 21 insertions(+), 12 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml index ada28c249b..1e2848dfde 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md -2026-09-06-frontend-performance-budgets.md: de97e902ed7d51837fbb387081206ad5eb1bcce2 -2026-09-06-frontend-performance-budgets.zh.md: 15ba01813cf76252a5aaa1bfe509b7b34fcc49c7 +2026-09-06-frontend-performance-budgets.md: 7dc7af97d8bb65c17109ab675c250085c9c5831c +2026-09-06-frontend-performance-budgets.zh.md: 9935e382ec4c3c3ede762b23339f14c014b67c4e diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md index de97e902ed..7dc7af97d8 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.md @@ -58,11 +58,13 @@ All six browser samples report `inputOverlapped: true` and finish after the 241s ### Standard hosted expectations and input scheduling -[Run 34033336246, job 101487216170](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336246/job/101487216170) on standard hosted Ubuntu with two CPUs records reconnect replacements of 46.574411, 46.067910, and 44.193704 ms, with 23.028 MiB retained heap. The endpoint-specific expectation is 50 ms; the existing 1.25× headroom gives a 63 ms integer ceiling. The 30 MiB memory budget and shared machine factor remain unchanged. Browser open records 681.276514 and 541.051233 ms before the third sample fails input overlap; both exceed the historical 500 ms limit. Its hosted expectation is 700 ms, giving an 875 ms ceiling. Deterministic controls pass these recorded values and reject values above the new ceilings through the same assertions as the measured verdicts. Complete repeated hosted verdicts remain required; the two open values are not a three-sample median. +[Run 34033336246, job 101487216170](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336246/job/101487216170) on standard hosted Ubuntu with two CPUs records reconnect replacements of 46.574411, 46.067910, and 44.193704 ms, with 23.028 MiB retained heap. The endpoint-specific expectation is 50 ms; the existing 1.25× headroom gives a 63 ms integer ceiling. The 30 MiB memory budget and shared machine factor remain unchanged. Browser open records 681.276514 and 541.051233 ms before the third sample fails input overlap; both exceed the historical 500 ms limit. Repeated hosted open measurements below set its expectation and ceiling. Deterministic controls pass these recorded values and reject values above the new ceilings through the same assertions as the measured verdicts. Complete repeated hosted verdicts remain required; the two open values are not a three-sample median. A local diagnostic with temporary 3× Chromium CPU throttling reproduces the overlap failure: the first marker becomes visible at 1321 ms, two animation frames finish at 1370 ms, and the composer click finishes at 1660 ms; the actual input is trusted but already sees DONE. Removing the frame wait and installing the witness before Send still leaves a run with first visibility at 1415 ms and click completion at 1726 ms, after the original 992 ms scripted stream. The fixed 16 ms cadence keeps the same 120 deltas and payload, providing 1984 ms of scripted pacing for this workload. Only that pacing term changes in the complete-wall allowance (4484 ms); input, first-reply, and main-thread overhead allowances remain unchanged. With the same diagnostic slowdown, three 16 ms samples reach first visibility at 1307/1479/1599 ms and accept trusted input before DONE; their post-DONE controls reject it. The diagnostic is not a CPU-ratio calibration. Each measured sample still requires trusted input while FIRST is present and DONE absent; a post-measurement trusted key after DONE must fail that same assertion. Host settlement and the 241st rendered turn-tail remain completion witnesses. -[Run 34034524861, job 101490135303](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34034524861/job/101490135303) records three complete browser samples with trusted input overlap and passing post-DONE rejection controls. Slowest-page samples are 843.941625/672.834329/684.461818 ms (median 684.461818); first-Trajectory samples are 605.788061/367.754027/485.931656 ms (median 485.931656). Their endpoint-specific hosted expectations are 700 and 500 ms, with the same 1.25× headroom producing 875 and 625 ms limits. Recorded-median controls reject the historical 650/400 ms limits, accept these hosted limits, and reject one millisecond above each limit through the measured verdict's assertion. Open, first reply, main-thread task, input, and complete-wall medians are 713.910/1486.206/2806.415/947.398/2986.983 ms; their limits remain unchanged. This run supplies calibration data, not a passing benchmark verdict; a complete hosted repeat remains required. +[Run 34034524861, job 101490135303](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34034524861/job/101490135303) records three complete browser samples with trusted input overlap and passing post-DONE rejection controls. Slowest-page samples are 843.941625/672.834329/684.461818 ms (median 684.461818); first-Trajectory samples are 605.788061/367.754027/485.931656 ms (median 485.931656). Their endpoint-specific hosted expectations are 700 and 500 ms, with the same 1.25× headroom producing 875 and 625 ms limits. Recorded-median controls reject the historical 650/400 ms limits, accept these hosted limits, and reject one millisecond above each limit through the measured verdict's assertion. Open, first reply, main-thread task, input, and complete-wall medians are 713.910/1486.206/2806.415/947.398/2986.983 ms; only the open limit is recalibrated by the repeated measurements below. This run supplies calibration data, not a passing benchmark verdict; a complete hosted repeat remains required. + +[Run 34036109842, job 101494445658](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34036109842/job/101494445658) records open samples of 875.306861/1083.683529/814.700998 ms, with a median of 875.306861 ms versus 713.909727 ms in the preceding hosted run. The endpoint-specific expectation is 900 ms, rounding up the larger repeated median rather than adding an epsilon to the 875 ms limit; unchanged 1.25× headroom gives 1125 ms. The same enforced assertion accepts the recorded median, rejects it at both historical 500 and 875 ms limits, and rejects a synthetic 1126 ms value at the current limit. All three samples retain trusted input overlap and post-DONE rejection; every other frontend median remains within its unchanged limit. This calibration does not claim a green CI run. ## Alternatives considered diff --git a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md index 15ba01813c..9935e382ec 100644 --- a/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-frontend-performance-budgets.zh.md @@ -58,11 +58,13 @@ Node 对话折叠很快，并不能证明浏览器能绘制长对话或在流式 ### 标准托管预期与输入调度 -[运行 34033336246，job 101487216170](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336246/job/101487216170) 在双 CPU 标准托管 Ubuntu 上记录重连替换时间 46.574411、46.067910 和 44.193704 ms，保留 heap 为 23.028 MiB。该终点的预期为 50 ms；现有 1.25× 余量产生向上取整后的 63 ms 上限。30 MiB 内存预算及共享机器倍率不变。浏览器打开记录 681.276514 和 541.051233 ms，第三个样本因输入重叠失败而中止；两个值均超过历史 500 ms 上限。其托管预期为 700 ms，上限为 875 ms。确定性对照通过这些记录值，并使用与测量判定相同的断言拒绝超过新上限的值。仍需完整的托管重复运行判定；这两个打开值不是三样本中位数。 +[运行 34033336246，job 101487216170](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34033336246/job/101487216170) 在双 CPU 标准托管 Ubuntu 上记录重连替换时间 46.574411、46.067910 和 44.193704 ms，保留 heap 为 23.028 MiB。该终点的预期为 50 ms；现有 1.25× 余量产生向上取整后的 63 ms 上限。30 MiB 内存预算及共享机器倍率不变。浏览器打开记录 681.276514 和 541.051233 ms，第三个样本因输入重叠失败而中止；两个值均超过历史 500 ms 上限。下文的托管打开重复测量决定其预期与上限。确定性对照通过这些记录值，并使用与测量判定相同的断言拒绝超过新上限的值。仍需完整的托管重复运行判定；这两个打开值不是三样本中位数。 临时使用 3× Chromium CPU 降速的本地诊断复现重叠失败：首个标记在 1321 ms 可见，两次动画帧在 1370 ms 结束，输入框点击在 1660 ms 完成；实际输入是真实事件，但已看到 DONE。移除帧等待并在发送前安装观察器后，一次运行仍在 1415 ms 才看到首个标记，点击在 1726 ms 完成，晚于原先 992 ms 的脚本流。固定 16 ms 节奏保留相同的 120 个 delta 和负载，为该工作负载提供 1984 ms 脚本节奏。完整壁钟额度仅改变该节奏项（4484 ms）；输入、首段回复及主线程额外开销额度不变。在相同诊断降速下，三个 16 ms 样本在 1307/1479/1599 ms 达到首段可见状态，并接受 DONE 之前的真实输入；其 DONE 之后的对照拒绝该输入。该诊断不是 CPU 比率校准。每个测量样本仍要求真实输入发生时 FIRST 存在且 DONE 不存在；测量后在 DONE 之后发送的真实按键必须无法通过同一个断言。Host 结算和第 241 个已渲染 turn-tail 仍是完成证据。 -[运行 34034524861，job 101490135303](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34034524861/job/101490135303) 记录三个完整浏览器样本，均具有真实输入重叠，并通过 DONE 之后的拒绝对照。最慢分页样本为 843.941625/672.834329/684.461818 ms（中位数 684.461818）；首次 Trajectory 样本为 605.788061/367.754027/485.931656 ms（中位数 485.931656）。两者的终点专属托管预期分别为 700 和 500 ms，相同的 1.25× 余量产生 875 和 625 ms 上限。记录中位数对照拒绝历史 650/400 ms 上限，接受这些托管上限，并通过测量判定所用断言拒绝超过各上限一毫秒的值。打开、首段回复、主线程任务、输入及完整壁钟的中位数为 713.910/1486.206/2806.415/947.398/2986.983 ms；其上限不变。该运行提供校准数据，不代表基准判定通过；仍需完整的托管重复运行。 +[运行 34034524861，job 101490135303](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34034524861/job/101490135303) 记录三个完整浏览器样本，均具有真实输入重叠，并通过 DONE 之后的拒绝对照。最慢分页样本为 843.941625/672.834329/684.461818 ms（中位数 684.461818）；首次 Trajectory 样本为 605.788061/367.754027/485.931656 ms（中位数 485.931656）。两者的终点专属托管预期分别为 700 和 500 ms，相同的 1.25× 余量产生 875 和 625 ms 上限。记录中位数对照拒绝历史 650/400 ms 上限，接受这些托管上限，并通过测量判定所用断言拒绝超过各上限一毫秒的值。打开、首段回复、主线程任务、输入及完整壁钟的中位数为 713.910/1486.206/2806.415/947.398/2986.983 ms；仅打开上限根据下文的重复测量重新校准。该运行提供校准数据，不代表基准判定通过；仍需完整的托管重复运行。 + +[运行 34036109842，job 101494445658](https://github.com/deepseek-harness/deepseek-harness/actions/runs/34036109842/job/101494445658) 记录打开样本 875.306861/1083.683529/814.700998 ms，中位数为 875.306861 ms，前一次托管运行的中位数为 713.909727 ms。该终点的预期为 900 ms，向上取整较大的重复测量中位数，而非向 875 ms 上限增加微量余量；不变的 1.25× 余量产生 1125 ms 上限。同一个强制断言接受记录中位数，在历史 500 和 875 ms 上限下均拒绝它，并在当前上限下拒绝合成的 1126 ms 值。三个样本均保留真实输入重叠与 DONE 之后的拒绝；其他所有前端中位数均在不变的上限内。此校准不代表 CI 运行通过。 ## 考虑过的替代方案 diff --git a/benchmarks/long-session-browser/README.i18n.yaml b/benchmarks/long-session-browser/README.i18n.yaml index 449255bb5c..dfb24532ca 100644 --- a/benchmarks/long-session-browser/README.i18n.yaml +++ b/benchmarks/long-session-browser/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write benchmarks/long-session-browser/README.md -README.md: 0ae9478dbcfa85fbc31d82e76b9e26cc1975a33e -README.zh.md: e04d6e6bd5684180d943c766056dde98f56331e9 +README.md: 6b6777687947bee42337568ad3748e37a563be87 +README.zh.md: 139bd237d45275ea165a3e90f055f8ce507c5c53 diff --git a/benchmarks/long-session-browser/README.md b/benchmarks/long-session-browser/README.md index 0ae9478dbc..6b67776879 100644 --- a/benchmarks/long-session-browser/README.md +++ b/benchmarks/long-session-browser/README.md @@ -10,7 +10,7 @@ This reference describes the required Chromium workflow in [long-session.bench.t ## Measurements -Three fresh browser processes and scaffold worlds produce raw samples and median verdicts. Open and paging end after the expected transcript state and two animation frames; this includes a rendering opportunity, not a hardware presentation timestamp. Paging reports every page and gates the median of each sample’s slowest page. Stream reports first visible reply, trusted draft typing, complete reply wall time, and Chromium main-thread task duration. Send lookup is scoped to the composer seat; reply-marker lookups and the input-event text witness read only the latest Assistant step, avoiding repeated whole-history text and accessibility scans. The input witness is installed before Send, and draft typing starts as soon as the first marker is visible, without an extra pre-input animation-frame wait. The actual first input event must observe an unfinished reply; completion waits for the new rendered turn-tail after Host settlement. After measurement, a trusted keystroke after DONE must fail the same overlap assertion. Open and the slowest older page each use a standard-hosted expectation of 700 ms; first Trajectory uses 500 ms. Shared 1.25× headroom gives limits of 875/875/625 ms respectively; stream endpoint overhead budgets are unchanged. Heap after forced GC and DOM counts are diagnostics, not leak budgets. +Three fresh browser processes and scaffold worlds produce raw samples and median verdicts. Open and paging end after the expected transcript state and two animation frames; this includes a rendering opportunity, not a hardware presentation timestamp. Paging reports every page and gates the median of each sample’s slowest page. Stream reports first visible reply, trusted draft typing, complete reply wall time, and Chromium main-thread task duration. Send lookup is scoped to the composer seat; reply-marker lookups and the input-event text witness read only the latest Assistant step, avoiding repeated whole-history text and accessibility scans. The input witness is installed before Send, and draft typing starts as soon as the first marker is visible, without an extra pre-input animation-frame wait. The actual first input event must observe an unfinished reply; completion waits for the new rendered turn-tail after Host settlement. After measurement, a trusted keystroke after DONE must fail the same overlap assertion. Open, the slowest older page, and first Trajectory use standard-hosted expectations of 900/700/500 ms. Shared 1.25× headroom gives limits of 1125/875/625 ms respectively; stream endpoint overhead budgets are unchanged. Heap after forced GC and DOM counts are diagnostics, not leak budgets. The fixture contains mixed-language prompts, prose, reasoning, 20 code fences, and 40 synthetic tool results. Every historical Assistant includes a compact stream built by the production accumulator from matching reasoning, text, tool arguments, usage, and finish chunks. No model, tool, external network, recorded Session, or private Harness home supplies its content. Streaming uses 120 text deltas at 16 ms replay pacing through the real composer, agent loop, transport, and persistence. diff --git a/benchmarks/long-session-browser/README.zh.md b/benchmarks/long-session-browser/README.zh.md index e04d6e6bd5..139bd237d4 100644 --- a/benchmarks/long-session-browser/README.zh.md +++ b/benchmarks/long-session-browser/README.zh.md @@ -10,7 +10,7 @@ ## 测量 -三个全新浏览器进程与 scaffold 环境产生原始样本及中位数判定。打开和分页在预期对话状态出现且经过两次动画帧后结束；这包含一次渲染机会，而非硬件显示时间戳。分页报告每一页，并对各样本最慢分页时间的中位数执行预算检查。流式报告首段可见回复、真实草稿键入、完整回复壁钟时间和 Chromium 主线程任务时间。发送控件查找限定在 composer seat；回复标记查找与输入事件文本证据仅读取最新 Assistant step，避免重复扫描全部历史文本与无障碍属性。输入观察器在发送前安装，首个标记可见后立即开始草稿键入，不额外等待输入前动画帧。实际首个输入事件必须观察到未完成的回复；完成测量在 Host 结算后等待新 turn-tail 渲染。测量后，在 DONE 之后发送的真实按键必须无法通过同一个重叠断言。打开和最慢更早分页各使用标准托管预期 700 ms；首次 Trajectory 使用 500 ms。共享的 1.25× 余量分别产生 875/875/625 ms 上限；流式终点的额外开销预算不变。强制 GC 后的 heap 与 DOM 数量仅供诊断，不作为泄漏预算。 +三个全新浏览器进程与 scaffold 环境产生原始样本及中位数判定。打开和分页在预期对话状态出现且经过两次动画帧后结束；这包含一次渲染机会，而非硬件显示时间戳。分页报告每一页，并对各样本最慢分页时间的中位数执行预算检查。流式报告首段可见回复、真实草稿键入、完整回复壁钟时间和 Chromium 主线程任务时间。发送控件查找限定在 composer seat；回复标记查找与输入事件文本证据仅读取最新 Assistant step，避免重复扫描全部历史文本与无障碍属性。输入观察器在发送前安装，首个标记可见后立即开始草稿键入，不额外等待输入前动画帧。实际首个输入事件必须观察到未完成的回复；完成测量在 Host 结算后等待新 turn-tail 渲染。测量后，在 DONE 之后发送的真实按键必须无法通过同一个重叠断言。打开、最慢更早分页和首次 Trajectory 使用标准托管预期 900/700/500 ms。共享的 1.25× 余量分别产生 1125/875/625 ms 上限；流式终点的额外开销预算不变。强制 GC 后的 heap 与 DOM 数量仅供诊断，不作为泄漏预算。 fixture（测试前置数据）包含混合语言提示、正文、推理、20 个代码块和 40 个合成工具结果。每条历史 Assistant 都含紧凑 stream，由生产 accumulator 从匹配的推理、文本、工具参数、usage 和 finish chunk 构建。其内容不来自模型、工具、外部网络、录制 Session 或私有 Harness 主目录。流式回复以 16 ms 重放间隔发送 120 个文本 delta，经过真实输入框、agent loop（智能体循环）、传输与持久化。 diff --git a/benchmarks/long-session-browser/long-session.bench.ts b/benchmarks/long-session-browser/long-session.bench.ts index cd1031a8f2..789ebeea9a 100644 --- a/benchmarks/long-session-browser/long-session.bench.ts +++ b/benchmarks/long-session-browser/long-session.bench.ts @@ -13,7 +13,7 @@ import { HISTORY_TURNS, SESSION_ID, FIRST, DONE, DELTAS, PACE_MS, syntheticHisto const SAMPLES = 3 const TAIL = '[data-chat-flow-key^="9:turn-tail"]' const REFERENCE = { open: 200, page: 260, trajectory: 160, first: 1100, streamTask: 1800, input: 500, streamWall: 1000 } -const EXPECTED_OPEN_CI_MS = 700 +const EXPECTED_OPEN_CI_MS = 900 const EXPECTED_PAGE_CI_MS = 700 const EXPECTED_TRAJECTORY_CI_MS = 500 const OPEN_BUDGET_MS = Math.ceil(EXPECTED_OPEN_CI_MS * PERFORMANCE_BUDGET_HEADROOM) @@ -69,7 +69,12 @@ it('accepts recorded hosted open samples and rejects slower endpoints', () => { expect(() => expectEndpointWithinBudget(value, ciTimeBudget(REFERENCE.open))).toThrow() expectEndpointWithinBudget(value, OPEN_BUDGET_MS) } - expect(OPEN_BUDGET_MS).toBe(875) + const repeatedMedian = median([875.306861, 1083.683529, 814.700998]) + expect(repeatedMedian).toBe(875.306861) + expect(() => expectEndpointWithinBudget(repeatedMedian, ciTimeBudget(REFERENCE.open))).toThrow() + expect(() => expectEndpointWithinBudget(repeatedMedian, 875)).toThrow() + expectEndpointWithinBudget(repeatedMedian, OPEN_BUDGET_MS) + expect(OPEN_BUDGET_MS).toBe(1125) expect(() => expectEndpointWithinBudget(OPEN_BUDGET_MS + 1, OPEN_BUDGET_MS)).toThrow() expect(() => expectEndpointWithinBudget(2000, OPEN_BUDGET_MS)).toThrow() }) From 9ef426d7294eb4f788744b6e750a91892d4efff4 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 7 Sep 2026 13:46:30 +0800 Subject: [PATCH 22/22] fix(chat): keep genuine near-floor scroll gestures pending --- ...ed-scroll-delivery-before-layout.i18n.yaml | 4 +-- ...07-pinned-scroll-delivery-before-layout.md | 2 +- ...pinned-scroll-delivery-before-layout.zh.md | 2 +- packages/client/ui-chat/README.i18n.yaml | 4 +-- packages/client/ui-chat/README.md | 2 +- packages/client/ui-chat/README.zh.md | 2 +- .../ui-chat/src/client/chat/ChatView.tsx | 20 +++++++---- .../ui-chat/tests/chat-view.client.spec.tsx | 34 +++++++++++++++++++ 8 files changed, 56 insertions(+), 14 deletions(-) diff --git a/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.i18n.yaml index ac6b884af9..5df3dbdf86 100644 --- a/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.i18n.yaml +++ b/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.md -2026-09-07-pinned-scroll-delivery-before-layout.md: 7a00cf653824df13272fcf0cc2baf35b298f170c -2026-09-07-pinned-scroll-delivery-before-layout.zh.md: d78c388e4e7a8f6f2fa6070149e652e0e25cc358 +2026-09-07-pinned-scroll-delivery-before-layout.md: f3b428dbe0d2fdc7cbaedf0383d2ab80ea1bfd45 +2026-09-07-pinned-scroll-delivery-before-layout.zh.md: 9b200f45535c67cfbbb75feaf3025c5a3b5c1fa2 diff --git a/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.md b/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.md index 7a00cf6538..f3b428dbe0 100644 --- a/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.md +++ b/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.md @@ -10,7 +10,7 @@ A delayed scroll sample compares positions from different layouts. While Chat is ## Decision -[ChatView](../../../../packages/client/ui-chat/src/client/chat/ChatView.tsx) samples pinned scroll deliveries synchronously through the same sample operation that clears pending work. This preserves the existing observed-top comparison for genuine reader movement and releases layout follow before further growth. Pinned samples use scroll metrics, not semantic-row geometry; moving away still disarms follow immediately. Away-reader samples remain coalesced at the existing interval or `scrollend`. +[ChatView](../../../../packages/client/ui-chat/src/client/chat/ChatView.tsx) uses the existing observed-top comparison to sample non-reader pinned scroll deliveries synchronously through the same sample operation that clears pending work. This releases layout follow before further growth. Genuine reader movement remains pending until the existing interval or `scrollend`, even within the follow threshold: growth must not erase small gestures before they accumulate into a scroll-away. Immediate pinned samples use scroll metrics, not semantic-row geometry. ## Alternatives considered diff --git a/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.zh.md b/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.zh.md index d78c388e4e..9b200f4553 100644 --- a/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.zh.md +++ b/.agents/notes/implemented/bug-fix/2026-09-07-pinned-scroll-delivery-before-layout.zh.md @@ -10,7 +10,7 @@ Status: implemented ## Decision -[ChatView](../../../../packages/client/ui-chat/src/client/chat/ChatView.tsx) 通过同一个清除待处理工作的采样操作，同步采样贴底滚动事件。该机制保留现有的已观察顶部位置比较来识别真实读者移动，并在后续增长前恢复布局跟随。贴底采样只读取滚动指标，不读取语义行几何；离底移动仍会立即关闭跟随。离底读者的采样仍合并到现有周期或 `scrollend` 时执行。 +[ChatView](../../../../packages/client/ui-chat/src/client/chat/ChatView.tsx) 使用现有的已观察顶部位置比较，通过同一个清除待处理工作的采样操作，同步采样非读者引起的贴底滚动事件。这会在后续增长前恢复布局跟随。真实读者移动即使位于跟随阈值内，也保持待处理直到现有周期或 `scrollend`：增长不能在小幅操作累积为离底滚动前将其抵消。立即执行的贴底采样只读取滚动指标，不读取语义行几何。 ## Alternatives considered diff --git a/packages/client/ui-chat/README.i18n.yaml b/packages/client/ui-chat/README.i18n.yaml index 80d26dca50..746b9fc2a6 100644 --- a/packages/client/ui-chat/README.i18n.yaml +++ b/packages/client/ui-chat/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/client/ui-chat/README.md -README.md: 34b66da24b35f8cfd4c26da9b759e7991c5cb856 -README.zh.md: a9b7c8fca3e8137950dd02cd5398c11f15344757 +README.md: fa9653f7acab47f352e59971b333c6e691f44191 +README.zh.md: f8dc1968fffc57336d6aece8fe07113d5b99893b diff --git a/packages/client/ui-chat/README.md b/packages/client/ui-chat/README.md index 34b66da24b..fa9653f7ac 100644 --- a/packages/client/ui-chat/README.md +++ b/packages/client/ui-chat/README.md @@ -46,7 +46,7 @@ Settings → General exposes a persisted `Normal` / `Compact` conversation-displ ## Scroll ownership -Chat restores semantic anchors across history prepend and renderer remounts. Pinned scroll deliveries update follow ownership immediately, before subsequent layout changes can invalidate their floor; away-reader anchor sampling remains coalesced until the sampling interval or `scrollend`. While the reader is pinned to the floor, `ResizeObserver` follows the new floor and selects the latest loaded Turn without reading row geometry. Once the reader moves away, flow-height changes preserve the top position and the reading-line geometry selects the active Turn. Turn-rail previews paint above sticky Markdown code-block banners, while the rail frame remains inside the transcript band above the composer. +Chat restores semantic anchors across history prepend and renderer remounts. Pinned scroll deliveries without reader movement update follow ownership immediately, before subsequent layout changes can invalidate their floor. Reader movement remains pending until the sampling interval or `scrollend`, even inside the follow threshold, so layout growth cannot erase small scroll gestures. While the reader is pinned to the floor, `ResizeObserver` follows the new floor and selects the latest loaded Turn without reading row geometry. Once the reader moves away, flow-height changes preserve the top position and the reading-line geometry selects the active Turn. Turn-rail previews paint above sticky Markdown code-block banners, while the rail frame remains inside the transcript band above the composer. ----- diff --git a/packages/client/ui-chat/README.zh.md b/packages/client/ui-chat/README.zh.md index a9b7c8fca3..f8dc1968ff 100644 --- a/packages/client/ui-chat/README.zh.md +++ b/packages/client/ui-chat/README.zh.md @@ -46,7 +46,7 @@ Chat 会为非空的初始请求、显式消息序列起点、真实 system 字 ## 滚动归属 -Chat 会在历史前插与 renderer 重新挂载时恢复语义锚点。贴底滚动事件会立即更新跟随归属，避免后续布局变化使其底部位置失效；离底读者的锚点采样仍合并到采样周期或 `scrollend` 时执行。读者跟随底部时，`ResizeObserver` 追随新的底部，并且无需读取行几何就选中最后一个已加载 Turn；读者离开底部后，高度变化会保持顶部位置，再由阅读线几何选择活跃 Turn。轮次导航预览位于 Markdown 代码块粘性头栏上方，而导航外框始终处于 composer 上方的 transcript 区域内。 +Chat 会在历史前插与 renderer 重新挂载时恢复语义锚点。没有读者移动的贴底滚动事件会立即更新跟随归属，避免后续布局变化使其底部位置失效。读者移动即使位于跟随阈值内，也保持待处理直到采样周期或 `scrollend`，防止布局增长抵消小幅滚动操作。读者跟随底部时，`ResizeObserver` 追随新的底部，并且无需读取行几何就选中最后一个已加载 Turn；读者离开底部后，高度变化会保持顶部位置，再由阅读线几何选择活跃 Turn。轮次导航预览位于 Markdown 代码块粘性头栏上方，而导航外框始终处于 composer 上方的 transcript 区域内。 ----- diff --git a/packages/client/ui-chat/src/client/chat/ChatView.tsx b/packages/client/ui-chat/src/client/chat/ChatView.tsx index e8b4438266..21e1f78a29 100644 --- a/packages/client/ui-chat/src/client/chat/ChatView.tsx +++ b/packages/client/ui-chat/src/client/chat/ChatView.tsx @@ -24,6 +24,11 @@ function scrollerOf(from: HTMLElement): HTMLElement { return (from.closest('[data-conversation-scroll]')) ?? from } +/** Browser shrink clamps and recorded writes do not transfer scroll ownership. */ +function readerMovedScroll(top: number, floor: number, observedTop: number): boolean { + return Math.abs(top - Math.min(observedTop, floor)) > 0.5 +} + interface PagingAnchor { /** Stable node/call identity, independent of boundary-spanning group keys. */ key: string @@ -555,7 +560,7 @@ export function ChatView({ // programmatic deliveries land on the ledger itself, so both preserve // the current ownership state. const floor = Math.max(0, el.scrollHeight - el.clientHeight) - const movedByReader = Math.abs(el.scrollTop - Math.min(observedTopRef.current, floor)) > 0.5 + const movedByReader = readerMovedScroll(el.scrollTop, floor, observedTopRef.current) const isAtBottom = movedByReader ? floor - el.scrollTop <= FOLLOW_THRESHOLD + 1 : atBottomRef.current @@ -579,9 +584,9 @@ export function ChatView({ scheduleActiveTurn() } - // Pinned deliveries must settle before layout growth can invalidate their - // floor. Away-reader anchor geometry stays coalesced until the interval or - // scrollend; pinned samples read only scroll metrics unless the reader leaves. + // Non-reader pinned deliveries must settle before layout growth invalidates + // their floor. Reader movement stays pending even inside the follow threshold, + // so growth cannot erase small gestures before they accumulate off the floor. useEffect(() => { const local = listRef.current /* v8 ignore next -- ref-null guard: effect runs after the list node commits. */ @@ -599,8 +604,11 @@ export function ChatView({ const onScroll = (): void => { scrollSamplePendingRef.current = true if (atBottomRef.current) { - sample() - return + const floor = Math.max(0, el.scrollHeight - el.clientHeight) + if (!readerMovedScroll(el.scrollTop, floor, observedTopRef.current)) { + sample() + return + } } sampleTimer ??= window.setTimeout(sample, SCROLL_SAMPLE_INTERVAL_MS) } diff --git a/packages/client/ui-chat/tests/chat-view.client.spec.tsx b/packages/client/ui-chat/tests/chat-view.client.spec.tsx index f6b7ec63a9..074ae15330 100644 --- a/packages/client/ui-chat/tests/chat-view.client.spec.tsx +++ b/packages/client/ui-chat/tests/chat-view.client.spec.tsx @@ -2445,6 +2445,39 @@ describe('ChatView', () => { } }) + it('lets small reader movements accumulate past the follow threshold during growth', () => { + let notify: (() => void) | undefined + class ResizeObserverStub { + constructor(callback: ResizeObserverCallback) { + notify = () => { callback([], this as unknown as ResizeObserver) } + } + + observe = vi.fn() + disconnect = vi.fn() + } + vi.stubGlobal('ResizeObserver', ResizeObserverStub) + const h = makeHarness({ nodes: [user(1, 'q'), assistant(2, 'a')] }) + const view = render() + const scroller = view.container.querySelector('[class*="scroll"]') as HTMLDivElement + const metrics = installScrollMetrics(scroller, 1_000, 300) + expect(notify).toBeDefined() + scroller.scrollTop = 700 + fireEvent.scroll(scroller) + fireEvent(scroller, new Event('scrollend')) + scroller.scrollTop = 690 + fireEvent.scroll(scroller) + metrics.setHeight(1_020) + act(() => { notify?.() }) + expect(scroller.scrollTop).toBe(690) + scroller.scrollTop = 680 + fireEvent.scroll(scroller) + fireEvent(scroller, new Event('scrollend')) + expect(view.getByLabelText('回到底部')).toBeTruthy() + metrics.setHeight(1_040) + act(() => { notify?.() }) + expect(scroller.scrollTop).toBe(680) + }) + it('clears an away sample when a back-to-bottom delivery restores pinned ownership', () => { const h = makeHarness({ nodes: [user(1, 'q'), assistant(2, 'a')] }) const view = render() @@ -2476,6 +2509,7 @@ describe('ChatView', () => { fireEvent.scroll(scroller) scroller.scrollTop = 500 fireEvent.scroll(scroller) + fireEvent(scroller, new Event('scrollend')) expect(view.getByLabelText('回到底部')).toBeTruthy() const rect = vi.spyOn(HTMLElement.prototype, 'getBoundingClientRect') try {