test(history): add packed transport benchmark

This commit is contained in:
kingwl
2026-08-25 20:10:43 +08:00
committed by imccyu
parent f2ca913756
commit dec9732d1f
6 changed files with 329 additions and 5 deletions
@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-08-15-packed-session-history-transport.md
2026-08-15-packed-session-history-transport.md: f005bdc3e2815505d64996ad1e003d8e601e3ec8
2026-08-15-packed-session-history-transport.zh.md: a8260c2a1d1c38512a2a4cad3d5c2bc67280923d
2026-08-15-packed-session-history-transport.md: 7f85bfedde5d3789beb19708c5b5e5690947ae5e
2026-08-15-packed-session-history-transport.zh.md: 8d4b37b1cdc35e260f8bb5f997ca0e76f651b447
@@ -32,6 +32,8 @@ A production-sized private session sample was measured without retaining or comm
Packing reduced uncompressed JSON by 90.8% relative to raw logical events and by 83.4% relative to the lossy completed-step projection candidate. Brotli output was 73.2% smaller than raw and 44.8% smaller than that projection candidate. These figures describe this sample rather than a protocol guarantee; savings scale with the length and regularity of delta runs.
The opt-in `packages/client/runtime/tests/history-transport.perf.client.ts` benchmark constructs the same logical-event, ordinary-event, and delta-run cardinalities from synthetic content. `DSH_SNAPSHOT=replay pnpm exec vitest run --config vitest.web.perf.config.ts packages/client/runtime/tests/history-transport.perf.client.ts` reports wire sizes and Host/client timing under `HISTORY_TRANSPORT_PERF_RESULT`. The manual performance inventory does not run in CI and carries no machine-dependent timing assertions; structural assertions pin the fixture cardinalities, compact input count, and identical final state from its two-consumer Assistant fold fixture.
## Alternatives considered
**Discard completed-step chunks on the Host.** This lowers logical event count but makes transport semantics depend on the current transcript policy, removes exact evidence from all consumers, and still sends every retained incomplete-step token as a separate envelope. The measured packed response is smaller while remaining lossless.
@@ -32,6 +32,8 @@ Status: implemented
与原始逻辑事件相比,打包使未压缩 JSON 减少 90.8%;与有损的已完成步骤投影候选相比减少 83.4%。Brotli 输出相对原始形式减少 73.2%,相对该投影候选减少 44.8%。这些数字描述该样本,并非协议保证;收益随 delta run 的长度与规律性变化。
可选运行的 `packages/client/runtime/tests/history-transport.perf.client.ts` benchmark 使用合成内容构造相同的逻辑事件数、普通事件数与 delta run 数。`DSH_SNAPSHOT=replay pnpm exec vitest run --config vitest.web.perf.config.ts packages/client/runtime/tests/history-transport.perf.client.ts` 会在 `HISTORY_TRANSPORT_PERF_RESULT` 下报告协议体积与 Host/client 计时。CI 不执行这组手动性能用例,其中也没有依赖机器性能的耗时断言;结构断言固定 fixture 的事件规模、紧凑输入数,以及双消费方 Assistant 折叠 fixture 的一致最终状态。
## 曾考虑的替代方案
**在 Host 丢弃已完成步骤的分片。** 这会减少逻辑事件数,但会让传输语义取决于当前 transcript 策略,从所有消费方移除精确证据,同时仍把保留的未完成步骤 token 逐个装入信封。实测打包响应在保持无损的同时更小。
+10
View File
@@ -67,6 +67,16 @@
"tests/**/*.{ts,tsx}"
]
},
"packages/api/session-controller": {
"entry": [
"tests/**/*.spec.ts",
"tests/**/*.perf.client.ts"
],
"project": [
"src/**/*.ts",
"tests/**/*.ts"
]
},
"packages/api/remotes": {
"entry": [
"tests/**/*.e2e.ts"
@@ -0,0 +1,307 @@
/** Opt-in synthetic benchmark for packed session-history transport and folding. */
import { createHash } from 'node:crypto'
import { performance } from 'node:perf_hooks'
import { brotliCompressSync, gzipSync } from 'node:zlib'
import { expect, it } from 'vitest'
import { createUserMessage } from '@deepseek-ai/dsh-llm'
import type { HistoryEntry, HistoryRecord } from '@deepseek-ai/dsh-api-remotes/client'
import { isChunkRow, packChunkRuns } from '@deepseek-ai/dsh-session/chunk-rows'
import type { SessionEvent, SessionEventMap } from '@deepseek-ai/dsh-session/types'
import {
historyEntrySchema,
sessionHistoryValueSchema,
} from '@deepseek-ai/dsh-host-apiproxy/api/sessions.schema'
import type {
ConversationEventInput,
ConversationNodeDefinition,
ConversationViewDefinition,
ConversationViewNode,
} from '../src/client/contract/conversation.ts'
import { ConversationNodeAssembler } from '../src/client/sessions/conversation-assembler.ts'
import { historyEntries } from '../src/client/sessions/history-records.ts'
const LOGICAL_EVENTS = 416_756
const DELTA_EVENTS = 416_176
const ORDINARY_EVENTS = LOGICAL_EVENTS - DELTA_EVENTS
const DELTA_RUNS = 116
const TIME_ZERO = 1_700_000_000_000
interface Timed<T> {
readonly value: T
readonly ms: number
}
interface FoldState {
readonly blocks: readonly string[]
readonly firstTokenTime?: number
readonly firstVisibleSeq?: number
readonly firstVisibleTime?: number
}
interface FoldSnapshots {
readonly chat: unknown
readonly trajectory: unknown
}
interface RawHistoryValue {
readonly events: HistoryEntry[]
readonly hasMore: boolean
}
interface PackedHistoryValue {
readonly records: HistoryRecord[]
readonly hasMore: boolean
readonly fromSeq: number
readonly toSeq: number
}
function timed<T>(run: () => T): Timed<T> {
const start = performance.now()
const value = run()
return { value, ms: performance.now() - start }
}
function rounded(value: number): number {
return Math.round(value * 100) / 100
}
function reduction(before: number, after: number): number {
return rounded((1 - after / before) * 100)
}
function append<Type extends keyof SessionEventMap>(
events: SessionEvent[],
type: Type,
data: SessionEventMap[Type],
options: { readonly surfaceOp?: 'append'; readonly ignorable?: true } = {},
): void {
const seq = events.length
events.push({ type, seq, time: TIME_ZERO + seq, data, ...options } as SessionEvent<Type>)
}
function appendSeparator(events: SessionEvent[], run: number, separator: number): void {
const seq = events.length
events.push({
type: 'benchmark/separator',
seq,
time: TIME_ZERO + seq,
data: { run, separator },
ignorable: true,
} as SessionEvent)
}
function fragment(run: number, index: number): string {
const value = (Math.imul(run + 1, 0x9E3779B1) ^ Math.imul(index + 1, 0x85EBCA6B)) >>> 0
return value.toString(36).padStart(7, '0').slice(-2)
}
/** Build the private sample's event/run cardinality from deterministic synthetic content. */
function buildEvents(): SessionEvent[] {
const events: SessionEvent[] = []
append(events, 'turn/start', { turn: 1 })
append(events, 'user/message', createUserMessage({
content: [{ type: 'text', text: 'synthetic history transport benchmark' }],
source: { kind: 'user' },
}), { surfaceOp: 'append' })
append(events, 'step/start', { turn: 1, step: 1 })
const baseRunLength = Math.floor(DELTA_EVENTS / DELTA_RUNS)
const longerRuns = DELTA_EVENTS % DELTA_RUNS
for (let run = 0; run < DELTA_RUNS; run++) {
const runLength = baseRunLength + (run < longerRuns ? 1 : 0)
for (let index = 0; index < runLength; index++) {
append(events, 'assistant/chunk', {
turn: 1,
step: 1,
chunk: {
type: 'reasoning-delta',
index: run,
text: fragment(run, index),
},
})
}
const separators = run < 3 ? 4 : 5
for (let separator = 0; separator < separators; separator++) {
appendSeparator(events, run, separator)
}
}
return events
}
function foldDefinition(kind: string, target: string): ConversationNodeDefinition<FoldState> {
return {
kind,
target,
match: (event) => {
if (event.type === 'step/start') return { id: `${String(event.data.turn)}:${String(event.data.step)}`, role: 'start' }
if (event.type === 'assistant/chunk' && event.data.chunk.type === 'reasoning-delta') {
return { id: `${String(event.data.turn)}:${String(event.data.step)}`, role: 'update' }
}
return null
},
start: () => ({ blocks: [] }),
update: (context, match) => {
if (match.event.type !== 'assistant/chunk' || match.event.data.chunk.type !== 'reasoning-delta') {
return context.state
}
const chunk = match.event.data.chunk
const blocks = [...context.state.blocks]
blocks[chunk.index] = (blocks[chunk.index] ?? '') + chunk.text
const visible = blocks.some(block => block.trim() !== '')
return {
...context.state,
blocks,
...context.state.firstTokenTime === undefined ? { firstTokenTime: match.event.time } : {},
...visible && context.state.firstVisibleSeq === undefined
? { firstVisibleSeq: match.event.seq, firstVisibleTime: match.event.time }
: {},
}
},
buildViewNode: context => context.state === undefined
? null
: {
key: context.key,
kind: context.kind,
id: context.id,
target,
data: context.state,
},
}
}
function viewDefinition(target: string): ConversationViewDefinition<ConversationViewNode, readonly ConversationViewNode[]> {
return {
target,
create: () => ({
empty: [],
replace: ({ nodes }) => nodes,
apply: ({ upserts }) => upserts,
}),
}
}
function conversationInputs(entries: readonly HistoryEntry[]): ConversationEventInput[] {
return entries.map(entry => ({ event: entry.event, view: entry.view }))
}
function assemble(entries: readonly ConversationEventInput[]): FoldSnapshots {
const definitions = [
foldDefinition('benchmark-chat-assistant', 'chat'),
foldDefinition('benchmark-trajectory-assistant', 'trajectory'),
]
const assembler = new ConversationNodeAssembler(
{ entries: () => definitions, fallbackEntry: () => undefined },
{ entries: () => [viewDefinition('chat'), viewDefinition('trajectory')] },
)
assembler.replaceWindow(entries, false)
assembler.flush()
return {
chat: assembler.snapshot('chat'),
trajectory: assembler.snapshot('trajectory'),
}
}
function digest(value: unknown): string {
return createHash('sha256').update(JSON.stringify(value)).digest('hex')
}
it('reports packed history transport and compact fold costs', () => {
const fixture = timed(buildEvents)
const rawEntries = timed(() => fixture.value.map(event => ({ event })))
const packed = timed(() => packChunkRuns(fixture.value))
const packedRecords = timed(() => packed.value.map((record): HistoryRecord =>
isChunkRow(record) ? { chunks: record } : { event: record }))
const rawValue: RawHistoryValue = { events: rawEntries.value, hasMore: false }
const packedValue: PackedHistoryValue = {
records: packedRecords.value,
hasMore: false,
fromSeq: 0,
toSeq: fixture.value.length,
}
const rawJson = timed(() => JSON.stringify(rawValue))
const packedJson = timed(() => JSON.stringify(packedValue))
const rawGzip = timed(() => gzipSync(rawJson.value).byteLength)
const packedGzip = timed(() => gzipSync(packedJson.value).byteLength)
const rawBrotli = timed(() => brotliCompressSync(rawJson.value).byteLength)
const packedBrotli = timed(() => brotliCompressSync(packedJson.value).byteLength)
const parsedRaw = timed(() => JSON.parse(rawJson.value) as RawHistoryValue)
const parsedPacked = timed(() => JSON.parse(packedJson.value) as PackedHistoryValue)
const rawValidation = timed(() => {
for (const entry of parsedRaw.value.events) historyEntrySchema.parse(entry)
})
const packedValidation = timed(() => sessionHistoryValueSchema.parse(parsedPacked.value))
const rawPreparation = timed(() => conversationInputs(parsedRaw.value.events))
const packedPreparation = timed(() => conversationInputs(historyEntries(parsedPacked.value.records)))
assemble(rawPreparation.value.slice(0, 1_000))
assemble(packedPreparation.value)
const rawFold = timed(() => assemble(rawPreparation.value))
const packedFold = timed(() => assemble(packedPreparation.value))
const rawBytes = Buffer.byteLength(rawJson.value)
const packedBytes = Buffer.byteLength(packedJson.value)
const packedRows = packed.value.filter(isChunkRow)
expect(fixture.value).toHaveLength(LOGICAL_EVENTS)
expect(fixture.value.filter(event => event.type !== 'assistant/chunk')).toHaveLength(ORDINARY_EVENTS)
expect(packedRows).toHaveLength(DELTA_RUNS)
expect(packed.value).toHaveLength(696)
expect(packedPreparation.value).toHaveLength(696)
expect(digest(packedFold.value)).toBe(digest(rawFold.value))
expect(packedBytes).toBeLessThan(rawBytes)
const rawResponseMs = rawEntries.ms + rawJson.ms
const packedResponseMs = packed.ms + packedRecords.ms + packedJson.ms
const rawClientMs = parsedRaw.ms + rawValidation.ms + rawPreparation.ms + rawFold.ms
const packedClientMs = parsedPacked.ms + packedValidation.ms + packedPreparation.ms + packedFold.ms
process.stdout.write(`HISTORY_TRANSPORT_PERF_RESULT ${JSON.stringify({
fixture: {
buildMs: rounded(fixture.ms),
logicalEvents: fixture.value.length,
ordinaryEvents: ORDINARY_EVENTS,
deltaEvents: DELTA_EVENTS,
deltaRuns: packedRows.length,
packedRecords: packed.value.length,
compactFoldInputs: packedPreparation.value.length,
},
bytes: {
rawJson: rawBytes,
packedJson: packedBytes,
jsonReductionPct: reduction(rawBytes, packedBytes),
rawGzip: rawGzip.value,
packedGzip: packedGzip.value,
gzipReductionPct: reduction(rawGzip.value, packedGzip.value),
rawBrotli: rawBrotli.value,
packedBrotli: packedBrotli.value,
brotliReductionPct: reduction(rawBrotli.value, packedBrotli.value),
},
host: {
rawEntryWrapMs: rounded(rawEntries.ms),
packMs: rounded(packed.ms),
packedRecordWrapMs: rounded(packedRecords.ms),
rawStringifyMs: rounded(rawJson.ms),
packedStringifyMs: rounded(packedJson.ms),
rawGzipMs: rounded(rawGzip.ms),
packedGzipMs: rounded(packedGzip.ms),
rawBrotliMs: rounded(rawBrotli.ms),
packedBrotliMs: rounded(packedBrotli.ms),
rawResponseMs: rounded(rawResponseMs),
packedResponseMs: rounded(packedResponseMs),
responseReductionPct: reduction(rawResponseMs, packedResponseMs),
},
client: {
rawParseMs: rounded(parsedRaw.ms),
packedParseMs: rounded(parsedPacked.ms),
rawValidationMs: rounded(rawValidation.ms),
packedValidationMs: rounded(packedValidation.ms),
rawPrepareMs: rounded(rawPreparation.ms),
packedPrepareMs: rounded(packedPreparation.ms),
rawFoldMs: rounded(rawFold.ms),
packedFoldMs: rounded(packedFold.ms),
rawHistoryMs: rounded(rawClientMs),
packedHistoryMs: rounded(packedClientMs),
historyReductionPct: reduction(rawClientMs, packedClientMs),
},
})}\n`)
}, 600_000)
+6 -3
View File
@@ -1,13 +1,16 @@
import { defineConfig } from 'vitest/config'
import webConfig from './vitest.web.config.ts'
// Manual high-cardinality diagnostics stay outside vitest.web.config.ts's
// .e2e.ts/.snapshot.ts inventory and therefore outside the CI web gate.
// Manual high-cardinality diagnostics stay outside every default Vitest
// inventory and therefore outside CI's executed test lanes.
export default defineConfig({
...webConfig,
test: {
...webConfig.test,
include: ['apps/web/tests/**/*.perf.ts'],
include: [
'apps/web/tests/**/*.perf.ts',
'packages/api/session-controller/tests/**/*.perf.client.ts',
],
disableConsoleIntercept: true,
hookTimeout: 180_000,
testTimeout: 600_000,