mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-08-29 04:26:38 +00:00
test(history): add packed transport benchmark
This commit is contained in:
+2
-2
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-08-15-packed-session-history-transport.md
|
||||
2026-08-15-packed-session-history-transport.md: f005bdc3e2815505d64996ad1e003d8e601e3ec8
|
||||
2026-08-15-packed-session-history-transport.zh.md: a8260c2a1d1c38512a2a4cad3d5c2bc67280923d
|
||||
2026-08-15-packed-session-history-transport.md: 7f85bfedde5d3789beb19708c5b5e5690947ae5e
|
||||
2026-08-15-packed-session-history-transport.zh.md: 8d4b37b1cdc35e260f8bb5f997ca0e76f651b447
|
||||
|
||||
@@ -32,6 +32,8 @@ A production-sized private session sample was measured without retaining or comm
|
||||
|
||||
Packing reduced uncompressed JSON by 90.8% relative to raw logical events and by 83.4% relative to the lossy completed-step projection candidate. Brotli output was 73.2% smaller than raw and 44.8% smaller than that projection candidate. These figures describe this sample rather than a protocol guarantee; savings scale with the length and regularity of delta runs.
|
||||
|
||||
The opt-in `packages/client/runtime/tests/history-transport.perf.client.ts` benchmark constructs the same logical-event, ordinary-event, and delta-run cardinalities from synthetic content. `DSH_SNAPSHOT=replay pnpm exec vitest run --config vitest.web.perf.config.ts packages/client/runtime/tests/history-transport.perf.client.ts` reports wire sizes and Host/client timing under `HISTORY_TRANSPORT_PERF_RESULT`. The manual performance inventory does not run in CI and carries no machine-dependent timing assertions; structural assertions pin the fixture cardinalities, compact input count, and identical final state from its two-consumer Assistant fold fixture.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
**Discard completed-step chunks on the Host.** This lowers logical event count but makes transport semantics depend on the current transcript policy, removes exact evidence from all consumers, and still sends every retained incomplete-step token as a separate envelope. The measured packed response is smaller while remaining lossless.
|
||||
|
||||
+2
@@ -32,6 +32,8 @@ Status: implemented
|
||||
|
||||
与原始逻辑事件相比,打包使未压缩 JSON 减少 90.8%;与有损的已完成步骤投影候选相比减少 83.4%。Brotli 输出相对原始形式减少 73.2%,相对该投影候选减少 44.8%。这些数字描述该样本,并非协议保证;收益随 delta run 的长度与规律性变化。
|
||||
|
||||
可选运行的 `packages/client/runtime/tests/history-transport.perf.client.ts` benchmark 使用合成内容构造相同的逻辑事件数、普通事件数与 delta run 数。`DSH_SNAPSHOT=replay pnpm exec vitest run --config vitest.web.perf.config.ts packages/client/runtime/tests/history-transport.perf.client.ts` 会在 `HISTORY_TRANSPORT_PERF_RESULT` 下报告协议体积与 Host/client 计时。CI 不执行这组手动性能用例,其中也没有依赖机器性能的耗时断言;结构断言固定 fixture 的事件规模、紧凑输入数,以及双消费方 Assistant 折叠 fixture 的一致最终状态。
|
||||
|
||||
## 曾考虑的替代方案
|
||||
|
||||
**在 Host 丢弃已完成步骤的分片。** 这会减少逻辑事件数,但会让传输语义取决于当前 transcript 策略,从所有消费方移除精确证据,同时仍把保留的未完成步骤 token 逐个装入信封。实测打包响应在保持无损的同时更小。
|
||||
|
||||
@@ -67,6 +67,16 @@
|
||||
"tests/**/*.{ts,tsx}"
|
||||
]
|
||||
},
|
||||
"packages/api/session-controller": {
|
||||
"entry": [
|
||||
"tests/**/*.spec.ts",
|
||||
"tests/**/*.perf.client.ts"
|
||||
],
|
||||
"project": [
|
||||
"src/**/*.ts",
|
||||
"tests/**/*.ts"
|
||||
]
|
||||
},
|
||||
"packages/api/remotes": {
|
||||
"entry": [
|
||||
"tests/**/*.e2e.ts"
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
/** Opt-in synthetic benchmark for packed session-history transport and folding. */
|
||||
|
||||
import { createHash } from 'node:crypto'
|
||||
import { performance } from 'node:perf_hooks'
|
||||
import { brotliCompressSync, gzipSync } from 'node:zlib'
|
||||
import { expect, it } from 'vitest'
|
||||
import { createUserMessage } from '@deepseek-ai/dsh-llm'
|
||||
import type { HistoryEntry, HistoryRecord } from '@deepseek-ai/dsh-api-remotes/client'
|
||||
import { isChunkRow, packChunkRuns } from '@deepseek-ai/dsh-session/chunk-rows'
|
||||
import type { SessionEvent, SessionEventMap } from '@deepseek-ai/dsh-session/types'
|
||||
import {
|
||||
historyEntrySchema,
|
||||
sessionHistoryValueSchema,
|
||||
} from '@deepseek-ai/dsh-host-apiproxy/api/sessions.schema'
|
||||
import type {
|
||||
ConversationEventInput,
|
||||
ConversationNodeDefinition,
|
||||
ConversationViewDefinition,
|
||||
ConversationViewNode,
|
||||
} from '../src/client/contract/conversation.ts'
|
||||
import { ConversationNodeAssembler } from '../src/client/sessions/conversation-assembler.ts'
|
||||
import { historyEntries } from '../src/client/sessions/history-records.ts'
|
||||
|
||||
const LOGICAL_EVENTS = 416_756
|
||||
const DELTA_EVENTS = 416_176
|
||||
const ORDINARY_EVENTS = LOGICAL_EVENTS - DELTA_EVENTS
|
||||
const DELTA_RUNS = 116
|
||||
const TIME_ZERO = 1_700_000_000_000
|
||||
|
||||
interface Timed<T> {
|
||||
readonly value: T
|
||||
readonly ms: number
|
||||
}
|
||||
|
||||
interface FoldState {
|
||||
readonly blocks: readonly string[]
|
||||
readonly firstTokenTime?: number
|
||||
readonly firstVisibleSeq?: number
|
||||
readonly firstVisibleTime?: number
|
||||
}
|
||||
|
||||
interface FoldSnapshots {
|
||||
readonly chat: unknown
|
||||
readonly trajectory: unknown
|
||||
}
|
||||
|
||||
interface RawHistoryValue {
|
||||
readonly events: HistoryEntry[]
|
||||
readonly hasMore: boolean
|
||||
}
|
||||
|
||||
interface PackedHistoryValue {
|
||||
readonly records: HistoryRecord[]
|
||||
readonly hasMore: boolean
|
||||
readonly fromSeq: number
|
||||
readonly toSeq: number
|
||||
}
|
||||
|
||||
function timed<T>(run: () => T): Timed<T> {
|
||||
const start = performance.now()
|
||||
const value = run()
|
||||
return { value, ms: performance.now() - start }
|
||||
}
|
||||
|
||||
function rounded(value: number): number {
|
||||
return Math.round(value * 100) / 100
|
||||
}
|
||||
|
||||
function reduction(before: number, after: number): number {
|
||||
return rounded((1 - after / before) * 100)
|
||||
}
|
||||
|
||||
function append<Type extends keyof SessionEventMap>(
|
||||
events: SessionEvent[],
|
||||
type: Type,
|
||||
data: SessionEventMap[Type],
|
||||
options: { readonly surfaceOp?: 'append'; readonly ignorable?: true } = {},
|
||||
): void {
|
||||
const seq = events.length
|
||||
events.push({ type, seq, time: TIME_ZERO + seq, data, ...options } as SessionEvent<Type>)
|
||||
}
|
||||
|
||||
function appendSeparator(events: SessionEvent[], run: number, separator: number): void {
|
||||
const seq = events.length
|
||||
events.push({
|
||||
type: 'benchmark/separator',
|
||||
seq,
|
||||
time: TIME_ZERO + seq,
|
||||
data: { run, separator },
|
||||
ignorable: true,
|
||||
} as SessionEvent)
|
||||
}
|
||||
|
||||
function fragment(run: number, index: number): string {
|
||||
const value = (Math.imul(run + 1, 0x9E3779B1) ^ Math.imul(index + 1, 0x85EBCA6B)) >>> 0
|
||||
return value.toString(36).padStart(7, '0').slice(-2)
|
||||
}
|
||||
|
||||
/** Build the private sample's event/run cardinality from deterministic synthetic content. */
|
||||
function buildEvents(): SessionEvent[] {
|
||||
const events: SessionEvent[] = []
|
||||
append(events, 'turn/start', { turn: 1 })
|
||||
append(events, 'user/message', createUserMessage({
|
||||
content: [{ type: 'text', text: 'synthetic history transport benchmark' }],
|
||||
source: { kind: 'user' },
|
||||
}), { surfaceOp: 'append' })
|
||||
append(events, 'step/start', { turn: 1, step: 1 })
|
||||
|
||||
const baseRunLength = Math.floor(DELTA_EVENTS / DELTA_RUNS)
|
||||
const longerRuns = DELTA_EVENTS % DELTA_RUNS
|
||||
for (let run = 0; run < DELTA_RUNS; run++) {
|
||||
const runLength = baseRunLength + (run < longerRuns ? 1 : 0)
|
||||
for (let index = 0; index < runLength; index++) {
|
||||
append(events, 'assistant/chunk', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
chunk: {
|
||||
type: 'reasoning-delta',
|
||||
index: run,
|
||||
text: fragment(run, index),
|
||||
},
|
||||
})
|
||||
}
|
||||
const separators = run < 3 ? 4 : 5
|
||||
for (let separator = 0; separator < separators; separator++) {
|
||||
appendSeparator(events, run, separator)
|
||||
}
|
||||
}
|
||||
return events
|
||||
}
|
||||
|
||||
function foldDefinition(kind: string, target: string): ConversationNodeDefinition<FoldState> {
|
||||
return {
|
||||
kind,
|
||||
target,
|
||||
match: (event) => {
|
||||
if (event.type === 'step/start') return { id: `${String(event.data.turn)}:${String(event.data.step)}`, role: 'start' }
|
||||
if (event.type === 'assistant/chunk' && event.data.chunk.type === 'reasoning-delta') {
|
||||
return { id: `${String(event.data.turn)}:${String(event.data.step)}`, role: 'update' }
|
||||
}
|
||||
return null
|
||||
},
|
||||
start: () => ({ blocks: [] }),
|
||||
update: (context, match) => {
|
||||
if (match.event.type !== 'assistant/chunk' || match.event.data.chunk.type !== 'reasoning-delta') {
|
||||
return context.state
|
||||
}
|
||||
const chunk = match.event.data.chunk
|
||||
const blocks = [...context.state.blocks]
|
||||
blocks[chunk.index] = (blocks[chunk.index] ?? '') + chunk.text
|
||||
const visible = blocks.some(block => block.trim() !== '')
|
||||
return {
|
||||
...context.state,
|
||||
blocks,
|
||||
...context.state.firstTokenTime === undefined ? { firstTokenTime: match.event.time } : {},
|
||||
...visible && context.state.firstVisibleSeq === undefined
|
||||
? { firstVisibleSeq: match.event.seq, firstVisibleTime: match.event.time }
|
||||
: {},
|
||||
}
|
||||
},
|
||||
buildViewNode: context => context.state === undefined
|
||||
? null
|
||||
: {
|
||||
key: context.key,
|
||||
kind: context.kind,
|
||||
id: context.id,
|
||||
target,
|
||||
data: context.state,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function viewDefinition(target: string): ConversationViewDefinition<ConversationViewNode, readonly ConversationViewNode[]> {
|
||||
return {
|
||||
target,
|
||||
create: () => ({
|
||||
empty: [],
|
||||
replace: ({ nodes }) => nodes,
|
||||
apply: ({ upserts }) => upserts,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
function conversationInputs(entries: readonly HistoryEntry[]): ConversationEventInput[] {
|
||||
return entries.map(entry => ({ event: entry.event, view: entry.view }))
|
||||
}
|
||||
|
||||
function assemble(entries: readonly ConversationEventInput[]): FoldSnapshots {
|
||||
const definitions = [
|
||||
foldDefinition('benchmark-chat-assistant', 'chat'),
|
||||
foldDefinition('benchmark-trajectory-assistant', 'trajectory'),
|
||||
]
|
||||
const assembler = new ConversationNodeAssembler(
|
||||
{ entries: () => definitions, fallbackEntry: () => undefined },
|
||||
{ entries: () => [viewDefinition('chat'), viewDefinition('trajectory')] },
|
||||
)
|
||||
assembler.replaceWindow(entries, false)
|
||||
assembler.flush()
|
||||
return {
|
||||
chat: assembler.snapshot('chat'),
|
||||
trajectory: assembler.snapshot('trajectory'),
|
||||
}
|
||||
}
|
||||
|
||||
function digest(value: unknown): string {
|
||||
return createHash('sha256').update(JSON.stringify(value)).digest('hex')
|
||||
}
|
||||
|
||||
it('reports packed history transport and compact fold costs', () => {
|
||||
const fixture = timed(buildEvents)
|
||||
const rawEntries = timed(() => fixture.value.map(event => ({ event })))
|
||||
const packed = timed(() => packChunkRuns(fixture.value))
|
||||
const packedRecords = timed(() => packed.value.map((record): HistoryRecord =>
|
||||
isChunkRow(record) ? { chunks: record } : { event: record }))
|
||||
const rawValue: RawHistoryValue = { events: rawEntries.value, hasMore: false }
|
||||
const packedValue: PackedHistoryValue = {
|
||||
records: packedRecords.value,
|
||||
hasMore: false,
|
||||
fromSeq: 0,
|
||||
toSeq: fixture.value.length,
|
||||
}
|
||||
|
||||
const rawJson = timed(() => JSON.stringify(rawValue))
|
||||
const packedJson = timed(() => JSON.stringify(packedValue))
|
||||
const rawGzip = timed(() => gzipSync(rawJson.value).byteLength)
|
||||
const packedGzip = timed(() => gzipSync(packedJson.value).byteLength)
|
||||
const rawBrotli = timed(() => brotliCompressSync(rawJson.value).byteLength)
|
||||
const packedBrotli = timed(() => brotliCompressSync(packedJson.value).byteLength)
|
||||
const parsedRaw = timed(() => JSON.parse(rawJson.value) as RawHistoryValue)
|
||||
const parsedPacked = timed(() => JSON.parse(packedJson.value) as PackedHistoryValue)
|
||||
const rawValidation = timed(() => {
|
||||
for (const entry of parsedRaw.value.events) historyEntrySchema.parse(entry)
|
||||
})
|
||||
const packedValidation = timed(() => sessionHistoryValueSchema.parse(parsedPacked.value))
|
||||
const rawPreparation = timed(() => conversationInputs(parsedRaw.value.events))
|
||||
const packedPreparation = timed(() => conversationInputs(historyEntries(parsedPacked.value.records)))
|
||||
|
||||
assemble(rawPreparation.value.slice(0, 1_000))
|
||||
assemble(packedPreparation.value)
|
||||
const rawFold = timed(() => assemble(rawPreparation.value))
|
||||
const packedFold = timed(() => assemble(packedPreparation.value))
|
||||
|
||||
const rawBytes = Buffer.byteLength(rawJson.value)
|
||||
const packedBytes = Buffer.byteLength(packedJson.value)
|
||||
const packedRows = packed.value.filter(isChunkRow)
|
||||
expect(fixture.value).toHaveLength(LOGICAL_EVENTS)
|
||||
expect(fixture.value.filter(event => event.type !== 'assistant/chunk')).toHaveLength(ORDINARY_EVENTS)
|
||||
expect(packedRows).toHaveLength(DELTA_RUNS)
|
||||
expect(packed.value).toHaveLength(696)
|
||||
expect(packedPreparation.value).toHaveLength(696)
|
||||
expect(digest(packedFold.value)).toBe(digest(rawFold.value))
|
||||
expect(packedBytes).toBeLessThan(rawBytes)
|
||||
|
||||
const rawResponseMs = rawEntries.ms + rawJson.ms
|
||||
const packedResponseMs = packed.ms + packedRecords.ms + packedJson.ms
|
||||
const rawClientMs = parsedRaw.ms + rawValidation.ms + rawPreparation.ms + rawFold.ms
|
||||
const packedClientMs = parsedPacked.ms + packedValidation.ms + packedPreparation.ms + packedFold.ms
|
||||
process.stdout.write(`HISTORY_TRANSPORT_PERF_RESULT ${JSON.stringify({
|
||||
fixture: {
|
||||
buildMs: rounded(fixture.ms),
|
||||
logicalEvents: fixture.value.length,
|
||||
ordinaryEvents: ORDINARY_EVENTS,
|
||||
deltaEvents: DELTA_EVENTS,
|
||||
deltaRuns: packedRows.length,
|
||||
packedRecords: packed.value.length,
|
||||
compactFoldInputs: packedPreparation.value.length,
|
||||
},
|
||||
bytes: {
|
||||
rawJson: rawBytes,
|
||||
packedJson: packedBytes,
|
||||
jsonReductionPct: reduction(rawBytes, packedBytes),
|
||||
rawGzip: rawGzip.value,
|
||||
packedGzip: packedGzip.value,
|
||||
gzipReductionPct: reduction(rawGzip.value, packedGzip.value),
|
||||
rawBrotli: rawBrotli.value,
|
||||
packedBrotli: packedBrotli.value,
|
||||
brotliReductionPct: reduction(rawBrotli.value, packedBrotli.value),
|
||||
},
|
||||
host: {
|
||||
rawEntryWrapMs: rounded(rawEntries.ms),
|
||||
packMs: rounded(packed.ms),
|
||||
packedRecordWrapMs: rounded(packedRecords.ms),
|
||||
rawStringifyMs: rounded(rawJson.ms),
|
||||
packedStringifyMs: rounded(packedJson.ms),
|
||||
rawGzipMs: rounded(rawGzip.ms),
|
||||
packedGzipMs: rounded(packedGzip.ms),
|
||||
rawBrotliMs: rounded(rawBrotli.ms),
|
||||
packedBrotliMs: rounded(packedBrotli.ms),
|
||||
rawResponseMs: rounded(rawResponseMs),
|
||||
packedResponseMs: rounded(packedResponseMs),
|
||||
responseReductionPct: reduction(rawResponseMs, packedResponseMs),
|
||||
},
|
||||
client: {
|
||||
rawParseMs: rounded(parsedRaw.ms),
|
||||
packedParseMs: rounded(parsedPacked.ms),
|
||||
rawValidationMs: rounded(rawValidation.ms),
|
||||
packedValidationMs: rounded(packedValidation.ms),
|
||||
rawPrepareMs: rounded(rawPreparation.ms),
|
||||
packedPrepareMs: rounded(packedPreparation.ms),
|
||||
rawFoldMs: rounded(rawFold.ms),
|
||||
packedFoldMs: rounded(packedFold.ms),
|
||||
rawHistoryMs: rounded(rawClientMs),
|
||||
packedHistoryMs: rounded(packedClientMs),
|
||||
historyReductionPct: reduction(rawClientMs, packedClientMs),
|
||||
},
|
||||
})}\n`)
|
||||
}, 600_000)
|
||||
@@ -1,13 +1,16 @@
|
||||
import { defineConfig } from 'vitest/config'
|
||||
import webConfig from './vitest.web.config.ts'
|
||||
|
||||
// Manual high-cardinality diagnostics stay outside vitest.web.config.ts's
|
||||
// .e2e.ts/.snapshot.ts inventory and therefore outside the CI web gate.
|
||||
// Manual high-cardinality diagnostics stay outside every default Vitest
|
||||
// inventory and therefore outside CI's executed test lanes.
|
||||
export default defineConfig({
|
||||
...webConfig,
|
||||
test: {
|
||||
...webConfig.test,
|
||||
include: ['apps/web/tests/**/*.perf.ts'],
|
||||
include: [
|
||||
'apps/web/tests/**/*.perf.ts',
|
||||
'packages/api/session-controller/tests/**/*.perf.client.ts',
|
||||
],
|
||||
disableConsoleIntercept: true,
|
||||
hookTimeout: 180_000,
|
||||
testTimeout: 600_000,
|
||||
|
||||
Reference in New Issue
Block a user