test(history): add packed transport benchmark

2026-08-29 04:26:38 +00:00 · 2026-08-25 20:10:43 +08:00
parent f2ca913756
commit dec9732d1f
6 changed files with 329 additions and 5 deletions
@@ -0,0 +1,307 @@
+/** Opt-in synthetic benchmark for packed session-history transport and folding. */
+
+import { createHash } from 'node:crypto'
+import { performance } from 'node:perf_hooks'
+import { brotliCompressSync, gzipSync } from 'node:zlib'
+import { expect, it } from 'vitest'
+import { createUserMessage } from '@deepseek-ai/dsh-llm'
+import type { HistoryEntry, HistoryRecord } from '@deepseek-ai/dsh-api-remotes/client'
+import { isChunkRow, packChunkRuns } from '@deepseek-ai/dsh-session/chunk-rows'
+import type { SessionEvent, SessionEventMap } from '@deepseek-ai/dsh-session/types'
+import {
+  historyEntrySchema,
+  sessionHistoryValueSchema,
+} from '@deepseek-ai/dsh-host-apiproxy/api/sessions.schema'
+import type {
+  ConversationEventInput,
+  ConversationNodeDefinition,
+  ConversationViewDefinition,
+  ConversationViewNode,
+} from '../src/client/contract/conversation.ts'
+import { ConversationNodeAssembler } from '../src/client/sessions/conversation-assembler.ts'
+import { historyEntries } from '../src/client/sessions/history-records.ts'
+
+const LOGICAL_EVENTS = 416_756
+const DELTA_EVENTS = 416_176
+const ORDINARY_EVENTS = LOGICAL_EVENTS - DELTA_EVENTS
+const DELTA_RUNS = 116
+const TIME_ZERO = 1_700_000_000_000
+
+interface Timed<T> {
+  readonly value: T
+  readonly ms: number
+}
+
+interface FoldState {
+  readonly blocks: readonly string[]
+  readonly firstTokenTime?: number
+  readonly firstVisibleSeq?: number
+  readonly firstVisibleTime?: number
+}
+
+interface FoldSnapshots {
+  readonly chat: unknown
+  readonly trajectory: unknown
+}
+
+interface RawHistoryValue {
+  readonly events: HistoryEntry[]
+  readonly hasMore: boolean
+}
+
+interface PackedHistoryValue {
+  readonly records: HistoryRecord[]
+  readonly hasMore: boolean
+  readonly fromSeq: number
+  readonly toSeq: number
+}
+
+function timed<T>(run: () => T): Timed<T> {
+  const start = performance.now()
+  const value = run()
+  return { value, ms: performance.now() - start }
+}
+
+function rounded(value: number): number {
+  return Math.round(value * 100) / 100
+}
+
+function reduction(before: number, after: number): number {
+  return rounded((1 - after / before) * 100)
+}
+
+function append<Type extends keyof SessionEventMap>(
+  events: SessionEvent[],
+  type: Type,
+  data: SessionEventMap[Type],
+  options: { readonly surfaceOp?: 'append'; readonly ignorable?: true } = {},
+): void {
+  const seq = events.length
+  events.push({ type, seq, time: TIME_ZERO + seq, data, ...options } as SessionEvent<Type>)
+}
+
+function appendSeparator(events: SessionEvent[], run: number, separator: number): void {
+  const seq = events.length
+  events.push({
+    type: 'benchmark/separator',
+    seq,
+    time: TIME_ZERO + seq,
+    data: { run, separator },
+    ignorable: true,
+  } as SessionEvent)
+}
+
+function fragment(run: number, index: number): string {
+  const value = (Math.imul(run + 1, 0x9E3779B1) ^ Math.imul(index + 1, 0x85EBCA6B)) >>> 0
+  return value.toString(36).padStart(7, '0').slice(-2)
+}
+
+/** Build the private sample's event/run cardinality from deterministic synthetic content. */
+function buildEvents(): SessionEvent[] {
+  const events: SessionEvent[] = []
+  append(events, 'turn/start', { turn: 1 })
+  append(events, 'user/message', createUserMessage({
+    content: [{ type: 'text', text: 'synthetic history transport benchmark' }],
+    source: { kind: 'user' },
+  }), { surfaceOp: 'append' })
+  append(events, 'step/start', { turn: 1, step: 1 })
+
+  const baseRunLength = Math.floor(DELTA_EVENTS / DELTA_RUNS)
+  const longerRuns = DELTA_EVENTS % DELTA_RUNS
+  for (let run = 0; run < DELTA_RUNS; run++) {
+    const runLength = baseRunLength + (run < longerRuns ? 1 : 0)
+    for (let index = 0; index < runLength; index++) {
+      append(events, 'assistant/chunk', {
+        turn: 1,
+        step: 1,
+        chunk: {
+          type: 'reasoning-delta',
+          index: run,
+          text: fragment(run, index),
+        },
+      })
+    }
+    const separators = run < 3 ? 4 : 5
+    for (let separator = 0; separator < separators; separator++) {
+      appendSeparator(events, run, separator)
+    }
+  }
+  return events
+}
+
+function foldDefinition(kind: string, target: string): ConversationNodeDefinition<FoldState> {
+  return {
+    kind,
+    target,
+    match: (event) => {
+      if (event.type === 'step/start') return { id: `${String(event.data.turn)}:${String(event.data.step)}`, role: 'start' }
+      if (event.type === 'assistant/chunk' && event.data.chunk.type === 'reasoning-delta') {
+        return { id: `${String(event.data.turn)}:${String(event.data.step)}`, role: 'update' }
+      }
+      return null
+    },
+    start: () => ({ blocks: [] }),
+    update: (context, match) => {
+      if (match.event.type !== 'assistant/chunk' || match.event.data.chunk.type !== 'reasoning-delta') {
+        return context.state
+      }
+      const chunk = match.event.data.chunk
+      const blocks = [...context.state.blocks]
+      blocks[chunk.index] = (blocks[chunk.index] ?? '') + chunk.text
+      const visible = blocks.some(block => block.trim() !== '')
+      return {
+        ...context.state,
+        blocks,
+        ...context.state.firstTokenTime === undefined ? { firstTokenTime: match.event.time } : {},
+        ...visible && context.state.firstVisibleSeq === undefined
+          ? { firstVisibleSeq: match.event.seq, firstVisibleTime: match.event.time }
+          : {},
+      }
+    },
+    buildViewNode: context => context.state === undefined
+      ? null
+      : {
+        key: context.key,
+        kind: context.kind,
+        id: context.id,
+        target,
+        data: context.state,
+      },
+  }
+}
+
+function viewDefinition(target: string): ConversationViewDefinition<ConversationViewNode, readonly ConversationViewNode[]> {
+  return {
+    target,
+    create: () => ({
+      empty: [],
+      replace: ({ nodes }) => nodes,
+      apply: ({ upserts }) => upserts,
+    }),
+  }
+}
+
+function conversationInputs(entries: readonly HistoryEntry[]): ConversationEventInput[] {
+  return entries.map(entry => ({ event: entry.event, view: entry.view }))
+}
+
+function assemble(entries: readonly ConversationEventInput[]): FoldSnapshots {
+  const definitions = [
+    foldDefinition('benchmark-chat-assistant', 'chat'),
+    foldDefinition('benchmark-trajectory-assistant', 'trajectory'),
+  ]
+  const assembler = new ConversationNodeAssembler(
+    { entries: () => definitions, fallbackEntry: () => undefined },
+    { entries: () => [viewDefinition('chat'), viewDefinition('trajectory')] },
+  )
+  assembler.replaceWindow(entries, false)
+  assembler.flush()
+  return {
+    chat: assembler.snapshot('chat'),
+    trajectory: assembler.snapshot('trajectory'),
+  }
+}
+
+function digest(value: unknown): string {
+  return createHash('sha256').update(JSON.stringify(value)).digest('hex')
+}
+
+it('reports packed history transport and compact fold costs', () => {
+  const fixture = timed(buildEvents)
+  const rawEntries = timed(() => fixture.value.map(event => ({ event })))
+  const packed = timed(() => packChunkRuns(fixture.value))
+  const packedRecords = timed(() => packed.value.map((record): HistoryRecord =>
+    isChunkRow(record) ? { chunks: record } : { event: record }))
+  const rawValue: RawHistoryValue = { events: rawEntries.value, hasMore: false }
+  const packedValue: PackedHistoryValue = {
+    records: packedRecords.value,
+    hasMore: false,
+    fromSeq: 0,
+    toSeq: fixture.value.length,
+  }
+
+  const rawJson = timed(() => JSON.stringify(rawValue))
+  const packedJson = timed(() => JSON.stringify(packedValue))
+  const rawGzip = timed(() => gzipSync(rawJson.value).byteLength)
+  const packedGzip = timed(() => gzipSync(packedJson.value).byteLength)
+  const rawBrotli = timed(() => brotliCompressSync(rawJson.value).byteLength)
+  const packedBrotli = timed(() => brotliCompressSync(packedJson.value).byteLength)
+  const parsedRaw = timed(() => JSON.parse(rawJson.value) as RawHistoryValue)
+  const parsedPacked = timed(() => JSON.parse(packedJson.value) as PackedHistoryValue)
+  const rawValidation = timed(() => {
+    for (const entry of parsedRaw.value.events) historyEntrySchema.parse(entry)
+  })
+  const packedValidation = timed(() => sessionHistoryValueSchema.parse(parsedPacked.value))
+  const rawPreparation = timed(() => conversationInputs(parsedRaw.value.events))
+  const packedPreparation = timed(() => conversationInputs(historyEntries(parsedPacked.value.records)))
+
+  assemble(rawPreparation.value.slice(0, 1_000))
+  assemble(packedPreparation.value)
+  const rawFold = timed(() => assemble(rawPreparation.value))
+  const packedFold = timed(() => assemble(packedPreparation.value))
+
+  const rawBytes = Buffer.byteLength(rawJson.value)
+  const packedBytes = Buffer.byteLength(packedJson.value)
+  const packedRows = packed.value.filter(isChunkRow)
+  expect(fixture.value).toHaveLength(LOGICAL_EVENTS)
+  expect(fixture.value.filter(event => event.type !== 'assistant/chunk')).toHaveLength(ORDINARY_EVENTS)
+  expect(packedRows).toHaveLength(DELTA_RUNS)
+  expect(packed.value).toHaveLength(696)
+  expect(packedPreparation.value).toHaveLength(696)
+  expect(digest(packedFold.value)).toBe(digest(rawFold.value))
+  expect(packedBytes).toBeLessThan(rawBytes)
+
+  const rawResponseMs = rawEntries.ms + rawJson.ms
+  const packedResponseMs = packed.ms + packedRecords.ms + packedJson.ms
+  const rawClientMs = parsedRaw.ms + rawValidation.ms + rawPreparation.ms + rawFold.ms
+  const packedClientMs = parsedPacked.ms + packedValidation.ms + packedPreparation.ms + packedFold.ms
+  process.stdout.write(`HISTORY_TRANSPORT_PERF_RESULT ${JSON.stringify({
+    fixture: {
+      buildMs: rounded(fixture.ms),
+      logicalEvents: fixture.value.length,
+      ordinaryEvents: ORDINARY_EVENTS,
+      deltaEvents: DELTA_EVENTS,
+      deltaRuns: packedRows.length,
+      packedRecords: packed.value.length,
+      compactFoldInputs: packedPreparation.value.length,
+    },
+    bytes: {
+      rawJson: rawBytes,
+      packedJson: packedBytes,
+      jsonReductionPct: reduction(rawBytes, packedBytes),
+      rawGzip: rawGzip.value,
+      packedGzip: packedGzip.value,
+      gzipReductionPct: reduction(rawGzip.value, packedGzip.value),
+      rawBrotli: rawBrotli.value,
+      packedBrotli: packedBrotli.value,
+      brotliReductionPct: reduction(rawBrotli.value, packedBrotli.value),
+    },
+    host: {
+      rawEntryWrapMs: rounded(rawEntries.ms),
+      packMs: rounded(packed.ms),
+      packedRecordWrapMs: rounded(packedRecords.ms),
+      rawStringifyMs: rounded(rawJson.ms),
+      packedStringifyMs: rounded(packedJson.ms),
+      rawGzipMs: rounded(rawGzip.ms),
+      packedGzipMs: rounded(packedGzip.ms),
+      rawBrotliMs: rounded(rawBrotli.ms),
+      packedBrotliMs: rounded(packedBrotli.ms),
+      rawResponseMs: rounded(rawResponseMs),
+      packedResponseMs: rounded(packedResponseMs),
+      responseReductionPct: reduction(rawResponseMs, packedResponseMs),
+    },
+    client: {
+      rawParseMs: rounded(parsedRaw.ms),
+      packedParseMs: rounded(parsedPacked.ms),
+      rawValidationMs: rounded(rawValidation.ms),
+      packedValidationMs: rounded(packedValidation.ms),
+      rawPrepareMs: rounded(rawPreparation.ms),
+      packedPrepareMs: rounded(packedPreparation.ms),
+      rawFoldMs: rounded(rawFold.ms),
+      packedFoldMs: rounded(packedFold.ms),
+      rawHistoryMs: rounded(rawClientMs),
+      packedHistoryMs: rounded(packedClientMs),
+      historyReductionPct: reduction(rawClientMs, packedClientMs),
+    },
+  })}\n`)
+}, 600_000)