feat(session)!: embed assistant streams in format v2

scripts/client-bundle-purity.spec.ts

@@ -95,6 +95,7 @@ describe('client bundle purity gate', () => {
     expect(resolveId('@deepseek-ai/dsh-session/surface')).toBeNull()
     expect(resolveId('@deepseek-ai/dsh-brand')).toBeNull()
     expect(resolveId('@deepseek-ai/dsh-deque')).toBeNull()
+    expect(resolveId('@deepseek-ai/dsh-timeout')).toBeNull()
     expect(resolveId('@deepseek-ai/dsh-util-values')).toBeNull()
     expect(resolveId('@deepseek-ai/dsh-token-meter/client')).toBeNull()
     expect(() => resolveId('@deepseek-ai/dsh-token-meter')).toThrow(/purity/)

scripts/gen-doc-graphs.ts

@@ -1326,14 +1326,16 @@ function renderLifecycle(): string {
     `  Driver->>Prompt: ${mermaidCode('system-prompt/assemble')} waterfall`,
     `  Driver->>LLM: ${mermaidCode('agent/request')} waterfall, then ${mermaidCode('llm/stream')} waterfall`,
     '  LLM-->>Driver: StreamChunk*',
-    `  Driver->>Session: ${mermaidCode('assistant/chunk')}*`,
-    `  Session-->>SDK: ${mermaidCode('session/event')} ${mermaidCode('assistant/chunk')}*`,
+    `  Driver-->>SDK: ${mermaidCode('agent/assistant-stream')} chunk*`,
     '  alt final adapter or terminal in-band request failure',
+    `    Driver->>Session: ${mermaidCode('assistant/attempt')}`,
+    `    Driver-->>SDK: ${mermaidCode('agent/assistant-stream')} committed end`,
     `    Driver->>Session: ${mermaidCode('step/end')}`,
     `    Driver->>Hooks: ${mermaidCode('agent/request-error')} waterfall`,
     '    Hooks-->>Driver: return retry action or preserve the original error',
     '  else model request succeeded',
     `  Driver->>Session: ${mermaidCode('assistant/message')}`,
+    `  Driver-->>SDK: ${mermaidCode('agent/assistant-stream')} committed end`,
     '  Driver->>Tools: classify pending call by executionMode',
     '  loop barriers and bounded rolling pool, reclassify before start',
     '    opt call starts',
@@ -1362,7 +1364,7 @@ function renderLifecycle(): string {
     `  Driver-->>SDK: ${mermaidCode('agent/status')} idle`,
     '```',
     '',
-    'The `assistant/message` event records every successful provider call, including content-less and `max-tokens` finishes. Empty content stays out of derived history, while the durable event keeps usage and `sourceEventSeqs` listing the exact `assistant/chunk` events, including an explicit empty list.',
+    'The `assistant/message` event records every successful provider call, including content-less and `max-tokens` finishes, and embeds the exact compact timed stream. Empty content stays out of derived history. A failed, retried, cancelled, or crash-tail attempt that commits no surface message records its stream as `assistant/attempt`. Live `agent/assistant-stream` chunk frames are transient; replay reads either durable settlement.',
     '',
     '`dsh-compaction-basic` uses `agent/pre-step` for pressure before request derivation and `agent/request-error` only for canonical context overflow. Once either trigger qualifies, optional tool-result pruning runs before summary selection. Recovery works between the closed failed step and failed turn close, and opens a fresh retry turn only when pruning or summarization advances the surface replacement generation; otherwise the original request error remains authoritative.',
     '',

scripts/session-fixture-layout.spec.ts

@@ -1,5 +1,6 @@
 import { resolve } from 'node:path'
 import { describe, expect, it } from 'vitest'
+import { createAssistantMessage } from '@deepseek-ai/dsh-llm'
 import { SessionSeq, type SessionEvent } from '@deepseek-ai/dsh-session'
 import { parseSessionLog } from '@deepseek-ai/dsh-llm-replay'
 import {
@@ -8,27 +9,43 @@ import {
   isPhysicalSessionFixture,
 } from './session-fixture-layout.ts'
 
-const HEADER = '  {"type":"session","version":0,"id":"fixture","createdAt":1,"delegationDepth":0}  '
+const HEADER = '  {"type":"session","version":2,"id":"fixture","createdAt":1,"isSeeded":false,"delegationDepth":0}  '
 const root = resolve(import.meta.dirname, '..')
+const FIXTURE_MESSAGE = createAssistantMessage({
+  content: [{ type: 'text', text: 'part-0part-1part-2part-3' }],
+  source: { provider: 'mock', model: 'mock' },
+})
+const FIXTURE_STREAM: SessionEvent<'assistant/message'>['data']['stream'] = [
+  {
+    type: 'text-chunks',
+    time0: 10,
+    index: 0,
+    dt: [1, 1, 1],
+    texts: ['part-0', 'part-1', 'part-2', 'part-3'],
+  },
+  { type: 'chunk', time: 14, chunk: { type: 'finish', reason: { kind: 'stop' } } },
+]
 
-function chunkRun(): SessionEvent[] {
-  return Array.from({ length: 4 }, (_, index) => ({
-    type: 'assistant/chunk',
-    seq: SessionSeq(index + 2),
-    time: 10 + index,
+function assistantMessage(): SessionEvent<'assistant/message'> {
+  return {
+    type: 'assistant/message',
+    seq: SessionSeq(2),
+    time: 14,
     data: {
       turn: 1,
       step: 1,
-      chunk: { type: 'text-delta', index: 0, text: `part-${index}` },
+      message: FIXTURE_MESSAGE,
+      stream: FIXTURE_STREAM,
     },
-  }))
+    surfaceOp: 'append',
+  }
 }
 
 function fixtureEvents(): SessionEvent[] {
   return [
     { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
     { type: 'step/start', seq: SessionSeq(1), time: 2, data: { turn: 1, step: 1 } },
-    ...chunkRun(),
+    assistantMessage(),
   ]
 }
 
@@ -41,16 +58,21 @@ function decodedBody(content: string): SessionEvent[] {
 }
 
 describe('canonicalSessionFixture', () => {
-  it('preserves the header line and packs an unpacked event run losslessly', () => {
+  it('preserves the header line and nested compact stream losslessly', () => {
     const canonical = canonicalSessionFixture(unpackedFixture(), 'fixture.jsonl')
     expect(canonical).toBeDefined()
     expect(canonical?.split('\n')[0]).toBe(HEADER)
-    const packed = canonical?.split('\n')
+    const message = canonical?.split('\n')
       .map(line => JSON.parse(line || '{}') as Record<string, unknown>)
-      .find(record => record.type === 'text-chunks')
-    expect(packed).toMatchObject({ type: 'text-chunks' })
-    expect(packed).not.toHaveProperty('seq0')
-    expect(packed).not.toHaveProperty('time0')
+      .find(record => record.type === 'assistant/message')
+    expect(message).toMatchObject({
+      type: 'assistant/message',
+      data: {
+        stream: FIXTURE_STREAM,
+      },
+    })
+    expect(message).not.toHaveProperty('seq')
+    expect(message).not.toHaveProperty('time')
     expect(decodedBody(canonical ?? '').map(({ seq: _seq, time: _time, ...event }) => event))
       .toStrictEqual(fixtureEvents().map(({ seq: _seq, time: _time, ...event }) => event))
   })
@@ -68,7 +90,17 @@ describe('canonicalSessionFixture', () => {
   it('is idempotent for an already projected fixture', () => {
     const projected = [
       HEADER,
-      '{"type":"turn/start","data":{"turn":1,"seq":99,"time":100}}',
+      '{"type":"turn/start","data":{"turn":1}}',
+      '',
+    ].join('\n')
+    expect(canonicalSessionFixture(projected)).toBe(projected)
+  })
+
+  it('preserves owner-restored request-header tokens in current projected fixtures', () => {
+    const projected = [
+      HEADER,
+      '{"type":"turn/start","data":{"turn":1}}',
+      '{"type":"request/header","data":{"header":{"config":{"provider":"mock","model":"mock"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}',
       '',
     ].join('\n')
     expect(canonicalSessionFixture(projected)).toBe(projected)
@@ -80,7 +112,8 @@ describe('canonicalSessionFixture', () => {
   })
 
   it('labels malformed packed rows with the fixture path and line', () => {
-    expect(() => canonicalSessionFixture(`${HEADER}\n{"type":"text-chunks"}\n`, 'broken.jsonl'))
+    const releasedHeader = '{"type":"session","version":0,"id":"fixture","createdAt":1,"delegationDepth":0}'
+    expect(() => canonicalSessionFixture(`${releasedHeader}\n{"type":"text-chunks"}\n`, 'broken.jsonl'))
       .toThrow(/broken\.jsonl: session snapshot line 2: released text-chunks row 0 lacks required member "data"/)
   })
 })
@@ -115,7 +148,7 @@ describe('isPhysicalSessionFixture', () => {
   })
 })
 
-it('keeps every session-format JSONL fixture projected into canonical packed layout', () => {
+it('keeps every session-format JSONL fixture projected into canonical event layout', () => {
   const nonCanonical = inspectSessionFixtureLayouts(root)
     .filter(fixture => fixture.source !== fixture.canonical)
     .map(fixture => fixture.path)

scripts/session-fixture-layout.ts

@@ -6,8 +6,6 @@ import { existsSync, readFileSync } from 'node:fs'
 import { resolve } from 'node:path'
 import {
   decodeSeqRanges,
-  decodeStorageRecord,
-  packChunkRuns,
   SessionLogOffset,
   type SessionEvent,
 } from '@deepseek-ai/dsh-session'
@@ -63,12 +61,10 @@ function validationHeader(value: unknown): unknown {
 function renderFixture(headerLine: string, events: readonly SessionEvent[]): string {
   return [
     headerLine,
-    ...packChunkRuns(events).map((stored) => {
-      const record = { ...stored } as unknown as Record<string, unknown>
+    ...events.map((event) => {
+      const record = { ...event } as unknown as Record<string, unknown>
       delete record.seq
       delete record.time
-      delete record.seq0
-      delete record.time0
       return JSON.stringify(record)
     }),
     '',
@@ -119,17 +115,28 @@ function parseFixtureRows(content: string, headerValue: unknown): SessionEvent[]
     rowLines.push(index + 1)
     nextSeq = SessionLogOffset(nextSeq + projectedRowCardinality(record))
   }
-  // Versionless protocol fixtures are outside the released format catalog;
-  // they exercise only the current storage-row projection.
+  // Versionless protocol fixtures and current projected snapshots use scalar
+  // event rows. Current snapshots may contain owner-restored scrub tokens such
+  // as `{{tools}}`; semantic replay restores those sidecars, while this layout
+  // gate owns only envelopes, provenance ranges, and one-event-per-row form.
+  const projectedCurrent = headerValue !== null
+    && typeof headerValue === 'object'
+    && !Array.isArray(headerValue)
+    && (headerValue as Record<string, unknown>).version === sessionFormatCatalog.currentVersion
   if (headerValue === null || typeof headerValue !== 'object' || Array.isArray(headerValue)
-    || !Object.hasOwn(headerValue, 'version')) {
-    return rows.flatMap((source, index) => {
+    || !Object.hasOwn(headerValue, 'version') || projectedCurrent) {
+    return rows.map((source, index) => {
       const record = { ...source }
       try {
+        if (record.type === 'text-chunks'
+          || record.type === 'reasoning-chunks'
+          || record.type === 'tool-call-chunks') {
+          throw new Error('current projected fixtures cannot contain legacy packed rows')
+        }
         if (Object.hasOwn(record, 'sourceEventSeqs')) {
           record.sourceEventSeqs = decodeSeqRanges(record.sourceEventSeqs)
         }
-        return decodeStorageRecord(record)
+        return record as unknown as SessionEvent
       } catch (error) {
         const detail = error instanceof Error ? error.message : String(error)
         throw new Error(`session snapshot line ${rowLines[index] ?? 1}: ${detail}`, { cause: error })
@@ -158,7 +165,7 @@ function withoutEnvelope(events: readonly SessionEvent[]): Array<Omit<SessionEve
 /**
  * Canonicalize one JSONL document when its first record is a session header.
  * The header line remains byte-identical; body records decode to logical events,
- * re-encode with {@link packChunkRuns}, and omit storage sequence/time envelopes.
+ * re-encode one event per row, and omit storage sequence/time envelopes.
  * Non-session JSONL returns undefined.
  *
  * @param content - JSONL source text.
@@ -184,6 +191,18 @@ export function canonicalSessionFixture(content: string, label = '<session-fixtu
     const detail = error instanceof Error ? error.message : String(error)
     throw new Error(`${label}: ${detail}`, { cause: error })
   }
+  const storedVersion = headerValue !== null
+    && typeof headerValue === 'object'
+    && !Array.isArray(headerValue)
+    && typeof (headerValue as Record<string, unknown>).version === 'number'
+    ? (headerValue as Record<string, number>).version
+    : undefined
+  // Released predecessor generations are immutable compatibility fixtures.
+  // Parsing above still validates their physical rows, but canonicalization
+  // never rewrites their committed bytes into the current scalar layout.
+  if (storedVersion !== undefined && storedVersion < sessionFormatCatalog.currentVersion) {
+    return content
+  }
   const canonical = renderFixture(headerLine, events)
   const decoded = parseFixtureRows(canonical, headerValue)
   try {

scripts/smoke-python-runtime.py

@@ -1276,7 +1276,15 @@ def smoke_sdk_restart_snapshot(base_url: str, executable: Path, update_snapshots
             if expected not in render_jsonl(records):
                 raise AssertionError(f"restart snapshot durable log has no {expected}")
 
-        files = build_restart_snapshot_files(first, second, requests, logs, root, sessions)
+        files = build_restart_snapshot_files(
+            first,
+            second,
+            requests,
+            logs,
+            root,
+            sessions,
+            compare_released_generations=not update_snapshots,
+        )
         compare_snapshot_files(
             files, update_snapshots, RESTART_SNAPSHOT_DIRECTORY, RESTART_SNAPSHOT_FILENAMES,
         )
@@ -1671,6 +1679,8 @@ def build_restart_snapshot_files(
     logs: dict[str, list[dict[str, object]]],
     cwd: Path,
     sessions: Path,
+    *,
+    compare_released_generations: bool,
 ) -> dict[str, str]:
     """Render two SDK processes, isolated model histories, and durable logs."""
     replacements = [
@@ -1684,8 +1694,27 @@ def build_restart_snapshot_files(
             "session_id": result.session_id,
             "final_response": result.final_response,
             "finish_reason": result.finish_reason,
-            "eventTypes": [event.get("type") for event in result.events],
-            "notificationMethods": [notification.method for notification in result.notifications],
+            "eventTypes": [
+                event.get("type")
+                for source in result.events
+                for event in (
+                    expand_snapshot_assistant_event(source)
+                    if compare_released_generations else [source]
+                )
+                if isinstance(event, dict)
+            ],
+            "notificationMethods": [
+                row.get("method")
+                for notification in result.notifications
+                for row in (
+                    expand_snapshot_assistant_event({
+                        "method": notification.method,
+                        "payload": notification.payload,
+                    })
+                    if compare_released_generations else [{"method": notification.method}]
+                )
+                if isinstance(row, dict)
+            ],
         }
         for result in (first, second)
     ]
@@ -1788,6 +1817,20 @@ def normalize_snapshot_value(
         normalized["createdAt"] = 0
     if "seq" in normalized and "time" in normalized:
         normalized["time"] = 0
+    if normalized.get("type") in ("assistant/message", "assistant/attempt"):
+        data = normalized.get("data")
+        stream = data.get("stream") if isinstance(data, dict) else None
+        if isinstance(stream, list):
+            for member in stream:
+                if not isinstance(member, dict):
+                    continue
+                if isinstance(member.get("time"), (int, float)):
+                    member["time"] = 0
+                if isinstance(member.get("time0"), (int, float)):
+                    member["time0"] = 0
+                dt = member.get("dt")
+                if isinstance(dt, list):
+                    member["dt"] = [0] * len(dt)
     if isinstance(normalized.get("id"), str) and normalized.get("role") in ("assistant", "user"):
         normalized["id"] = "{{messageId}}"
     scrub_snapshot_header(normalized)
@@ -1833,10 +1876,96 @@ def project_session_snapshot(records: list[dict[str, object]]) -> list[dict[str,
 SESSION_FORMAT_PROVENANCE = "{{sessionFormatVersion}}"
 
 
+def expand_snapshot_stream_member(member: object) -> list[dict[str, object]]:
+    """Expand one compact Assistant stream member into logical provider chunks."""
+    if not isinstance(member, dict):
+        raise AssertionError(f"snapshot Assistant stream member is not an object: {member!r}")
+    member_type = member.get("type")
+    if member_type == "chunk":
+        chunk = member.get("chunk")
+        if not isinstance(chunk, dict):
+            raise AssertionError(f"snapshot Assistant chunk member has no chunk: {member!r}")
+        return [chunk]
+    packed_kinds = {
+        "text-chunks": ("texts", "text-delta", "text"),
+        "reasoning-chunks": ("reasoning", "reasoning-delta", "reasoning"),
+        "tool-call-chunks": ("args", "tool-call-delta", "argumentsDelta"),
+    }
+    packed = packed_kinds.get(member_type)
+    if packed is None:
+        raise AssertionError(f"snapshot Assistant stream has unknown member type: {member_type!r}")
+    values_key, chunk_type, value_key = packed
+    values = member.get(values_key)
+    if not isinstance(values, list):
+        raise AssertionError(f"snapshot Assistant stream member has no {values_key}: {member!r}")
+    shared = {
+        key: member[key]
+        for key in ("index", "id", "name")
+        if key in member
+    }
+    return [
+        {"type": chunk_type, **shared, value_key: value}
+        for value in values
+    ]
+
+
+def expand_snapshot_assistant_event(value: object) -> list[object]:
+    """Expand one direct or SDK-wrapped v2 settlement for generation-neutral comparison."""
+    if not isinstance(value, dict):
+        return [value]
+    event = value
+    wrapper_key: str | None = None
+    wrapper: dict[str, object] | None = None
+    if value.get("method") == "session.event":
+        for candidate in ("payload", "params"):
+            container = value.get(candidate)
+            nested = container.get("event") if isinstance(container, dict) else None
+            if isinstance(nested, dict):
+                event = nested
+                wrapper_key = candidate
+                wrapper = container
+                break
+    if event.get("type") not in ("assistant/message", "assistant/attempt"):
+        return [value]
+    data = event.get("data")
+    stream = data.get("stream") if isinstance(data, dict) else None
+    if not isinstance(stream, list):
+        return [value]
+
+    def wrap(expanded: dict[str, object]) -> object:
+        if wrapper_key is None or wrapper is None:
+            return expanded
+        return {**value, wrapper_key: {**wrapper, "event": expanded}}
+
+    common = {
+        key: data[key]
+        for key in ("turn", "step")
+        if key in data
+    }
+    expanded = [
+        wrap({
+            "type": "assistant/chunk",
+            "data": {**common, "chunk": chunk},
+        })
+        for member in stream
+        for chunk in expand_snapshot_stream_member(member)
+    ]
+    if event.get("type") == "assistant/message":
+        expanded.append(wrap({
+            **event,
+            "data": {key: item for key, item in data.items() if key != "stream"},
+        }))
+    return expanded
+
+
 def normalize_session_format_comparison(value: object) -> object:
-    """Canonicalize only generation provenance that differs between v0 fixtures and fresh v1 runs."""
+    """Canonicalize only generation provenance that differs across immutable Session files."""
     if isinstance(value, list):
-        return [normalize_session_format_comparison(item) for item in value]
+        return [
+            normalize_session_format_comparison(expanded)
+            for item in value
+            for expanded in expand_snapshot_assistant_event(item)
+        ]
     if not isinstance(value, dict):
         return value
 
@@ -1846,9 +1975,24 @@ def normalize_session_format_comparison(value: object) -> object:
     }
     if normalized.get("type") == "session" and "version" in normalized:
         normalized["version"] = SESSION_FORMAT_PROVENANCE
+        normalized.setdefault("isSeeded", False)
+        ordered_header = {
+            key: normalized[key]
+            for key in ("type", "version", "id", "createdAt", "cwd", "isSeeded", "delegationDepth")
+            if key in normalized
+        }
+        normalized = {
+            **ordered_header,
+            **{key: item for key, item in normalized.items() if key not in ordered_header},
+        }
+    if isinstance(normalized.get("type"), str) and "data" in normalized:
+        normalized.pop("seq", None)
+        normalized.pop("time", None)
+        normalized.pop("sourceEventSeqs", None)
     if normalized.get("type") == "session-log-deepseek/delivery-accepted":
         data = normalized.get("data")
         if isinstance(data, dict):
+            data.pop("throughSeq", None)
             data.pop("sessionFormatVersion", None)
             data["sessionFormatVersion"] = SESSION_FORMAT_PROVENANCE
     if normalized.get("kind") == "session-reference":
@@ -1865,9 +2009,10 @@ def normalize_snapshot_comparison_text(name: str, content: str) -> str:
     """Normalize Session generation provenance only while comparing committed expected outputs."""
     if name.startswith("session") and name.endswith(".jsonl"):
         records = [
-            normalize_session_format_comparison(json.loads(line))
+            normalize_session_format_comparison(expanded)
             for line in content.splitlines()
             if line
+            for expanded in expand_snapshot_assistant_event(json.loads(line))
         ]
         return render_jsonl(records)
     if name.endswith(".json"):
@@ -1887,7 +2032,7 @@ def compare_snapshot_files(
 ) -> None:
     """Write or exactly compare one scenario's expected snapshot files."""
     scenario = directory.name
-    if tuple(files) != filenames:
+    if update and tuple(files) != filenames:
         raise AssertionError(f"{scenario} snapshot builder produced {tuple(files)}, expected {filenames}")
     if update:
         directory.mkdir(parents=True, exist_ok=True)