test(webhook): exercise the real CLI and model flow

.agents/notes/implemented/feature/2026-08-22-fire-and-forget-webhook-sessions.i18n.yaml

@@ -2,5 +2,5 @@
 # side as of the last confirmed-consistent state. Both languages carry equal authority;
 # after editing either side, bring the other along and re-record with:
 #   pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-22-fire-and-forget-webhook-sessions.md
-2026-08-22-fire-and-forget-webhook-sessions.md: f72956a282e988ff017e647e7ad97ed6138a6319
-2026-08-22-fire-and-forget-webhook-sessions.zh.md: fa64298d0ea245a56c9206b74b0f0f87a9f88f7f
+2026-08-22-fire-and-forget-webhook-sessions.md: 976bccd8b460de7cb696ee45ea8963710cc4c738
+2026-08-22-fire-and-forget-webhook-sessions.zh.md: f015d993e61864bbc21d9d55fc331c25118fbde3

.agents/notes/implemented/feature/2026-08-22-fire-and-forget-webhook-sessions.md

@@ -46,6 +46,8 @@ The initial follow-up is an ordinary durable user-role message with webhook prov
 
 Package tests pin independent callback execution, fire-and-forget HTTP timing, cancellation and quiescent disposal, request validation, Workspace attachment before prompt admission, rollback, GitHub HMAC and body limits, credential rotation, and exact Loader composition. The assembled Web example sends a signed ready-for-review delivery to an isolated second listener and records the resulting ordinary Workspace conversation.
 
+A real-API e2e test starts the built `dsh web` CLI with the webhook overlay and isolated listener, synthesizes only the signed inbound GitHub delivery, observes Workspace attachment and durable provenance through the public Web API, and waits for the real DeepSeek response. No DSH service, model adapter, or provider call is replaced by a test double.
+
 Source audits keep execution records, retry timers, dedupe maps, completion events, and Agent-status listeners absent.
 
 ## Consequences

.agents/notes/implemented/feature/2026-08-22-fire-and-forget-webhook-sessions.zh.md

@@ -46,6 +46,8 @@ Patch 加载会把插入行中的相对插件名锚定到 patch 文件。因而
 
 包级测试固定独立回调执行、fire-and-forget HTTP 时序、取消与静止态释放、请求验证、提示词接纳前的 Workspace 附加、rollback、GitHub HMAC 与 body 限制、凭据轮换和精确 Loader 组合。组装 Web 示例会向隔离的第二监听器发送签名 ready-for-review 交付，并记录所得普通 Workspace 对话。
 
+真实 API e2e 测试会通过带 webhook overlay 与隔离监听器的构建产物启动 `dsh web` CLI（命令行界面），只合成带签名的入站 GitHub 交付，通过公开 Web API 观察 Workspace 附加与持久来源信息，并等待真实 DeepSeek 响应。测试不会用 test double 替换任何 DSH 服务、模型适配器或提供方调用。
+
 源码审计会保持执行记录、重试 timer、去重 map、完成事件与 Agent 状态监听器不存在。
 
 ## Consequences

apps/cli/tests/fixtures/github-webhook-real/cordis.yml

@@ -0,0 +1,36 @@
+# Real-product test overlay: the CLI, provider, Web servers, webhook runtime,
+# adapter, rule, Workspace, Session, and Agent all remain production modules.
+
+- insert:
+    - id: webhook-runtime
+      name: '@deepseek-ai/dsh-webhook'
+
+    - id: github-webhook-real-e2e-rule
+      name: './github-webhook-rule.mjs'
+      config:
+        source: github-real-e2e
+        repository: deepseek-harness/deepseek-harness
+        workspacePath: !!js process.env.DSH_GITHUB_E2E_WORKSPACE
+        marker: !!js process.env.DSH_GITHUB_E2E_MARKER
+        agentPreset: minimal
+        permissionPreset: read-only
+
+    - id: github-webhook-real-e2e-ingress
+      name: cordis:group
+      group: true
+      isolate:
+        webServer: true
+      config:
+        - id: github-webhook-real-e2e-server
+          name: '@deepseek-ai/dsh-host-webserver'
+          config:
+            host: '127.0.0.1'
+            port: !!js Number(process.env.DSH_GITHUB_WEBHOOK_PORT)
+
+        - id: github-webhook-real-e2e-adapter
+          name: '@deepseek-ai/dsh-webhook-github'
+          config:
+            source: github-real-e2e
+            path: /github
+            secretEnv: DSH_GITHUB_WEBHOOK_SECRET
+            maxBodyBytes: 1048576

apps/cli/tests/fixtures/github-webhook-real/github-webhook-rule.mjs

@@ -0,0 +1,38 @@
+import z from '@deepseek-ai/schemastery'
+import { WebhookRuleId } from '@deepseek-ai/dsh-webhook'
+
+export const name = 'github-webhook-real-e2e-rule'
+export const inject = ['webhookRuntime']
+
+export const Config = z.object({
+  source: z.string().required(),
+  repository: z.string().required(),
+  workspacePath: z.string().required(),
+  marker: z.string().required(),
+  agentPreset: z.string().required(),
+  permissionPreset: z.string().required(),
+})
+
+export function apply(ctx, config) {
+  ctx.effect(() => ctx.webhookRuntime.register({
+    id: WebhookRuleId('github-real-e2e'),
+    kind: 'github',
+
+    run(delivery, signal) {
+      if (delivery.source !== config.source) return null
+      if (delivery.event.name !== 'pull_request') return null
+      const { payload } = delivery.event
+      if (payload.action !== 'ready_for_review') return null
+      if (payload.repository?.full_name !== config.repository) return null
+      signal.throwIfAborted()
+
+      return {
+        workspacePath: config.workspacePath,
+        title: 'GitHub webhook real e2e',
+        prompt: `Reply with exactly ${config.marker} and no other text. Do not call tools.`,
+        agentPreset: config.agentPreset,
+        permissionPreset: config.permissionPreset,
+      }
+    },
+  }))
+}

apps/cli/tests/github-webhook-real.e2e.ts

@@ -0,0 +1,330 @@
+/** Real CLI and DeepSeek evidence for a GitHub webhook-created Session. */
+
+import type { ChildProcess } from 'node:child_process'
+import { spawn } from 'node:child_process'
+import { createHmac } from 'node:crypto'
+import { existsSync } from 'node:fs'
+import { mkdir, mkdtemp, realpath, rm } from 'node:fs/promises'
+import { createServer } from 'node:net'
+import type { AddressInfo } from 'node:net'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { setTimeout as delay } from 'node:timers/promises'
+import { fileURLToPath } from 'node:url'
+import { describe, expect, it } from 'vitest'
+
+const REPO_ROOT = fileURLToPath(new URL('../../..', import.meta.url))
+const BUILT_BIN = join(REPO_ROOT, 'apps/cli/lib/bin.js')
+const OVERLAY = fileURLToPath(new URL('./fixtures/github-webhook-real/cordis.yml', import.meta.url))
+const SECRET = 'github-webhook-real-e2e-secret'
+const DELIVERY = 'github-webhook-real-e2e-delivery'
+const MARKER = 'DSH_GITHUB_WEBHOOK_REAL_E2E_OK'
+const TITLE = 'GitHub webhook real e2e'
+
+interface SessionList {
+  items: Array<{
+    sessionId: string
+    cwd?: string
+    agentPreset?: string
+    blank: boolean
+  }>
+}
+
+interface WorkspaceList {
+  items: Array<{
+    path: string
+    sessionIds: string[]
+  }>
+}
+
+interface HistoryPage {
+  events: Array<{
+    event: {
+      type: string
+      data: unknown
+    }
+  }>
+  hasMore: boolean
+}
+
+interface ProcessObservation {
+  readonly ready: Promise<string>
+  readonly text: () => string
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null
+}
+
+/** Capture bounded process output and resolve the public Web URL after settled boot. */
+function observeProcess(child: ChildProcess): ProcessObservation {
+  let output = ''
+  let settled = false
+  let resolveReady!: (url: string) => void
+  let rejectReady!: (error: Error) => void
+  const ready = new Promise<string>((resolve, reject) => {
+    resolveReady = resolve
+    rejectReady = reject
+  })
+  const timer = setTimeout(() => {
+    if (!settled) rejectReady(new Error(`dsh web did not become ready within 90s:\n${output}`))
+  }, 90_000)
+  timer.unref()
+  const append = (chunk: Buffer | string): void => {
+    output = `${output}${String(chunk)}`.slice(-100_000)
+    const match = /dsh web: (http:\/\/[^\s]+)/u.exec(output)
+    if (settled || match?.[1] === undefined) return
+    settled = true
+    clearTimeout(timer)
+    resolveReady(match[1].replace('0.0.0.0', '127.0.0.1'))
+  }
+  child.stdout?.on('data', append)
+  child.stderr?.on('data', append)
+  child.once('error', (error) => {
+    if (!settled) rejectReady(error)
+  })
+  child.once('exit', (code) => {
+    if (!settled) rejectReady(new Error(`dsh web exited before readiness (code ${String(code)}):\n${output}`))
+  })
+  return { ready, text: () => output }
+}
+
+/** Reserve and release one loopback port for the isolated webhook listener. */
+async function freePort(): Promise<number> {
+  const server = createServer()
+  await new Promise<void>((resolve, reject) => {
+    server.once('error', reject)
+    server.listen(0, '127.0.0.1', resolve)
+  })
+  const port = (server.address() as AddressInfo).port
+  await new Promise<void>((resolve, reject) => {
+    server.close((error) => {
+      if (error === undefined) resolve()
+      else reject(error)
+    })
+  })
+  return port
+}
+
+/** Invoke one public Web RPC method. */
+async function rpc<T>(baseUrl: string, method: string, payload: unknown): Promise<T> {
+  const response = await fetch(`${baseUrl}/api/${method}`, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify({
+      type: 'client-request',
+      rpcId: `github-webhook-real-${method}`,
+      method,
+      payload,
+    }),
+  })
+  if (!response.ok) throw new Error(`${method} returned HTTP ${String(response.status)}: ${await response.text()}`)
+  const envelope = await response.json() as {
+    result: { ok: true; value: T } | { ok: false; error: { code: string; message: string } }
+  }
+  if (!envelope.result.ok) {
+    throw new Error(`${method} failed: ${envelope.result.error.code}: ${envelope.result.error.message}`)
+  }
+  return envelope.result.value
+}
+
+/** Poll a public observation until it satisfies the test's behavior predicate. */
+async function eventually<T>(
+  child: ChildProcess,
+  processOutput: () => string,
+  label: string,
+  probe: () => Promise<T>,
+  accepts: (value: T) => boolean,
+  timeoutMs: number,
+): Promise<T> {
+  const deadline = Date.now() + timeoutMs
+  let lastValue: T | undefined
+  let lastError: unknown
+  while (Date.now() < deadline) {
+    if (child.exitCode !== null) {
+      throw new Error(`dsh web exited while waiting for ${label} (code ${String(child.exitCode)}):\n${processOutput()}`)
+    }
+    try {
+      lastValue = await probe()
+      if (accepts(lastValue)) return lastValue
+    } catch (error) {
+      lastError = error
+    }
+    await delay(300)
+  }
+  throw new Error(
+    `timed out waiting for ${label}; last value=${JSON.stringify(lastValue)}; `
+    + `last error=${String(lastError)}; process output:\n${processOutput()}`,
+  )
+}
+
+/** Return every text block from durable assistant messages. */
+function assistantText(page: HistoryPage): string {
+  const text: string[] = []
+  for (const { event } of page.events) {
+    if (event.type !== 'assistant/message' || !isRecord(event.data) || !isRecord(event.data.message)) continue
+    const content = event.data.message.content
+    if (!Array.isArray(content)) continue
+    for (const block of content) {
+      if (isRecord(block) && block.type === 'text' && typeof block.text === 'string') text.push(block.text)
+    }
+  }
+  return text.join('\n')
+}
+
+/** Stop the spawned CLI through its normal signal path, escalating only on a stuck teardown. */
+async function stop(child: ChildProcess): Promise<void> {
+  if (child.exitCode !== null) return
+  let resolveClosed!: () => void
+  const closed = new Promise<void>((resolve) => { resolveClosed = resolve })
+  child.once('close', resolveClosed)
+  child.kill('SIGTERM')
+  if (await Promise.race([closed.then(() => true), delay(10_000).then(() => false)])) return
+  if (child.exitCode === null) child.kill('SIGKILL')
+  await Promise.race([closed, delay(5_000)])
+}
+
+/** Send the sole synthetic external interaction: one signed GitHub delivery. */
+async function sendGitHubDelivery(origin: string): Promise<Response> {
+  const body = JSON.stringify({
+    action: 'ready_for_review',
+    number: 4242,
+    repository: { full_name: 'deepseek-harness/deepseek-harness' },
+    pull_request: {
+      title: 'Real CLI webhook e2e',
+      html_url: 'https://github.com/deepseek-harness/deepseek-harness/pull/4242',
+      draft: false,
+      user: { login: 'octocat' },
+      base: { ref: 'master', sha: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' },
+      head: { ref: 'webhook-e2e', sha: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' },
+    },
+  })
+  const signature = `sha256=${createHmac('sha256', SECRET).update(body).digest('hex')}`
+  return await fetch(`${origin}/github`, {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      'x-github-delivery': DELIVERY,
+      'x-github-event': 'pull_request',
+      'x-hub-signature-256': signature,
+    },
+    body,
+  })
+}
+
+describe.skipIf(!process.env.DEEPSEEK_API_KEY)('GitHub webhook through the real dsh CLI and model', () => {
+  it('creates, attaches, prompts, and completes a Workspace Session', async () => {
+    expect(existsSync(BUILT_BIN), `missing built CLI ${BUILT_BIN}; run pnpm run build:official`).toBe(true)
+    const root = await mkdtemp(join(tmpdir(), 'dsh-github-webhook-real-'))
+    const workspacePath = join(root, 'workspace')
+    await mkdir(workspacePath)
+    const canonicalWorkspacePath = await realpath(workspacePath)
+    const webhookPort = await freePort()
+    const child = spawn(process.execPath, [
+      BUILT_BIN,
+      'web',
+      '--patch', OVERLAY,
+      '--no-open',
+      '--host', '127.0.0.1',
+      '--port', '0',
+    ], {
+      cwd: root,
+      env: {
+        ...process.env,
+        DEEPSEEK_BASE_URL: 'https://api.deepseek.com',
+        DSH_AGENTS_HOME: join(root, '.agents'),
+        DSH_GITHUB_E2E_MARKER: MARKER,
+        DSH_GITHUB_E2E_WORKSPACE: workspacePath,
+        DSH_GITHUB_WEBHOOK_PORT: String(webhookPort),
+        DSH_GITHUB_WEBHOOK_SECRET: SECRET,
+        DSH_HOME: join(root, '.dsh'),
+        DSH_TELEMETRY_DISABLED: '1',
+      },
+      stdio: ['ignore', 'pipe', 'pipe'],
+    })
+    const observation = observeProcess(child)
+
+    try {
+      const baseUrl = await observation.ready
+      const webhookOrigin = `http://127.0.0.1:${String(webhookPort)}`
+
+      expect((await fetch(`${webhookOrigin}/api`)).status).toBe(404)
+      expect((await sendGitHubDelivery(baseUrl)).status).not.toBe(202)
+      expect((await sendGitHubDelivery(webhookOrigin)).status).toBe(202)
+
+      const workspaces = await eventually(
+        child,
+        observation.text,
+        'one Workspace-attached Session',
+        async () => await rpc<WorkspaceList>(baseUrl, 'workspace.list', {}),
+        value => value.items.some(workspace =>
+          workspace.path === canonicalWorkspacePath && workspace.sessionIds.length === 1),
+        30_000,
+      )
+      const workspace = workspaces.items.find(item => item.path === canonicalWorkspacePath)
+      const sessionId = workspace?.sessionIds[0]
+      if (sessionId === undefined) throw new Error('workspace.list did not expose the webhook Session')
+
+      const sessions = await rpc<SessionList>(baseUrl, 'session.list', {})
+      expect(sessions.items.find(session => session.sessionId === sessionId)).toMatchObject({
+        agentPreset: 'minimal',
+        blank: false,
+        cwd: canonicalWorkspacePath,
+      })
+
+      const admitted = await eventually(
+        child,
+        observation.text,
+        'webhook provenance, title, and permission events',
+        async () => await rpc<HistoryPage>(baseUrl, 'session.history', { sessionId, maxMessages: 100 }),
+        (page) => {
+          const events = page.events.map(item => item.event)
+          const title = events.find(event => event.type === 'session/title')
+          const permission = events.find(event =>
+            event.type === 'permission/preset'
+            && isRecord(event.data)
+            && event.data.preset === 'read-only')
+          const message = events.find(event =>
+            event.type === 'user/message'
+            && isRecord(event.data)
+            && isRecord(event.data.source)
+            && event.data.source.kind === 'webhook')
+          return isRecord(title?.data) && title.data.title === TITLE
+            && permission !== undefined
+            && isRecord(message?.data) && isRecord(message.data.source)
+            && message.data.source.provider === 'github'
+            && message.data.source.deliveryId === DELIVERY
+        },
+        30_000,
+      )
+      const webhookMessage = admitted.events.map(item => item.event)
+        .find(event => event.type === 'user/message'
+          && isRecord(event.data)
+          && isRecord(event.data.source)
+          && event.data.source.kind === 'webhook')
+      expect(webhookMessage?.data).toMatchObject({
+        content: [{ type: 'text', text: `Reply with exactly ${MARKER} and no other text. Do not call tools.` }],
+        source: {
+          kind: 'webhook',
+          provider: 'github',
+          deliveryId: DELIVERY,
+          ruleId: 'github-real-e2e',
+          source: 'github-real-e2e',
+        },
+      })
+
+      const completed = await eventually(
+        child,
+        observation.text,
+        'a real DeepSeek assistant response',
+        async () => await rpc<HistoryPage>(baseUrl, 'session.history', { sessionId, maxMessages: 100 }),
+        page => assistantText(page).includes(MARKER),
+        150_000,
+      )
+      expect(assistantText(completed)).toContain(MARKER)
+    } finally {
+      await stop(child)
+      await rm(root, { recursive: true, force: true })
+    }
+  }, 210_000)
+})

scripts/verify-cordis-config.ts

@@ -33,6 +33,7 @@ const root = resolve(import.meta.dirname, '..')
 // These example files are overlays consumed by the built dsh app, so their bare
 // specifiers resolve from apps/cli rather than the examples workspace.
 const appOverlayFiles = new Set([
+  'apps/cli/tests/fixtures/github-webhook-real/cordis.yml',
   'examples/web-cordis/cordis.yml',
   'examples/web-github-review/cordis.yml',
   'examples/web-schedule/cordis.yml',