From 3c1c6a89b15ef6b203c727018cf19644fe23b5cb Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 23 Aug 2026 16:53:55 +0800 Subject: [PATCH] test(python): gate installed runtime wheels across release targets (#2953) * test(python): exercise installed wheels as black boxes Add an installed-wheel mode that refuses source/editable imports, repository working directories, mismatched SDK/runtime versions, unpinned runtime dependencies, and executables outside the installed runtime distribution. The mode resolves the wheel-owned executable itself, so callers cannot accidentally prove an explicit checkout artifact. Add a real-API scenario that drives two tool-using turns through the public synchronous SDK, verifies the file bytes outside the agent, checks completed turn/tool events and persistence, and projects provider failures without retaining credential-bearing error text. The existing deterministic scenario set remains the keyless behavior oracle. Refs #2952. * ci(python): require installed-wheel checks on every release target Move the complete deterministic runtime scenarios behind construction and clean installation of the SDK and matching runtime wheels. Each native leg runs outside the checkout with source-resolution environment variables removed; Linux manylinux smokes assert the same installed provenance. Expand the required pull-request call from Linux x64 to Linux x64, Linux arm64, and macOS arm64. Trusted heads receive only DEEPSEEK_API_KEY_EXTERNAL for a fail-loud live two-turn smoke on each carrier, while fork and Dependabot heads retain the full keyless path without exposing secrets. Pin the reusable secret declaration, matrix call, aggregate dependency, untrusted-head condition, and live/keyless commands in the workflow contract test. Refs #2952. * docs(testing): make installed wheels the Python CI authority Record the clean-wheel provenance boundary, complete keyless scenario set, trusted real-API contract, secret handling, and three-target required topology in a new implemented testing decision. Update the SEA distribution and portable-CI authorities plus the Python contributor reference to describe the same current state. Archive the fully superseded Linux-x64-only decision after consolidating its rationale and alternatives into the new owner. Preserve its bilingual triplet as a sealed historical snapshot and redirect every active current-state reference. Refs #2952. --- .agents/notes/archived/manifest.json | 5 +- ...d-python-runtime-pull-request-ci.i18n.yaml | 4 +- ...required-python-runtime-pull-request-ci.md | 1 + ...uired-python-runtime-pull-request-ci.zh.md | 1 + ...cutable-sdk-runtime-distribution.i18n.yaml | 4 +- ...ile-executable-sdk-runtime-distribution.md | 4 +- ...-executable-sdk-runtime-distribution.zh.md | 4 +- ...ortable-required-pull-request-ci.i18n.yaml | 4 +- ...07-23-portable-required-pull-request-ci.md | 2 +- ...23-portable-required-pull-request-ci.zh.md | 2 +- ...talled-python-wheel-black-box-ci.i18n.yaml | 6 + ...-23-installed-python-wheel-black-box-ci.md | 51 +++ ...-installed-python-wheel-black-box-ci.zh.md | 51 +++ .../workflows/build-exe-for-python-sdk.yml | 70 +++- .github/workflows/ci.yml | 13 +- python/development.i18n.yaml | 4 +- python/development.md | 6 +- python/development.zh.md | 6 +- scripts/ci-workflow.spec.ts | 40 ++- scripts/smoke-python-runtime.py | 308 +++++++++++++++++- .../restart/requests.json | 58 ++++ .../python-sdk-single-exe/restart/result.json | 94 ++++++ .../restart/session.1.jsonl | 18 + .../restart/session.2.jsonl | 18 + 24 files changed, 727 insertions(+), 47 deletions(-) rename .agents/notes/{implemented => archived}/testing/2026-08-12-required-python-runtime-pull-request-ci.i18n.yaml (66%) rename .agents/notes/{implemented => archived}/testing/2026-08-12-required-python-runtime-pull-request-ci.md (99%) rename .agents/notes/{implemented => archived}/testing/2026-08-12-required-python-runtime-pull-request-ci.zh.md (99%) create mode 100644 .agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.i18n.yaml create mode 100644 .agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.md create mode 100644 .agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.zh.md create mode 100644 scripts/snapshots/python-sdk-single-exe/restart/requests.json create mode 100644 scripts/snapshots/python-sdk-single-exe/restart/result.json create mode 100644 scripts/snapshots/python-sdk-single-exe/restart/session.1.jsonl create mode 100644 scripts/snapshots/python-sdk-single-exe/restart/session.2.jsonl diff --git a/.agents/notes/archived/manifest.json b/.agents/notes/archived/manifest.json index 05fb2f3712..aa76eab57c 100644 --- a/.agents/notes/archived/manifest.json +++ b/.agents/notes/archived/manifest.json @@ -480,6 +480,9 @@ "testing/2026-07-18-tui-terminal-state-snapshots.zh.md": "sha256:26750f240f6c8a7b28746f62fe161b357e9c5dd52867cc7037399f1ed6ff37fa", "testing/2026-07-26-execa-for-test-subprocess-plumbing.i18n.yaml": "sha256:dd45cddb591b892739b75b0c180bde7f14008f4769227b863571475be295e1e0", "testing/2026-07-26-execa-for-test-subprocess-plumbing.md": "sha256:1f45a69d0a7367ec5afbf112a77b355339b35270af8ff52696bee879cdf770d3", - "testing/2026-07-26-execa-for-test-subprocess-plumbing.zh.md": "sha256:8a24bdc8376373d7a97f65cefc07078824bf918d6a9934056a025ecfafe8634b" + "testing/2026-07-26-execa-for-test-subprocess-plumbing.zh.md": "sha256:8a24bdc8376373d7a97f65cefc07078824bf918d6a9934056a025ecfafe8634b", + "testing/2026-08-12-required-python-runtime-pull-request-ci.i18n.yaml": "sha256:741e7e58e5e8a9c82d901c4a16a70cea9bd256eac0e94179b5a24a231bb9fe1f", + "testing/2026-08-12-required-python-runtime-pull-request-ci.md": "sha256:1f1273d7a550667533e29c76efd148aebf57581a91729c877b44a5e43a52d9ad", + "testing/2026-08-12-required-python-runtime-pull-request-ci.zh.md": "sha256:6b9bf126c6b83d9b21e135d38df677c0d5623168b4353c6ddb706f76762c2193" } } diff --git a/.agents/notes/implemented/testing/2026-08-12-required-python-runtime-pull-request-ci.i18n.yaml b/.agents/notes/archived/testing/2026-08-12-required-python-runtime-pull-request-ci.i18n.yaml similarity index 66% rename from .agents/notes/implemented/testing/2026-08-12-required-python-runtime-pull-request-ci.i18n.yaml rename to .agents/notes/archived/testing/2026-08-12-required-python-runtime-pull-request-ci.i18n.yaml index 773ac10950..d6f71ad4c3 100644 --- a/.agents/notes/implemented/testing/2026-08-12-required-python-runtime-pull-request-ci.i18n.yaml +++ b/.agents/notes/archived/testing/2026-08-12-required-python-runtime-pull-request-ci.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-08-12-required-python-runtime-pull-request-ci.md -2026-08-12-required-python-runtime-pull-request-ci.md: 61b1e832be6d29eafe5cb304d2bca3f0a59e3d84 -2026-08-12-required-python-runtime-pull-request-ci.zh.md: 1702af710837c45094711cf52e88bd54d71f7171 +2026-08-12-required-python-runtime-pull-request-ci.md: e7da767f22634bd50bc4fd38b1de34677c4124e7 +2026-08-12-required-python-runtime-pull-request-ci.zh.md: 702125b0da864eb35f1fe870748cc0e314b01a39 diff --git a/.agents/notes/implemented/testing/2026-08-12-required-python-runtime-pull-request-ci.md b/.agents/notes/archived/testing/2026-08-12-required-python-runtime-pull-request-ci.md similarity index 99% rename from .agents/notes/implemented/testing/2026-08-12-required-python-runtime-pull-request-ci.md rename to .agents/notes/archived/testing/2026-08-12-required-python-runtime-pull-request-ci.md index 61b1e832be..e7da767f22 100644 --- a/.agents/notes/implemented/testing/2026-08-12-required-python-runtime-pull-request-ci.md +++ b/.agents/notes/archived/testing/2026-08-12-required-python-runtime-pull-request-ci.md @@ -1,6 +1,7 @@ # Agent Note: Required Python runtime pull-request validation Status: implemented +Archived: 2026-08-23 English | [中文](2026-08-12-required-python-runtime-pull-request-ci.zh.md) diff --git a/.agents/notes/implemented/testing/2026-08-12-required-python-runtime-pull-request-ci.zh.md b/.agents/notes/archived/testing/2026-08-12-required-python-runtime-pull-request-ci.zh.md similarity index 99% rename from .agents/notes/implemented/testing/2026-08-12-required-python-runtime-pull-request-ci.zh.md rename to .agents/notes/archived/testing/2026-08-12-required-python-runtime-pull-request-ci.zh.md index 1702af7108..702125b0da 100644 --- a/.agents/notes/implemented/testing/2026-08-12-required-python-runtime-pull-request-ci.zh.md +++ b/.agents/notes/archived/testing/2026-08-12-required-python-runtime-pull-request-ci.zh.md @@ -1,6 +1,7 @@ # Agent Note: 必需的 Python 运行时拉取请求验证 Status: implemented +Archived: 2026-08-23 [English](2026-08-12-required-python-runtime-pull-request-ci.md) | 中文 diff --git a/.agents/notes/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.i18n.yaml b/.agents/notes/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.i18n.yaml index 68a1a8bbd0..ab17030d64 100644 --- a/.agents/notes/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md -2026-07-10-single-file-executable-sdk-runtime-distribution.md: 24d95be871b2cb06e07706e6d5e0a628e30ef1bd -2026-07-10-single-file-executable-sdk-runtime-distribution.zh.md: eb9ba37c28e89a894ae7f5132592aa98fa56b977 +2026-07-10-single-file-executable-sdk-runtime-distribution.md: 2a39409c7db2bf1de75843e3642ef27051ccfb17 +2026-07-10-single-file-executable-sdk-runtime-distribution.zh.md: 0f4bb7cf3e1914c008ae23590a3a26fdb87f0842 diff --git a/.agents/notes/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md b/.agents/notes/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md index 24d95be871..2a39409c7d 100644 --- a/.agents/notes/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md +++ b/.agents/notes/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md @@ -44,7 +44,7 @@ The deploy root includes `@deepseek-ai/dsh-mcp-client` as an explicitly supporte [`scripts/build-exe-for-python-sdk.ts`](../../../../scripts/build-exe-for-python-sdk.ts): runtime closure verification → `pnpm run build` → (after clearing) `pnpm --filter dsh-sdk-python-runtime-closure deploy --legacy --prod --config.node-linker=hoisted --config.auto-install-peers=false --config.link-workspace-packages=true` **directly into** `python/sdk-runtime/src/deepseek_harness_runtime/runtime/node/` → restore any direct workspace package that legacy deploy hoisted back under the source manifest's `node_modules`, omitting its package-local dependency tree and rejecting any remaining manifest gap → replace every staged dependency symlink with its target bytes, remove package-manager `.bin` links, and fail if any symlink remains → inject the pkg configuration (`bin` points at `node_modules/@deepseek-ai/dsh-sdk-python-runtime/lib/packaged-bin.js` inside the closure, `assets` is a full glob — dynamic import is invisible to pkg's static analysis, so everything must be packed in explicitly) → stage the target `node-pty` addon → one `pkg --sea` per target → the executables `dsh-jsonrpc-agent-pkg--` land in `dist-exe/` and are copied back into the runtime directory. Linux installs build `pty.node` from source; CI rebuilds that addon inside the matching manylinux 2.28 container before packaging, and the builder copies it from the root install into the staged closure because legacy deploy omits that side-effect directory. Every target copies its native `@vscode/ripgrep` binary beside the executable as the required `-rg` sidecar; pkg runtimes select that sidecar through `process.pkg`, while ordinary Node execution uses `@vscode/ripgrep` directly. macOS uses its target prebuild and also emits the required `-spawn-helper`. CI treats these products as intermediate test inputs and retains their platform wheels. All four deploy flags are grounded in measurement: `--legacy` is the mandatory path with inject-workspace-packages off; hoisted gives pkg a stable single-instance layout that the explicit materialization pass makes symlink-free; disabling automatic peer installation prevents undeclared peers from expanding the closure; link-workspace-packages selects direct workspace dependencies. [`pnpm-workspace.yaml`](../../../../pnpm-workspace.yaml) overrides the transitive `@deepseek-ai/cosmokit` and `@deepseek-ai/schemastery` semver requests to the pinned vendor sources so legacy deploy never resolves those unpublished names from a registry. -CI: [`.github/workflows/build-exe-for-python-sdk.yml`](../../../../.github/workflows/build-exe-for-python-sdk.yml), called for linux-x64 by the [required Python runtime pull-request validation](../testing/2026-08-12-required-python-runtime-pull-request-ci.md), triggered explicitly by `workflow_dispatch` or the `build-exe` label for selected targets, and called for all targets by the [public publication workflow](../process/2026-08-11-python-publication-workflow.md). Native builds run on linux-x64 / linux-arm64 (`ubuntu-24.04-arm`) / macos-arm64, with `~/.pkg-cache` cached, and pkg handles macOS ad-hoc signing. Each leg drives a mock SSE model through the SDK with the default config and a custom `cordis.yml`, drives the exe directly over NDJSON JSON-RPC, verifies the JSONL and final response, and installs release-shaped wheels into a clean venv without `runtime_bin`; Linux additionally inspects both the executable and native addon's GLIBC requirements and runs in a manylinux 2.28 container, while macOS verifies that the executable's deployment target fits the wheel tag. A full three-target run retains four artifacts, each containing one release file: the platform-independent SDK wheel and three native runtime wheels; a subset dispatch retains the SDK wheel and selected runtime wheels. Bare executables and source bundles remain intermediate test inputs. [`.gitlab-ci.yml`](../../../../.gitlab-ci.yml) accepts `python-v` tag pipelines whose version matches the root `package.json`, builds one SDK wheel and three native runtime wheels, then a single serialized job checks and publishes all four to the project PyPI registry. Windows is a non-goal. +CI: [`.github/workflows/build-exe-for-python-sdk.yml`](../../../../.github/workflows/build-exe-for-python-sdk.yml) is called for all three targets by the [installed-wheel Python runtime pull-request validation](../testing/2026-08-23-installed-python-wheel-black-box-ci.md) and the [public publication workflow](../process/2026-08-11-python-publication-workflow.md); `workflow_dispatch` and the `build-exe` label can still select a subset. Native builds run on linux-x64 / linux-arm64 (`ubuntu-24.04-arm`) / macos-arm64, with `~/.pkg-cache` cached, and pkg handles macOS ad-hoc signing. Each leg installs the release-shaped SDK and runtime wheels into a clean venv outside the checkout, proves their package and executable provenance, then drives the complete keyless scenario set through the public SDK and direct NDJSON JSON-RPC. Trusted pull requests additionally run a real DeepSeek two-turn tool smoke on every target; fork and Dependabot heads receive no key. Linux inspects the executable and native addon's GLIBC requirements and runs an additional manylinux 2.28 smoke, while macOS verifies that the executable's deployment target fits the wheel tag. A full three-target run retains four artifacts, each containing one release file: the platform-independent SDK wheel and three native runtime wheels; a subset dispatch retains the SDK wheel and selected runtime wheels. Bare executables and source bundles remain intermediate test inputs. [`.gitlab-ci.yml`](../../../../.gitlab-ci.yml) accepts `python-v` tag pipelines whose version matches the root `package.json`, builds one SDK wheel and three native runtime wheels, then a single serialized job checks and publishes all four to the project PyPI registry. Windows is a non-goal. ### Python SDK distribution: two carriers, exe for production, node for development @@ -64,7 +64,7 @@ The exe's "must be explicitly configured" hard semantic is unchanged; the zero-c ## Testing -The verification surface has three tiers. Mechanism tier: the measured conclusions for the `--sea` chain are embedded in the Decision sections (ESM dynamic import inside the VFS, single cordis instance, fail-loud config chain, `node:sqlite`, macOS ad-hoc signing runs). SDK tier: the complete keyless pytest suite covers the client protocol against a fake runtime peer, subprocess cleanup, absolute cwd propagation, dual-carrier launch, and carrier resolution; root CI runs it on Python 3.10. End-to-end tier: every platform build completes a turn against a mock endpoint through the default SDK path, a custom config, the checked-in standalone minimal composition, and the direct binary protocol, with final text and JSONL checked. The minimal run asserts its exact system prompt and two-tool catalog, retains Bash state across calls, and invokes the editor. The custom config additionally drives `run_code` and a zero-agent `workflow` through their real worker files inside the packaged VFS. The filesystem-search scenario requires the model to call both `glob` and `grep` through the target-native `-rg` sidecar. The MCP scenario starts a temporary external stdio server, deliberately delays its initial `tools/list` response, then immediately starts the first SDK prompt; the prompt must see and call the discovered tool, proving that `initialize` is a real Loader-settlement readiness boundary rather than a timing sleep. The same build leg runs a committed executable-specific snapshot through the Python SDK: a keyless scripted model mounts a Cordis plugin that registers a tool, invokes that tool from `run_code`, runs a direct spawn subagent and a workflow that starts a second spawn child, then unmounts the plugin. The fixture explicitly disables its unused bundled Bash and local skill discovery so its tool set does not depend on repository-external state, and the comparison normalizes opaque message, agent, workflow-run, and session IDs across the SDK result and notification stream plus the parent and two child JSONL logs. This harness stays separate from ACP's `pnpm run test:snapshot` because the protocols and build artifacts differ. The platform wheel is then installed in a clean venv and run without `runtime_bin`. +The verification surface has three tiers. Mechanism tier: the measured conclusions for the `--sea` chain are embedded in the Decision sections (ESM dynamic import inside the VFS, single cordis instance, fail-loud config chain, `node:sqlite`, macOS ad-hoc signing runs). SDK tier: the complete keyless pytest suite covers the client protocol against a fake runtime peer, subprocess cleanup, absolute cwd propagation, dual-carrier launch, and carrier resolution; root CI runs it on Python 3.10. End-to-end tier: every platform build installs both wheels into a clean venv outside the checkout, proves matching versions and installed module/executable locations, then completes turns against a mock endpoint through the default SDK path, a custom config, the checked-in standalone minimal composition, and the direct binary protocol, with final text and JSONL checked. The minimal run asserts its exact system prompt and two-tool catalog, retains Bash state across calls, and invokes the editor. The custom config additionally drives `run_code` and a zero-agent `workflow` through their real worker files inside the packaged VFS. The filesystem-search scenario requires the model to call both `glob` and `grep` through the target-native `-rg` sidecar. The MCP scenario starts a temporary external stdio server, deliberately delays its initial `tools/list` response, then immediately starts the first SDK prompt; the prompt must see and call the discovered tool, proving that `initialize` is a real Loader-settlement readiness boundary rather than a timing sleep. The same installed run compares a committed executable-specific snapshot through the Python SDK: a keyless scripted model mounts a Cordis plugin that registers a tool, invokes that tool from `run_code`, runs a direct spawn subagent and a workflow that starts a second spawn child, then unmounts the plugin. The fixture explicitly disables its unused bundled Bash and local skill discovery so its tool set does not depend on repository-external state, and the comparison normalizes opaque message, agent, workflow-run, and session IDs across the SDK result and notification stream plus the parent and two child JSONL logs. Trusted pull requests add a real-provider two-turn file write/read whose external bytes, tool calls, completed reasons, and persisted log must agree. This harness stays separate from ACP's `pnpm run test:snapshot` because the protocols and build artifacts differ. Manual-driving caveat: the bin treats stdin EOF as "the client is gone" and disposes immediately, so a short-lived pipe aborts an in-flight turn — pipe-driven runs must keep stdin open until the turn ends. diff --git a/.agents/notes/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.zh.md b/.agents/notes/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.zh.md index eb9ba37c28..0f4bb7cf3e 100644 --- a/.agents/notes/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.zh.md +++ b/.agents/notes/implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.zh.md @@ -44,7 +44,7 @@ exe 的 VFS 内是**构建产物形态的真实包树**（各包的 `lib/` + 真 [`scripts/build-exe-for-python-sdk.ts`](../../../../scripts/build-exe-for-python-sdk.ts)：运行时闭包校验 → `pnpm run build` →（清空后）`pnpm --filter dsh-sdk-python-runtime-closure deploy --legacy --prod --config.node-linker=hoisted --config.auto-install-peers=false --config.link-workspace-packages=true` **直接写入** `python/sdk-runtime/src/deepseek_harness_runtime/runtime/node/` → 恢复被 legacy deploy 提升回源 manifest 的 `node_modules` 下的任何直接工作区包，同时省略其包内依赖树，并拒绝剩余的 manifest 缺口 → 将暂存依赖中的每个符号链接替换为目标文件内容，删除包管理器的 `.bin` 链接，并在仍有任何符号链接时失败 → 注入 pkg 配置（`bin` 指向闭包内的 `node_modules/@deepseek-ai/dsh-sdk-python-runtime/lib/packaged-bin.js`；`assets` 使用全量 glob，因为动态 `import()` 对 pkg 静态分析不可见，必须显式打入全部内容）→ 暂存目标平台的 `node-pty` addon → 每个构建目标调用一次 `pkg --sea` → 可执行文件 `dsh-jsonrpc-agent-pkg--` 写入 `dist-exe/`，并拷回运行时目录。Linux 安装会从源码构建 `pty.node`；CI 会在打包前进入匹配架构的 manylinux 2.28 容器重新构建该 addon，而 `--legacy` 部署会省略这一副作用目录，因此构建器会把它从根安装目录复制到暂存闭包。每个目标都会把对应的原生 `@vscode/ripgrep` 二进制复制到可执行文件旁，作为必需的 `-rg` 伴随文件；pkg 运行时通过 `process.pkg` 选择该伴随文件，普通 Node 执行则直接使用 `@vscode/ripgrep`。macOS 使用对应目标的预构建产物，并额外生成所需的 `-spawn-helper`。CI 将这些产物作为测试中间输入，只保留对应平台的 wheel 包。四个部署标志都有实测依据：未启用 `inject-workspace-packages` 时必须使用 `--legacy`；`hoisted` 为 pkg 提供稳定的单实例布局，再由显式物化步骤消除符号链接；关闭对等依赖自动安装可防止未声明的对等依赖扩大闭包；`link-workspace-packages` 选择直接工作区依赖。[`pnpm-workspace.yaml`](../../../../pnpm-workspace.yaml) 将传递的 `@deepseek-ai/cosmokit` 与 `@deepseek-ai/schemastery` semver 请求覆盖到固定的 vendor 源码，使 legacy deploy 不会从注册表解析这些未发布名称。 -CI 使用 [`.github/workflows/build-exe-for-python-sdk.yml`](../../../../.github/workflows/build-exe-for-python-sdk.yml)：[必需的 Python 运行时拉取请求验证](../testing/2026-08-12-required-python-runtime-pull-request-ci.zh.md)调用它构建 linux-x64，手动派发 `workflow_dispatch` 或 PR（Pull Request）的 `build-exe` 标签可以显式选择构建目标，[公开发布工作流](../process/2026-08-11-python-publication-workflow.zh.md)则调用它构建全部目标。linux-x64、linux-arm64（`ubuntu-24.04-arm`）和 macos-arm64 三个平台分别进行原生构建，并缓存 `~/.pkg-cache`；macOS 的 ad-hoc 签名由 pkg 处理。每个平台都使用 mock SSE（Server-Sent Events）模型，分别通过默认配置和自定义 `cordis.yml` 驱动 SDK，再通过 NDJSON JSON-RPC 直接驱动 exe，校验 JSONL 与最终响应；最后把发布形态的 wheel 包安装到干净的 venv 中，并在不传 `runtime_bin` 的情况下运行。Linux 还会检查可执行文件和原生 addon 各自的 GLIBC 依赖，并在 manylinux 2.28 容器中运行；macOS 则验证可执行文件的部署目标符合 wheel 包标签。完整构建三个目标时保留 4 个产物，每个产物只含一个发布文件：平台无关的 SDK wheel 包与 3 个原生运行时 wheel 包；手动选择部分目标时保留 SDK wheel 与所选运行时 wheel。裸 exe 与源码包只作为测试中间输入。[`.gitlab-ci.yml`](../../../../.gitlab-ci.yml) 只接受版本与根目录 `package.json` 匹配的 `python-v` 标签流水线，构建一个 SDK wheel 包和 3 个原生运行时 wheel 包，再由单个串行任务校验并将这 4 个文件发布到项目的 PyPI 注册表。Windows 不在目标范围内。 +CI 使用 [`.github/workflows/build-exe-for-python-sdk.yml`](../../../../.github/workflows/build-exe-for-python-sdk.yml)：[安装后 wheel Python 运行时拉取请求验证](../testing/2026-08-23-installed-python-wheel-black-box-ci.zh.md)与[公开发布工作流](../process/2026-08-11-python-publication-workflow.zh.md)都会调用它构建全部三个目标；`workflow_dispatch` 与 `build-exe` 标签仍可选择部分目标。linux-x64、linux-arm64（`ubuntu-24.04-arm`）和 macos-arm64 三个平台分别进行原生构建，并缓存 `~/.pkg-cache`；macOS 的 ad-hoc 签名由 pkg 处理。每个平台都把发布形态的 SDK wheel 包与运行时 wheel 包安装到 checkout 外的干净 venv，证明包与可执行文件来源，再通过公开 SDK 与直接 NDJSON JSON-RPC 运行完整 keyless 场景。可信拉取请求还会在每个目标上运行真实 DeepSeek 双轮工具冒烟测试；fork 与 Dependabot head 不会获得密钥。Linux 会检查可执行文件和原生 addon 各自的 GLIBC 依赖，并额外运行 manylinux 2.28 冒烟测试；macOS 则验证可执行文件的部署目标符合 wheel 包标签。完整构建三个目标时保留 4 个产物，每个产物只含一个发布文件：平台无关的 SDK wheel 包与 3 个原生运行时 wheel 包；手动选择部分目标时保留 SDK wheel 与所选运行时 wheel。裸 exe 与源码包只作为测试中间输入。[`.gitlab-ci.yml`](../../../../.gitlab-ci.yml) 只接受版本与根目录 `package.json` 匹配的 `python-v` 标签流水线，构建一个 SDK wheel 包和 3 个原生运行时 wheel 包，再由单个串行任务校验并将这 4 个文件发布到项目的 PyPI 注册表。Windows 不在目标范围内。 ### Python SDK 分发：双载体，exe 用于生产，`node` 用于开发 @@ -64,7 +64,7 @@ exe 内支持 `dsh-workflow-worker-thread` 与 `dsh-code-runtime-worker-thread` ## 测试 -验证面分三层。机制层：`--sea` 链路的实测结论内嵌在「决策」各节（VFS 内 ESM 动态 `import()`、单一 Cordis 实例、明确报错的配置链路、`node:sqlite`、macOS ad-hoc 签名可运行）。SDK 层：完整的无密钥 pytest 套件以 mock 运行时对端覆盖客户端协议、子进程清理、绝对 `cwd` 传递、双载体启动与载体解析；根 CI 在 Python 3.10 上运行全部用例。端到端层：每个平台构建都通过默认 SDK 路径、自定义配置、仓库内置的独立 minimal 组合和直接二进制协议，对 mock 端点完成一个轮次，并校验最终文本与 JSONL。minimal 运行会断言其精确系统提示词与双工具目录，跨调用保留 Bash 状态，并调用编辑器。自定义配置还会通过打包进 VFS 的真实工作线程文件执行 `run_code` 和不启动 agent 的 `workflow`。文件系统搜索场景要求模型通过目标平台的 `-rg` 伴随文件调用 `glob` 与 `grep`。MCP 场景会启动临时外部 stdio server，刻意延迟首次 `tools/list` 响应，随后立即启动第一个 SDK 提示词；该提示词必须看到并调用已发现的工具，从而证明 `initialize` 是真正以 Loader 插件树完全稳定为准的就绪边界，而不是依赖定时 sleep。同一构建任务还会经 Python SDK 运行一组检入的 exe 专用快照：无密钥脚本化模型挂载一个会注册工具的 Cordis 插件，从 `run_code` 调用该工具，运行一个直接 spawn 的 subagent 和一个会通过 spawn 启动第二个 subagent 的工作流，随后卸载该插件。该 fixture（测试前置数据）会显式禁用组合包中未使用的 Bash 和本地 skill（技能）发现，使其工具集不依赖仓库外部状态；比较时会规范化 SDK 结果与通知流，以及父会话和两个子会话 JSONL 日志中不透明的消息、agent、工作流运行与会话 ID。该 harness 与 ACP 的 `pnpm run test:snapshot` 保持独立，因为二者的协议和构建产物不同。随后把平台 wheel 包安装进干净的 venv，并在不传 `runtime_bin` 的情况下运行。 +验证面分三层。机制层：`--sea` 链路的实测结论内嵌在「决策」各节（VFS 内 ESM 动态 `import()`、单一 Cordis 实例、明确报错的配置链路、`node:sqlite`、macOS ad-hoc 签名可运行）。SDK 层：完整的无密钥 pytest 套件以 mock 运行时对端覆盖客户端协议、子进程清理、绝对 `cwd` 传递、双载体启动与载体解析；根 CI 在 Python 3.10 上运行全部用例。端到端层：每个平台构建都会把两个 wheel 包安装进 checkout 外的干净 venv，证明版本相同以及已安装模块／可执行文件的位置，再通过默认 SDK 路径、自定义配置、仓库内置的独立 minimal 组合和直接二进制协议，对 mock 端点完成轮次，并校验最终文本与 JSONL。minimal 运行会断言其精确系统提示词与双工具目录，跨调用保留 Bash 状态，并调用编辑器。自定义配置还会通过打包进 VFS 的真实工作线程文件执行 `run_code` 和不启动 agent 的 `workflow`。文件系统搜索场景要求模型通过目标平台的 `-rg` 伴随文件调用 `glob` 与 `grep`。MCP 场景会启动临时外部 stdio server，刻意延迟首次 `tools/list` 响应，随后立即启动第一个 SDK 提示词；该提示词必须看到并调用已发现的工具，从而证明 `initialize` 是真正以 Loader 插件树完全稳定为准的就绪边界，而不是依赖定时 sleep。同一项安装后运行还会经 Python SDK 比较一组检入的 exe 专用快照：无密钥脚本化模型挂载一个会注册工具的 Cordis 插件，从 `run_code` 调用该工具，运行一个直接 spawn 的 subagent 和一个会通过 spawn 启动第二个 subagent 的工作流，随后卸载该插件。该 fixture（测试前置数据）会显式禁用组合包中未使用的 Bash 和本地 skill（技能）发现，使其工具集不依赖仓库外部状态；比较时会规范化 SDK 结果与通知流，以及父会话和两个子会话 JSONL 日志中不透明的消息、agent、工作流运行与会话 ID。可信拉取请求会增加真实提供方双轮文件写入／读取，并要求外部字节、工具调用、已完成原因与持久化日志一致。该 harness 与 ACP 的 `pnpm run test:snapshot` 保持独立，因为二者的协议和构建产物不同。 手工驱动注意：`bin` 将 stdin EOF 视为「客户端已离开」并立即 dispose，生命周期较短的管道会中止进行中的轮次——管道驱动必须保持 stdin 打开，直到轮次结束。 diff --git a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.i18n.yaml b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.i18n.yaml index 0bfe6c03b5..3f61c0bb86 100644 --- a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.md -2026-07-23-portable-required-pull-request-ci.md: 6520a16fb4aa5f03e364a17392c87fe0df459ea1 -2026-07-23-portable-required-pull-request-ci.zh.md: cf57ae0d409fca750b5e33e533b8650c747f1abf +2026-07-23-portable-required-pull-request-ci.md: 00a58136b8e6d5a2f282bede9876d2f96e6ddf52 +2026-07-23-portable-required-pull-request-ci.zh.md: e367408850efe97457f4921150d39a1cfe34aed3 diff --git a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.md b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.md index 6520a16fb4..00a58136b8 100644 --- a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.md +++ b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.md @@ -12,7 +12,7 @@ Billing health, a runner definition's `Ready` state, and a large autoscaling cei ## Decision -[CI](../../../../.github/workflows/ci.yml) (pull-request-only) runs the required primary Node 24 jobs, plus the stable `all checks passed` aggregate, on repo-restricted enterprise 32-core pools. The aggregate performs no checkout or repository gate, but sharing the enterprise pool prevents the required verdict from introducing a separate standard-hosted billing dependency after its substantive jobs have already succeeded. The required Windows job runs Windows Node under Wine on standard `ubuntu-latest` for the blocking surfaces; an independent native `windows-2025` job starts automatically but does not participate in the aggregate ([dual Windows decision](2026-08-08-native-windows-pull-request-ci.md)). Standard `ubuntu-latest` jobs retain Node 22.19, Node 26, the Python SDK unit suite, and the [release-shaped Linux x64 Python runtime validation](../testing/2026-08-12-required-python-runtime-pull-request-ci.md), while the serial references (in `ci-master.yml`) remain the complete unsharded cross-platform definitions. Those standard-hosted jobs keep the portable execution boundary observable without duplicating the primary inventory on every pull request. +[CI](../../../../.github/workflows/ci.yml) (pull-request-only) runs the required primary Node 24 jobs, plus the stable `all checks passed` aggregate, on repo-restricted enterprise 32-core pools. The aggregate performs no checkout or repository gate, but sharing the enterprise pool prevents the required verdict from introducing a separate standard-hosted billing dependency after its substantive jobs have already succeeded. The required Windows job runs Windows Node under Wine on standard `ubuntu-latest` for the blocking surfaces; an independent native `windows-2025` job starts automatically but does not participate in the aggregate ([dual Windows decision](2026-08-08-native-windows-pull-request-ci.md)). Standard-hosted jobs retain Node 22.19, Node 26, the Python SDK unit suite, and [installed-wheel Python runtime validation](../testing/2026-08-23-installed-python-wheel-black-box-ci.md) on every published native target, while the serial references (in `ci-master.yml`) remain the complete unsharded cross-platform definitions. Those standard-hosted jobs keep the portable execution boundary observable without duplicating the primary inventory on every pull request. The three Linux primary jobs, Node compatibility, Python SDK unit suite, Python runtime validation, and `windows node 24 / wine blocking` remain dependencies of `all checks passed`; `windows node 24 / native complete` is deliberately absent. Branch protection continues to require `e2e` and `all checks passed`. There is no automatic fallback when a remaining enterprise Linux label cannot allocate: the standard jobs continue to report their own contracts, but they cannot manufacture the missing required result. diff --git a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.zh.md b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.zh.md index cf57ae0d40..e367408850 100644 --- a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.zh.md +++ b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.zh.md @@ -12,7 +12,7 @@ Status: implemented ## 决策 -[CI](../../../../.github/workflows/ci.yml)（仅 pull request）在仅限本仓库使用的企业级 32 核运行器池上运行必需的主 Node 24 作业，以及稳定的 `all checks passed` 聚合流程。该聚合流程不执行代码检出或仓库门禁；但让它与所依赖的实质性作业共用企业级运行器池，可以避免这些作业已经成功后，必需判定结果又引入一项单独的标准托管计费依赖。必需的 Windows 作业在标准 `ubuntu-latest` 上通过 Wine 运行 Windows Node，覆盖阻断性检查范围；一个独立的原生 `windows-2025` 作业会自动启动，但不参与聚合流程（[双 Windows 决策](2026-08-08-native-windows-pull-request-ci.zh.md)）。标准 `ubuntu-latest` 作业保留 Node 22.19、Node 26、Python SDK 单元测试套件与[发布形态的 Linux x64 Python 运行时验证](../testing/2026-08-12-required-python-runtime-pull-request-ci.zh.md)，串行参考流程（在 `ci-master.yml` 中）仍是完整且未分片的跨平台定义。这些标准托管作业让可移植执行边界保持可观测，而不必在每个拉取请求中重复主清单。 +[CI](../../../../.github/workflows/ci.yml)（仅 pull request）在仅限本仓库使用的企业级 32 核运行器池上运行必需的主 Node 24 作业，以及稳定的 `all checks passed` 聚合流程。该聚合流程不执行代码检出或仓库门禁；但让它与所依赖的实质性作业共用企业级运行器池，可以避免这些作业已经成功后，必需判定结果又引入一项单独的标准托管计费依赖。必需的 Windows 作业在标准 `ubuntu-latest` 上通过 Wine 运行 Windows Node，覆盖阻断性检查范围；一个独立的原生 `windows-2025` 作业会自动启动，但不参与聚合流程（[双 Windows 决策](2026-08-08-native-windows-pull-request-ci.zh.md)）。标准托管 job 保留 Node 22.19、Node 26、Python SDK 单元测试套件，并在每个已发布原生目标上运行[安装后 wheel Python 运行时验证](../testing/2026-08-23-installed-python-wheel-black-box-ci.zh.md)；串行参考流程（在 `ci-master.yml` 中）仍是完整且未分片的跨平台定义。这些标准托管作业让可移植执行边界保持可观测，而不必在每个拉取请求中重复主清单。 三项 Linux 主作业、Node 兼容性、Python SDK 单元测试套件、Python 运行时验证和 `windows node 24 / wine blocking` 继续作为 `all checks passed` 的依赖项；`windows node 24 / native complete` 被刻意排除。分支保护继续要求 `e2e` 和 `all checks passed`。剩余的企业级 Linux 运行器标签无法分配运行器时没有自动后备机制：标准作业会继续报告各自的约定，但无法产出缺失的必需结果。 diff --git a/.agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.i18n.yaml b/.agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.i18n.yaml new file mode 100644 index 0000000000..93a20e387f --- /dev/null +++ b/.agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.md +2026-08-23-installed-python-wheel-black-box-ci.md: f2b5bd0edeb02a5d72e8010c62c3cfb59ee95c5d +2026-08-23-installed-python-wheel-black-box-ci.zh.md: fb0f5fb2da676f1a24a2630bd45f005f46a3095e diff --git a/.agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.md b/.agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.md new file mode 100644 index 0000000000..f2b5bd0ede --- /dev/null +++ b/.agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.md @@ -0,0 +1,51 @@ +# Agent Note: Installed-wheel Python runtime pull-request validation + +Status: implemented + +English | [中文](2026-08-23-installed-python-wheel-black-box-ci.zh.md) + +## Problem + +The Python SDK unit suite drives fake peers, while the packaged-runtime workflow can run the source SDK against a newly built executable before either Python distribution exists. Its clean virtual environment exercises only the default and MCP cases, and required pull-request CI builds only Linux x64. A source checkout, editable install, mismatched SDK/runtime pair, broken native wheel, platform-specific closure, or real-provider integration can therefore escape the evidence that blocks a merge. + +## Decision + +### Installed artifact boundary + +The required Python runtime workflow builds the pure SDK wheel and each platform runtime wheel before behavior tests. Every native target installs those two local files into a new Python 3.10 virtual environment, changes to a temporary directory outside the repository, unsets `PYTHONPATH` and `DSH_RUNTIME_MODE`, and invokes only the public Python modules plus the packaged executable. + +The black-box harness rejects a non-venv process, repository-relative working directory, source or editable import, unequal distribution versions, an SDK dependency that does not exactly pin the runtime version, an executable outside the installed runtime package, or an executable absent from the runtime distribution record. This provenance check runs before the first agent request, so a behavior pass cannot conceal that the wrong code ran. + +### Keyless behavior + +Every target runs the complete packaged-runtime scenario set after installation. A local SSE model keeps outputs deterministic while the public SDK exercises the default configuration, an external complete configuration, persistent PTY and editor behavior, worker-thread code and workflow execution, ripgrep-backed search, external stdio MCP discovery and execution, model-visible and durable snapshots, JSONL/Zstandard persistence, direct JSON-RPC, and shutdown. A restart snapshot launches two complete SDK runtime processes against one persistence root and pins their isolated model histories, high-level results, and separate durable logs. The installed run replaces the source-SDK pre-wheel run; the executable and wheel are tested together once rather than maintaining two behavior inventories. + +Linux additionally retains its manylinux 2.28 clean-install smoke and GLIBC checks. macOS retains deployment-target and native helper checks. These platform constraints supplement the common black-box behavior rather than substituting for it. + +### Real DeepSeek API + +Trusted pull requests run a second installed-wheel check on every native target with `DEEPSEEK_API_KEY_EXTERNAL`, mapped only into a preflight and the live test step. The preflight fails when the secret is empty, so the provider suite cannot self-skip to green. The test starts the public SDK against `https://api.deepseek.com`, asks the model to write an exact sentinel file through Bash, asks a second turn in the same session to read it, and verifies the external bytes, final responses, completed turn reasons, model-requested tool calls, and the existence and Zstandard framing of its session log. Decoded record content and completed-turn durability are deterministic keyless obligations owned by the restart snapshot rather than inferred from compressed live-provider bytes. + +Fork and Dependabot pull requests never receive the repository secret. Their native jobs run the complete keyless path and skip both secret-bearing steps; `pull_request_target` is forbidden because it would execute untrusted code with the key. + +### Required targets + +The pull-request `python-runtime` job calls the reusable builder for Linux x64, Linux arm64, and macOS arm64. Its aggregate result remains a dependency of `all checks passed`, so a failed, cancelled, or missing native carrier blocks the required verdict. Windows has no runtime wheel in the platform manifest and is not claimed by this decision. + +## Existing decisions and supersession + +This decision supersedes the single-target topology in the archived [required Python runtime pull-request validation](../../archived/testing/2026-08-12-required-python-runtime-pull-request-ci.md) while retaining its requirement that the real executable, snapshots, wheels, and clean installation meet before merge. The [single-file Python SDK runtime distribution](../architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md) remains authoritative for SEA packaging, the closed dependency set, native sidecars, wheel tags, and release artifacts. + +## Alternatives considered + +**Keep Linux x64 as the only required carrier.** Rejected because native addons, executable construction, wheel tags, and helper files differ across the three published targets. Release-time discovery is too late for an artifact that every Python SDK installation selects by platform. + +**Run full behavior before wheel construction and keep two small installed smokes.** Rejected because that proves the executable against source imports, then proves too little through the distribution users install. The clean installed environment is the stronger common location for the same scenarios. + +**Use keyless model emulation only.** Rejected because a local SSE endpoint cannot prove authentication, request compatibility, streaming, tool-call interpretation, or a complete turn against the real provider. + +**Expose the key to forked pull requests through `pull_request_target`.** Rejected because arbitrary fork code could exfiltrate the repository secret. Missing credentialed evidence on an untrusted ref is explicit and security-preserving; trusted heads and post-merge provider CI retain the live signal. + +## Consequences + +Every pull request pays for three native executable and wheel builds plus deterministic installed-artifact scenarios. Trusted same-repository pull requests also pay for one two-turn DeepSeek task per target. In exchange, the required result describes the files Python users install, proves every published carrier before merge, and cannot pass by importing the checkout or silently skipping the real provider. diff --git a/.agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.zh.md b/.agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.zh.md new file mode 100644 index 0000000000..fb0f5fb2da --- /dev/null +++ b/.agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.zh.md @@ -0,0 +1,51 @@ +# Agent Note: 安装后 Python wheel 黑盒拉取请求验证 + +Status: implemented + +[English](2026-08-23-installed-python-wheel-black-box-ci.md) | 中文 + +## Problem + +Python SDK 单元测试驱动 fake peer，而打包运行时工作流可以在两个 Python distribution 尚未生成时，用源码 SDK 驱动新构建的可执行文件。干净虚拟环境只覆盖默认与 MCP 场景，必需的拉取请求 CI 也只构建 Linux x64。因此，源码 checkout、editable install、不匹配的 SDK／运行时组合、损坏的原生 wheel 包、平台相关闭包或真实提供方集成都可能绕过阻止合并的证据。 + +## Decision + +### 安装产物边界 + +必需的 Python 运行时工作流先构建纯 SDK wheel 包与各平台运行时 wheel 包，再进行行为测试。每个原生目标都把这两个本地文件安装进新的 Python 3.10 虚拟环境，切换到仓库外的临时目录，清除 `PYTHONPATH` 与 `DSH_RUNTIME_MODE`，并且只调用公开 Python 模块与打包后的可执行文件。 + +黑盒测试会拒绝非 venv 进程、仓库内工作目录、源码或 editable import、不相等的 distribution 版本、未精确固定运行时版本的 SDK 依赖、位于已安装运行时包之外的可执行文件，以及未出现在运行时 distribution 记录中的可执行文件。该来源校验发生在首个 agent 请求之前，因此行为通过也不能掩盖实际运行了错误代码。 + +### Keyless 行为 + +每个目标都会在安装后运行完整的打包运行时场景。一个本地 SSE mock 模型提供确定性输出，公开 SDK 则覆盖默认配置、外部完整配置、持久 PTY 与 editor 行为、worker thread 代码与 workflow 执行、基于 ripgrep 的搜索、外部 stdio MCP 发现与执行、模型可见及持久化快照、JSONL／Zstandard 持久化、直接 JSON-RPC 与关闭。Restart 快照针对同一持久化根目录启动两个完整 SDK 运行时进程，并固定其彼此隔离的模型历史、高层结果与独立持久日志。安装后运行取代 wheel 构建前的源码 SDK 运行，因此可执行文件与 wheel 包共同接受一次验证，而不是维护两套行为清单。 + +Linux 另外保留 manylinux 2.28 干净安装冒烟测试与 GLIBC 检查。macOS 保留部署目标与原生 helper 检查。这些平台约束补充共同黑盒行为，不能替代它。 + +### 真实 DeepSeek API + +可信拉取请求会在每个原生目标上运行第二项安装后 wheel 检查，并且只在预检与 live 测试步骤中把 `DEEPSEEK_API_KEY_EXTERNAL` 映射进去。密钥为空时预检失败，因此提供方测试不能通过自行 skip 产生假绿。该测试通过公开 SDK 访问 `https://api.deepseek.com`，要求模型通过 Bash 写入内容精确的 sentinel 文件，再在同一 session 的第二个轮次中读取它，并校验外部文件字节、最终响应、已完成的轮次结束原因、模型请求的工具调用，以及 session 日志存在且采用 Zstandard framing。解码后的记录内容与已完成轮次的持久性是由 restart 快照负责的确定性 keyless 要求，不从压缩后的 live 提供方字节推断。 + +Fork 与 Dependabot 拉取请求永远不会获得仓库密钥。它们的原生 job 运行完整 keyless 路径并跳过两个带密钥的步骤；禁止使用 `pull_request_target`，因为它会让不可信代码带着密钥执行。 + +### 必需目标 + +拉取请求的 `python-runtime` job 会针对 Linux x64、Linux arm64 与 macOS arm64 调用可复用构建器。其聚合结果仍是 `all checks passed` 的依赖项，因此任一原生载体失败、取消或缺失都会阻止必需判定通过。Windows 不在运行时平台 manifest 中，本决策不声称支持它。 + +## Existing decisions and supersession + +本决策取代已归档的[必需 Python 运行时拉取请求验证](../../archived/testing/2026-08-12-required-python-runtime-pull-request-ci.md)中的单目标拓扑，同时保留真实可执行文件、快照、wheel 包与干净安装必须在合并前相遇的要求。[单文件 Python SDK 运行时 distribution](../architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.zh.md)仍负责 SEA 打包、封闭依赖集合、原生 sidecar、wheel 包标签与发布产物。 + +## Alternatives considered + +**只保留 Linux x64 必需载体。** 否决：三个已发布目标的原生 addon、可执行文件构建、wheel 包标签与 helper 文件不同。等到发布时才发现问题，对每个 Python SDK 安装都会按平台选择的产物而言太晚。 + +**在 wheel 构建前运行完整行为，并保留两个很小的安装后冒烟测试。** 否决：这只能证明可执行文件配合源码 import 工作，再通过 distribution 证明很少的行为。干净安装环境是在同一批场景中验证用户实际安装内容的更强位置。 + +**只使用 keyless 模型模拟。** 否决：本地 SSE endpoint 不能证明真实提供方的认证、请求兼容性、流式输出、工具调用解释或完整轮次。 + +**通过 `pull_request_target` 向 fork 拉取请求暴露密钥。** 否决：任意 fork 代码都可以窃取仓库密钥。不可信 ref 缺少带凭据证据是明确且保留安全性的结果；可信 head 与合并后提供方 CI 继续提供 live 信号。 + +## Consequences + +每个拉取请求都会承担三个原生可执行文件及 wheel 包构建，并运行确定性的安装后产物场景。可信的同仓库拉取请求还会在每个目标上承担一次双轮 DeepSeek 任务。相应地，必需结果描述 Python 用户实际安装的文件，在合并前证明每个已发布载体，并且不能通过导入 checkout 或静默跳过真实提供方而通过。 diff --git a/.github/workflows/build-exe-for-python-sdk.yml b/.github/workflows/build-exe-for-python-sdk.yml index ba17869f29..b9ba5e148e 100644 --- a/.github/workflows/build-exe-for-python-sdk.yml +++ b/.github/workflows/build-exe-for-python-sdk.yml @@ -21,10 +21,14 @@ on: required: false default: false ci: - description: Run as the required Linux x64 Python runtime pull-request check. + description: Run as the required all-target Python runtime pull-request check. type: boolean required: false default: false + secrets: + DEEPSEEK_API_KEY_EXTERNAL: + description: Real DeepSeek API key for trusted installed-wheel pull-request tests. + required: false workflow_dispatch: inputs: targets: @@ -243,13 +247,6 @@ jobs: echo "exe=$exe" >> "$GITHUB_OUTPUT" echo "wheel=$wheel" >> "$GITHUB_OUTPUT" - - name: Full-turn SDK, executable snapshot, and direct-binary smoke - run: >- - uv run --python 3.10 --group test --project python/sdk - python scripts/smoke-python-runtime.py - --scenario all - --exe "${{ steps.runtime.outputs.exe }}" - - name: Build release-shaped runtime wheel run: >- python scripts/build-python-release.py @@ -273,10 +270,53 @@ jobs: "$RUNNER_TEMP/dsh-sdk-smoke/bin/python" -m pip install \ "dist-python/$SDK_WHEEL" \ "dist-python/$RUNTIME_WHEEL" - "$RUNNER_TEMP/dsh-sdk-smoke/bin/python" scripts/smoke-python-runtime.py \ - --scenario sdk-default - "$RUNNER_TEMP/dsh-sdk-smoke/bin/python" scripts/smoke-python-runtime.py \ - --scenario sdk-mcp + + - name: Run installed-wheel keyless black-box tests + run: | + set -euo pipefail + blackbox_root="$RUNNER_TEMP/dsh-sdk-blackbox" + mkdir -p "$blackbox_root" + cd "$blackbox_root" + env -u PYTHONPATH -u DSH_RUNTIME_MODE \ + "$RUNNER_TEMP/dsh-sdk-smoke/bin/python" \ + "$GITHUB_WORKSPACE/scripts/smoke-python-runtime.py" \ + --scenario all \ + --installed-wheel + + - name: Preflight installed-wheel real API test + if: >- + inputs.ci + && (github.event_name != 'pull_request' + || !(github.event.pull_request.head.repo.fork + || github.event.pull_request.user.login == 'dependabot[bot]')) + env: + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }} + run: | + set -euo pipefail + if [ -z "${DEEPSEEK_API_KEY:-}" ]; then + echo "::error::DEEPSEEK_API_KEY_EXTERNAL is empty; the installed-wheel real API test cannot self-skip." + exit 1 + fi + + - name: Run installed-wheel real API black-box test + if: >- + inputs.ci + && (github.event_name != 'pull_request' + || !(github.event.pull_request.head.repo.fork + || github.event.pull_request.user.login == 'dependabot[bot]')) + env: + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }} + DEEPSEEK_BASE_URL: https://api.deepseek.com + run: | + set -euo pipefail + blackbox_root="$RUNNER_TEMP/dsh-sdk-blackbox-live" + mkdir -p "$blackbox_root" + cd "$blackbox_root" + env -u PYTHONPATH -u DSH_RUNTIME_MODE \ + "$RUNNER_TEMP/dsh-sdk-smoke/bin/python" \ + "$GITHUB_WORKSPACE/scripts/smoke-python-runtime.py" \ + --scenario sdk-live \ + --installed-wheel - name: Check Linux GLIBC requirements if: runner.os == 'Linux' @@ -314,8 +354,10 @@ jobs: docker run --rm -e RUNTIME_WHEEL -e SDK_WHEEL -e DSH_TELEMETRY_DISABLED -v "$PWD:/work" -w /work "$image" bash -euxo pipefail -c ' /opt/python/cp310-cp310/bin/python -m venv /tmp/dsh-sdk /tmp/dsh-sdk/bin/python -m pip install "/work/dist-python/$SDK_WHEEL" "/work/dist-python/$RUNTIME_WHEEL" - /tmp/dsh-sdk/bin/python /work/scripts/smoke-python-runtime.py --scenario sdk-default - /tmp/dsh-sdk/bin/python /work/scripts/smoke-python-runtime.py --scenario sdk-mcp + mkdir -p /tmp/dsh-sdk-manylinux-smoke + cd /tmp/dsh-sdk-manylinux-smoke + env -u PYTHONPATH -u DSH_RUNTIME_MODE /tmp/dsh-sdk/bin/python /work/scripts/smoke-python-runtime.py --scenario sdk-default --installed-wheel + env -u PYTHONPATH -u DSH_RUNTIME_MODE /tmp/dsh-sdk/bin/python /work/scripts/smoke-python-runtime.py --scenario sdk-mcp --installed-wheel ' - uses: actions/upload-artifact@v7 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e540c1cb15..1f02c2919f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -294,17 +294,18 @@ jobs: - name: Run complete keyless Python suite run: uv run --python 3.10 --group test --project python/sdk pytest - # One native target makes the complete release-shaped Python path required - # without duplicating platform-independent behavior across the release matrix. - # The reusable builder owns the executable, snapshot, wheel, clean-install, - # GLIBC, and manylinux checks; release validation retains all native targets. + # The reusable builder owns each published executable, wheel, clean-install, + # keyless black-box, and trusted real-API path. All native release targets are + # required because a platform wheel cannot be validated by another carrier. python-runtime: if: github.event_name == 'pull_request' - name: python runtime / release-shaped Linux x64 + name: python runtime / release-shaped matrix uses: ./.github/workflows/build-exe-for-python-sdk.yml with: - targets: node24-linux-x64 + targets: node24-linux-x64,node24-linux-arm64,node24-macos-arm64 ci: true + secrets: + DEEPSEEK_API_KEY_EXTERNAL: ${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }} # The pull-request Windows signals cover complementary hosts. The two fast # win32 toolchain surfaces (workspace build, production site) execute with diff --git a/python/development.i18n.yaml b/python/development.i18n.yaml index 0e515656a3..75c657c153 100644 --- a/python/development.i18n.yaml +++ b/python/development.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write python/development.md -development.md: 6d326a1ac273e9594cbea4714e1bc9449f5e5ca7 -development.zh.md: d07ce7f95a195ea0fe24fa44529dc53b527c766a +development.md: e96be7af10e0008cc0fe5dea4ca51529f102fd7b +development.zh.md: e4ca1c980c36fdea381e9b8a6c615276e279b4d2 diff --git a/python/development.md b/python/development.md index 6d326a1ac2..e96be7af10 100644 --- a/python/development.md +++ b/python/development.md @@ -27,14 +27,16 @@ uv run --project python/sdk pytest `python/sdk/tests/test_bundled_runtime.py` exercises available bundled carriers and skips a carrier when its artifact has not been built. For repository-wide test policy, see [Testing](../docs/testing.md). -That suite drives fake runtime peers. `scripts/smoke-python-runtime.py` drives the real packaged runtime instead, and the required `python-runtime` CI job runs every scenario against a freshly built executable: +That suite drives fake runtime peers. `scripts/smoke-python-runtime.py` drives the packaged runtime instead. The required `python-runtime` CI job builds every published native target, installs the matching SDK and runtime wheels into a new Python 3.10 virtual environment, runs outside the checkout with `PYTHONPATH` and `DSH_RUNTIME_MODE` unset, proves that both modules and the executable came from those distributions, and then runs every keyless scenario. A focused local source-SDK run can select one built executable and scenario: ```sh uv run --project python/sdk python scripts/smoke-python-runtime.py \ --scenario sdk-minimal --exe dist-exe/dsh-jsonrpc-agent-pkg-macos-arm64 ``` -Two scenarios compare committed expected output under `scripts/snapshots/python-sdk-single-exe/`. `minimal/model-visible.json` pins the checked-in minimal composition's assembled system prompts, advertised tool schemas, and model-visible messages, so a plugin that contributes an unintended system section or user message fails the job; it drops the dynamic runtime-context snapshot, which the same composition emits on macOS and not on Linux ([#2488](https://github.com/deepseek-harness/deepseek-harness/issues/2488)). `advanced/` pins the SDK result and the persisted session logs. Rerun the owning scenario with `--update-snapshots` and review that diff before committing it. +Three scenarios compare committed expected output under `scripts/snapshots/python-sdk-single-exe/`. `minimal/model-visible.json` pins the checked-in minimal composition's assembled system prompts, advertised tool schemas, and model-visible messages, so a plugin that contributes an unintended system section or user message fails the job; it drops the dynamic runtime-context snapshot, which the same composition emits on macOS and not on Linux ([#2488](https://github.com/deepseek-harness/deepseek-harness/issues/2488)). `advanced/` pins one complex process's SDK result and parent/child session logs. `restart/` launches two complete SDK runtime processes against one persistence root and snapshots their isolated model histories, high-level results, and separate durable logs. Rerun the owning scenario with `--update-snapshots` and review that diff before committing it. + +Trusted pull requests also run `--scenario sdk-live --installed-wheel` on every native target. That scenario performs two tool-using turns against `https://api.deepseek.com`, verifies the created file externally, and fails when the repository secret is absent instead of self-skipping. Fork and Dependabot pull requests run the complete keyless installed-wheel path but receive no key. An interactive smoke test needs `DEEPSEEK_API_KEY` in the environment or repository-root `.env`: diff --git a/python/development.zh.md b/python/development.zh.md index d07ce7f95a..e4ca1c980c 100644 --- a/python/development.zh.md +++ b/python/development.zh.md @@ -27,14 +27,16 @@ uv run --project python/sdk pytest `python/sdk/tests/test_bundled_runtime.py` 会运行可用的内置载体；某个载体的产物尚未构建时，会跳过该载体。仓库级测试政策见 [测试](../docs/testing.zh.md)。 -该套件面向的是伪造的运行时对端。`scripts/smoke-python-runtime.py` 面向真实的打包运行时；必需的 `python-runtime` CI 任务会用新构建的可执行文件运行全部场景： +该套件面向的是伪造的运行时对端。`scripts/smoke-python-runtime.py` 面向打包运行时。必需的 `python-runtime` CI 任务会构建每个已发布原生目标，把匹配的 SDK wheel 包与运行时 wheel 包安装进新的 Python 3.10 虚拟环境，在 checkout 外清除 `PYTHONPATH` 与 `DSH_RUNTIME_MODE` 后运行，证明两个模块及可执行文件都来自这些 distribution，然后运行全部 keyless 场景。聚焦的本地源码 SDK 运行可以选择一个已构建可执行文件与场景： ```sh uv run --project python/sdk python scripts/smoke-python-runtime.py \ --scenario sdk-minimal --exe dist-exe/dsh-jsonrpc-agent-pkg-macos-arm64 ``` -其中两个场景会比对 `scripts/snapshots/python-sdk-single-exe/` 下已提交的期望输出。`minimal/model-visible.json` 固定了签入的极简组合所组装的系统提示词、对外公布的工具 schema 以及模型可见消息，因此插件一旦贡献出计划外的系统分段或 user 消息，该任务即失败；它会丢弃动态运行时上下文快照——同一组合在 macOS 上会发出它，在 Linux 上不会（[#2488](https://github.com/deepseek-harness/deepseek-harness/issues/2488)）。`advanced/` 固定 SDK 结果与持久化的会话日志。重新运行对应场景时加上 `--update-snapshots`，并在提交前审阅该差异。 +其中三个场景会比对 `scripts/snapshots/python-sdk-single-exe/` 下已提交的期望输出。`minimal/model-visible.json` 固定了签入的极简组合所组装的系统提示词、对外公布的工具 schema 以及模型可见消息，因此插件一旦贡献出计划外的系统分段或 user 消息，该任务即失败；它会丢弃动态运行时上下文快照——同一组合在 macOS 上会发出它，在 Linux 上不会（[#2488](https://github.com/deepseek-harness/deepseek-harness/issues/2488)）。`advanced/` 固定一个复杂进程的 SDK 结果及父／子会话日志。`restart/` 针对同一持久化根目录启动两个完整 SDK 运行时进程，并固定其彼此隔离的模型历史、高层结果与独立持久日志。重新运行对应场景时加上 `--update-snapshots`，并在提交前审阅该差异。 + +可信拉取请求还会在每个原生目标上运行 `--scenario sdk-live --installed-wheel`。该场景面向 `https://api.deepseek.com` 执行两个使用工具的轮次，从外部验证已创建文件，并在仓库密钥缺失时失败而不是自行 skip。Fork 与 Dependabot 拉取请求会运行完整的 keyless 安装后 wheel 路径，但不会获得密钥。 交互式冒烟测试需要环境变量或仓库根目录 `.env` 中存在 `DEEPSEEK_API_KEY`： diff --git a/scripts/ci-workflow.spec.ts b/scripts/ci-workflow.spec.ts index 1c84f28c78..116bb84e3c 100644 --- a/scripts/ci-workflow.spec.ts +++ b/scripts/ci-workflow.spec.ts @@ -218,7 +218,7 @@ describe('CI workflow', () => { expect(config).not.toContain('packages/lsp/lsp-stdio/src/instance.ts') }) - it('requires one release-shaped Python runtime target on every pull request', () => { + it('requires release-shaped Python runtime validation on every published target', () => { const workflow = loadWorkflow('.github/workflows/ci.yml') const pythonRuntime = workflowJob(workflow, 'python-runtime') const aggregate = workflowJob(workflow, 'all-checks-passed') @@ -228,12 +228,15 @@ describe('CI workflow', () => { expect(pythonRuntime).toMatchObject({ if: "github.event_name == 'pull_request'", - name: 'python runtime / release-shaped Linux x64', + name: 'python runtime / release-shaped matrix', uses: './.github/workflows/build-exe-for-python-sdk.yml', with: { - targets: 'node24-linux-x64', + targets: 'node24-linux-x64,node24-linux-arm64,node24-macos-arm64', ci: true, }, + secrets: { + DEEPSEEK_API_KEY_EXTERNAL: '${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }}', + }, }) expect(aggregate.needs).toContain('python-runtime') }) @@ -382,7 +385,7 @@ describe('Python release workflows', () => { const call = workflowEvent(workflow, 'workflow_call') const plan = workflowJob(workflow, 'plan') const build = workflowJob(workflow, 'build') - if (!isRecord(call.inputs) || !Array.isArray(plan.steps) || !Array.isArray(build.steps)) { + if (!isRecord(call.inputs) || !isRecord(call.secrets) || !Array.isArray(plan.steps) || !Array.isArray(build.steps)) { throw new TypeError('Python wheel builder must define workflow_call inputs and plan steps') } @@ -390,11 +393,20 @@ describe('Python release workflows', () => { const manylinuxAddon = buildSteps.find(step => isRecord(step) && step.name === 'Rebuild Linux node-pty against manylinux 2.28') const macosCheck = buildSteps.find(step => isRecord(step) && step.name === 'Check macOS deployment target') const manylinuxSmoke = buildSteps.find(step => isRecord(step) && step.name === 'Run wheel in a manylinux 2.28 container') + const installedKeyless = buildSteps.find(step => isRecord(step) && step.name === 'Run installed-wheel keyless black-box tests') + const realApiPreflight = buildSteps.find(step => isRecord(step) && step.name === 'Preflight installed-wheel real API test') + const installedRealApi = buildSteps.find(step => isRecord(step) && step.name === 'Run installed-wheel real API black-box test') + if (!isRecord(installedKeyless) || !isRecord(realApiPreflight) || !isRecord(installedRealApi)) { + throw new TypeError('Python wheel builder must define installed-wheel keyless and real API steps') + } expect(call.inputs).toHaveProperty('targets') expect(call.inputs).toMatchObject({ ci: { type: 'boolean', default: false }, release: { type: 'boolean', default: false }, }) + expect(call.secrets).toMatchObject({ + DEEPSEEK_API_KEY_EXTERNAL: { required: false }, + }) expect(workflow.concurrency).toMatchObject({ group: 'build-single-exe-${{ github.workflow }}-${{ github.ref }}', }) @@ -419,6 +431,26 @@ describe('Python release workflows', () => { expect(macosCheck).toMatchObject({ if: "runner.os == 'macOS'" }) expect(JSON.stringify(macosCheck)).toContain('scripts/check-macos-deployment-target.py') expect(JSON.stringify(macosCheck)).toContain('$EXE-spawn-helper') + expect(JSON.stringify(installedKeyless)).toContain('--scenario all') + expect(JSON.stringify(installedKeyless)).toContain('--installed-wheel') + expect(JSON.stringify(installedKeyless)).toContain('env -u PYTHONPATH') + expect(JSON.stringify(installedKeyless)).toContain('-u DSH_RUNTIME_MODE') + expect(realApiPreflight).toMatchObject({ + env: { DEEPSEEK_API_KEY: '${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }}' }, + }) + expect(String(realApiPreflight.if)).toContain('inputs.ci') + expect(String(realApiPreflight.if)).toContain('head.repo.fork') + expect(String(realApiPreflight.if)).toContain('dependabot[bot]') + expect(installedRealApi).toMatchObject({ + env: { + DEEPSEEK_API_KEY: '${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }}', + DEEPSEEK_BASE_URL: 'https://api.deepseek.com', + }, + }) + expect(installedRealApi.if).toBe(realApiPreflight.if) + expect(JSON.stringify(installedRealApi)).toContain('--scenario sdk-live') + expect(JSON.stringify(installedRealApi)).toContain('--installed-wheel') + expect(JSON.stringify(installedRealApi)).toContain('-u DSH_RUNTIME_MODE') expect(manylinuxSmoke).toMatchObject({ if: "runner.os == 'Linux'" }) expect(JSON.stringify(manylinuxSmoke)).toContain('-e DSH_TELEMETRY_DISABLED') }) diff --git a/scripts/smoke-python-runtime.py b/scripts/smoke-python-runtime.py index 6c4010e427..71e7100268 100644 --- a/scripts/smoke-python-runtime.py +++ b/scripts/smoke-python-runtime.py @@ -5,6 +5,8 @@ from __future__ import annotations import argparse import difflib +import importlib +import importlib.metadata import json import os import queue @@ -22,6 +24,7 @@ if TYPE_CHECKING: EXPECTED_TEXT = "runtime smoke ok" +LIVE_API_SENTINEL = "PYTHON_SDK_LIVE_OK" CODE_PROMPT = "Use run_code to compute the packaged worker smoke value." CODE_WORKER_TEXT = "code worker smoke ok" WORKFLOW_PROMPT = "Use workflow to compute the packaged worker smoke value without agents." @@ -47,6 +50,12 @@ SNAPSHOT_SESSION_ID = "advanced-executable" SNAPSHOT_DIRECT_CHILD_PROMPT = "Reply with exactly DIRECT_CHILD_OK and nothing else." SNAPSHOT_WORKFLOW_CHILD_PROMPT = "Reply with exactly WORKFLOW_CHILD_OK and nothing else." SNAPSHOT_FINAL_TEXT = "ADVANCED_EXECUTABLE_OK" +RESTART_FIRST_PROMPT = "Complete the first isolated Python SDK process turn." +RESTART_FIRST_TEXT = "PROCESS_ONE_OK" +RESTART_SECOND_PROMPT = "Complete the second isolated Python SDK process turn." +RESTART_SECOND_TEXT = "PROCESS_TWO_OK" +RESTART_FIRST_SESSION_ID = "process-one" +RESTART_SECOND_SESSION_ID = "process-two" SNAPSHOT_PLUGIN_CODE = """\ return (ctx) => { harness.registerTool(ctx, harness.defineTool({ @@ -78,6 +87,10 @@ MINIMAL_SNAPSHOT_DIRECTORY = ( Path(__file__).resolve().parent / "snapshots" / "python-sdk-single-exe" / "minimal" ) MINIMAL_SNAPSHOT_FILENAMES = ("model-visible.json",) +RESTART_SNAPSHOT_DIRECTORY = ( + Path(__file__).resolve().parent / "snapshots" / "python-sdk-single-exe" / "restart" +) +RESTART_SNAPSHOT_FILENAMES = ("result.json", "requests.json", "session.1.jsonl", "session.2.jsonl") # The agent loop's dynamic runtime-context snapshot is the one model-visible message this # expected output cannot carry: the same composition emits it on macOS and not on Linux # (deepseek-harness#2488), and the file must replay on both. Everything else is compared. @@ -349,6 +362,8 @@ def completion_chunks(body: dict[str, object]) -> list[dict[str, object]]: WORKFLOW_PROMPT, FS_SEARCH_PROMPT, MCP_PROMPT, + RESTART_FIRST_PROMPT, + RESTART_SECOND_PROMPT, } prompt = next( (candidate for candidate in user_prompts if candidate in scenario_prompts), @@ -370,6 +385,16 @@ def completion_chunks(body: dict[str, object]) -> list[dict[str, object]]: "code": {"host": SNAPSHOT_PLUGIN_CODE}, }, ) + if prompt == RESTART_FIRST_PROMPT: + return text_chunks(RESTART_FIRST_TEXT) + if prompt == RESTART_SECOND_PROMPT: + if any( + isinstance(message, dict) + and RESTART_FIRST_TEXT in message_text(message.get("content")) + for message in messages + ): + raise AssertionError("second isolated process inherited the first process history") + return text_chunks(RESTART_SECOND_TEXT) if prompt == CODE_PROMPT: assert_advertised_tool(body, "run_code") return tool_call_chunks( @@ -685,19 +710,35 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--scenario", - choices=("all", "sdk-default", "sdk-custom", "sdk-minimal", "sdk-fs-search", "sdk-mcp", "sdk-snapshot", "direct"), + choices=("all", "sdk-default", "sdk-custom", "sdk-minimal", "sdk-fs-search", "sdk-mcp", "sdk-snapshot", "sdk-restart", "sdk-live", "direct"), default="all", ) parser.add_argument("--exe", type=Path) + parser.add_argument( + "--installed-wheel", + action="store_true", + help="require a clean virtual environment containing matching installed SDK and runtime wheels", + ) parser.add_argument("--update-snapshots", action="store_true") args = parser.parse_args() - if args.scenario in {"all", "sdk-custom", "sdk-minimal", "sdk-fs-search", "sdk-snapshot", "direct"} and args.exe is None: + if args.installed_wheel and args.exe is not None: + parser.error("--installed-wheel resolves the wheel's own runtime and cannot be combined with --exe") + if args.scenario == "sdk-live" and not args.installed_wheel: + parser.error("--scenario sdk-live requires --installed-wheel") + if args.installed_wheel: + args.exe = assert_installed_wheel_environment() + if args.scenario in {"all", "sdk-custom", "sdk-minimal", "sdk-fs-search", "sdk-snapshot", "sdk-restart", "direct"} and args.exe is None: parser.error("--exe is required for custom, minimal, snapshot, and direct scenarios") - if args.update_snapshots and args.scenario not in {"all", "sdk-minimal", "sdk-snapshot"}: - parser.error("--update-snapshots requires --scenario sdk-minimal, sdk-snapshot, or all") + if args.update_snapshots and args.scenario not in {"all", "sdk-minimal", "sdk-snapshot", "sdk-restart"}: + parser.error("--update-snapshots requires --scenario sdk-minimal, sdk-snapshot, sdk-restart, or all") if args.exe is not None and not args.exe.is_file(): parser.error(f"runtime executable does not exist: {args.exe}") + if args.scenario == "sdk-live": + smoke_sdk_live() + print("smoke-python-runtime: sdk-live passed") + return + with MockModel() as model: if args.scenario in {"all", "sdk-default"}: smoke_sdk_default(model.url) @@ -715,6 +756,9 @@ def main() -> None: if args.scenario in {"all", "sdk-snapshot"}: assert args.exe is not None smoke_sdk_snapshot(model.url, args.exe.resolve(), args.update_snapshots) + if args.scenario in {"all", "sdk-restart"}: + assert args.exe is not None + smoke_sdk_restart_snapshot(model.url, args.exe.resolve(), args.update_snapshots) if args.scenario in {"all", "direct"}: assert args.exe is not None smoke_direct(model.url, args.exe.resolve()) @@ -723,6 +767,144 @@ def main() -> None: print(f"smoke-python-runtime: {args.scenario} passed") +def assert_installed_wheel_environment() -> Path: + """Prove that this process imports matching non-editable wheel installations.""" + if sys.prefix == sys.base_prefix: + raise AssertionError("installed-wheel smoke must run inside a virtual environment") + if os.environ.get("PYTHONPATH"): + raise AssertionError("installed-wheel smoke requires PYTHONPATH to be unset") + if os.environ.get("DSH_RUNTIME_MODE"): + raise AssertionError("installed-wheel smoke requires DSH_RUNTIME_MODE to be unset") + + repo_root = Path(__file__).resolve().parent.parent + cwd = Path.cwd().resolve() + if cwd.is_relative_to(repo_root): + raise AssertionError(f"installed-wheel smoke must run outside the repository, got {cwd}") + + sdk_version = importlib.metadata.version("deepseek-harness-sdk") + runtime_version = importlib.metadata.version("deepseek-harness-runtime-bin") + if sdk_version != runtime_version: + raise AssertionError( + f"installed SDK/runtime versions differ: {sdk_version} != {runtime_version}" + ) + expected_runtime_requirement = f"deepseek-harness-runtime-bin=={sdk_version}" + requirements = importlib.metadata.requires("deepseek-harness-sdk") or [] + if expected_runtime_requirement not in requirements: + raise AssertionError( + f"installed SDK does not require {expected_runtime_requirement}: {requirements}" + ) + + prefix = Path(sys.prefix).resolve() + imported: dict[str, Path] = {} + for name in ("deepseek_harness", "deepseek_harness_runtime"): + module = importlib.import_module(name) + module_file = getattr(module, "__file__", None) + if not isinstance(module_file, str): + raise AssertionError(f"installed module {name} has no filesystem location") + path = Path(module_file).resolve() + if not path.is_relative_to(prefix): + raise AssertionError(f"installed module {name} came from outside the virtual environment: {path}") + if path.is_relative_to(repo_root): + raise AssertionError(f"installed module {name} came from the repository checkout: {path}") + imported[name] = path + + runtime_module = sys.modules["deepseek_harness_runtime"] + executable = runtime_module.bundled_runtime_path().resolve() + runtime_package = imported["deepseek_harness_runtime"].parent + if not executable.is_relative_to(runtime_package): + raise AssertionError(f"bundled runtime came from outside the installed runtime wheel: {executable}") + runtime_files = importlib.metadata.files("deepseek-harness-runtime-bin") or [] + if not any(Path(file).name == executable.name for file in runtime_files): + raise AssertionError(f"runtime executable is absent from installed distribution records: {executable}") + return executable + + +def smoke_sdk_live() -> None: + """Run a real-model, tool-using two-turn task through installed wheels.""" + from deepseek_harness import DeepSeekHarness + + api_key = os.environ.get("DEEPSEEK_API_KEY") + base_url = os.environ.get("DEEPSEEK_BASE_URL") + if not api_key: + raise AssertionError("sdk-live requires DEEPSEEK_API_KEY") + if not base_url: + raise AssertionError("sdk-live requires an explicit DEEPSEEK_BASE_URL") + + with tempfile.TemporaryDirectory(prefix="dsh-sdk-live-") as temporary: + root = Path(temporary).resolve() + sessions = root / "sessions" + marker = root / "live-api-marker.txt" + session_id = "installed-wheel-live-api" + create_prompt = ( + "Use the bash tool to create the file at the absolute path below with exactly one line " + f"containing {LIVE_API_SENTINEL}. Then reply with exactly {LIVE_API_SENTINEL}.\n{marker}" + ) + verify_prompt = ( + "Use a tool to read the file created in the previous turn. " + f"If its only line is {LIVE_API_SENTINEL}, reply with exactly {LIVE_API_SENTINEL}." + ) + with DeepSeekHarness( + provider="deepseek-official", + model="deepseek-v4-flash", + cwd=str(root), + session_root=str(sessions), + api_key=api_key, + base_url=base_url, + request_timeout_seconds=180, + ) as harness: + created = harness.run(create_prompt, session_id=session_id) + verified = harness.run(verify_prompt, session_id=session_id) + + for label, result in (("create", created), ("verify", verified)): + if result.finish_reason != "completed": + event_types = [event.get("type") for event in result.events] + turn_end_data = next( + (event.get("data") for event in reversed(result.events) if event.get("type") == "turn/end"), + None, + ) + turn_end = safe_turn_end(turn_end_data) + raise AssertionError( + f"{label} turn ended with {result.finish_reason!r}; " + f"final={result.final_response!r}; turn_end={turn_end!r}; events={event_types}" + ) + if not any(event.get("type") == "tool/call" for event in result.events): + raise AssertionError( + f"{label} turn made no model-requested tool call; " + f"final={result.final_response!r}" + ) + if result.final_response.strip() != LIVE_API_SENTINEL: + raise AssertionError(f"{label} turn returned {result.final_response!r}") + if not marker.is_file(): + raise AssertionError(f"real-model tool turn did not create {marker}") + if marker.read_bytes() != f"{LIVE_API_SENTINEL}\n".encode(): + raise AssertionError(f"real-model tool turn wrote unexpected bytes to {marker}") + assert_zstd_session_log(sessions) + + +def safe_turn_end(value: object) -> object: + """Project a live-provider failure without retaining credential-bearing text.""" + if not isinstance(value, dict): + return value + reason = value.get("reason") + if not isinstance(reason, dict): + return {"turn": value.get("turn"), "reason": reason} + error = reason.get("error") + safe_error = None + if isinstance(error, dict): + safe_error = { + key: error.get(key) + for key in ("code", "status") + if error.get(key) is not None + } + return { + "turn": value.get("turn"), + "reason": { + "kind": reason.get("kind"), + **({"error": safe_error} if safe_error is not None else {}), + }, + } + + def smoke_sdk_default(base_url: str) -> None: from deepseek_harness import DeepSeekHarness @@ -915,6 +1097,61 @@ def smoke_sdk_snapshot(base_url: str, executable: Path, update_snapshots: bool) ) +def smoke_sdk_restart_snapshot(base_url: str, executable: Path, update_snapshots: bool) -> None: + """Snapshot two isolated sessions across complete SDK runtime restarts.""" + from deepseek_harness import DeepSeekHarness + + with tempfile.TemporaryDirectory(prefix="dsh-sdk-restart-") as temporary: + root = Path(temporary).resolve() + sessions = root / "sessions" + cordis = root / "cordis.yml" + cordis.write_text(CUSTOM_CORDIS) + first_request = len(MockModelHandler.requests) + + def run(prompt: str, session_id: str) -> "RunResult": + with DeepSeekHarness( + provider="deepseek-official", + model="smoke-model", + cwd=str(root), + session_root=str(sessions), + cordis=str(cordis), + runtime_bin=str(executable), + api_key="sk-keyless-smoke", + base_url=base_url, + request_timeout_seconds=60, + ) as harness: + return harness.run(prompt, session_id=session_id) + + first = run(RESTART_FIRST_PROMPT, RESTART_FIRST_SESSION_ID) + second = run(RESTART_SECOND_PROMPT, RESTART_SECOND_SESSION_ID) + requests = MockModelHandler.requests[first_request:] + if len(requests) != 2: + raise AssertionError(f"restart snapshot expected two model requests: {requests}") + if first.final_response != RESTART_FIRST_TEXT or second.final_response != RESTART_SECOND_TEXT: + raise AssertionError( + f"restart snapshot responses differ: {first.final_response!r}, {second.final_response!r}" + ) + + logs = read_session_logs(sessions) + expected_ids = {RESTART_FIRST_SESSION_ID, RESTART_SECOND_SESSION_ID} + if set(logs) != expected_ids: + raise AssertionError(f"restart snapshot expected two durable sessions: {sorted(logs)}") + for session_id, expected in ( + (RESTART_FIRST_SESSION_ID, RESTART_FIRST_TEXT), + (RESTART_SECOND_SESSION_ID, RESTART_SECOND_TEXT), + ): + records = logs[session_id] + if sum(record.get("type") == "turn/end" for record in records) != 1: + raise AssertionError(f"restart snapshot {session_id} has an unexpected turn count") + if expected not in render_jsonl(records): + raise AssertionError(f"restart snapshot durable log has no {expected}") + + files = build_restart_snapshot_files(first, second, requests, logs, root, sessions) + compare_snapshot_files( + files, update_snapshots, RESTART_SNAPSHOT_DIRECTORY, RESTART_SNAPSHOT_FILENAMES, + ) + + def smoke_direct(base_url: str, executable: Path) -> None: with tempfile.TemporaryDirectory(prefix="dsh-direct-") as temporary: root = Path(temporary).resolve() @@ -1202,6 +1439,69 @@ def build_snapshot_files( return files +def build_restart_snapshot_files( + first: "RunResult", + second: "RunResult", + requests: list[dict[str, object]], + logs: dict[str, list[dict[str, object]]], + cwd: Path, + sessions: Path, +) -> dict[str, str]: + """Render two SDK processes, isolated model histories, and durable logs.""" + replacements = [ + (str(sessions), "{{sessions}}"), + (str(cwd), "{{cwd}}"), + (RESTART_FIRST_SESSION_ID, "{{session-1}}"), + (RESTART_SECOND_SESSION_ID, "{{session-2}}"), + ] + result_value = [ + { + "session_id": result.session_id, + "final_response": result.final_response, + "finish_reason": result.finish_reason, + "eventTypes": [event.get("type") for event in result.events], + "notificationMethods": [notification.method for notification in result.notifications], + "session_root": result.session_root, + } + for result in (first, second) + ] + request_value = [ + { + "model": request.get("model"), + "messages": restart_request_messages(request), + "toolNames": sorted(advertised_tool_names(request)), + } + for request in requests + ] + return { + "result.json": json.dumps( + normalize_snapshot_value(result_value, replacements), indent=2, ensure_ascii=False, + ) + "\n", + "requests.json": json.dumps( + normalize_snapshot_value(request_value, replacements), indent=2, ensure_ascii=False, + ) + "\n", + "session.1.jsonl": render_jsonl(project_session_snapshot([ + normalize_snapshot_value(record, replacements) for record in logs[RESTART_FIRST_SESSION_ID] + ])), + "session.2.jsonl": render_jsonl(project_session_snapshot([ + normalize_snapshot_value(record, replacements) for record in logs[RESTART_SECOND_SESSION_ID] + ])), + } + + +def restart_request_messages(request: dict[str, object]) -> list[object]: + """Project model history while tokenizing composition-owned system prose.""" + messages = request.get("messages") + if not isinstance(messages, list): + raise AssertionError(f"restart snapshot request has no messages: {request}") + return [ + {"role": "system", "content": "{{system}}"} + if isinstance(message, dict) and message.get("role") == "system" + else message + for message in messages + ] + + def snapshot_workflow_run_id(result: "RunResult") -> str: """Return the one workflow run id emitted by the advanced scenario.""" run_ids: set[str] = set() diff --git a/scripts/snapshots/python-sdk-single-exe/restart/requests.json b/scripts/snapshots/python-sdk-single-exe/restart/requests.json new file mode 100644 index 0000000000..dd5f925294 --- /dev/null +++ b/scripts/snapshots/python-sdk-single-exe/restart/requests.json @@ -0,0 +1,58 @@ +[ + { + "model": "smoke-model", + "messages": [ + { + "role": "system", + "content": "{{system}}" + }, + { + "role": "user", + "content": "Complete the first isolated Python SDK process turn." + } + ], + "toolNames": [ + "cordis_define", + "cordis_inspect_list", + "cordis_inspect_query", + "cordis_inspect_self", + "cordis_run", + "cordis_stop", + "cordis_undefine", + "job_kill", + "job_list", + "job_output", + "run_code", + "subagent", + "workflow" + ] + }, + { + "model": "smoke-model", + "messages": [ + { + "role": "system", + "content": "{{system}}" + }, + { + "role": "user", + "content": "Complete the second isolated Python SDK process turn." + } + ], + "toolNames": [ + "cordis_define", + "cordis_inspect_list", + "cordis_inspect_query", + "cordis_inspect_self", + "cordis_run", + "cordis_stop", + "cordis_undefine", + "job_kill", + "job_list", + "job_output", + "run_code", + "subagent", + "workflow" + ] + } +] diff --git a/scripts/snapshots/python-sdk-single-exe/restart/result.json b/scripts/snapshots/python-sdk-single-exe/restart/result.json new file mode 100644 index 0000000000..32911ef466 --- /dev/null +++ b/scripts/snapshots/python-sdk-single-exe/restart/result.json @@ -0,0 +1,94 @@ +[ + { + "session_id": "{{session-1}}", + "final_response": "PROCESS_ONE_OK", + "finish_reason": "completed", + "eventTypes": [ + "agent/inbox/spliced", + "turn/start", + "agent/inbox/spliced", + "step/start", + "user/message", + "session/title", + "request/header", + "request/context", + "session-log-deepseek/delivery-accepted", + "assistant/chunk", + "assistant/chunk", + "assistant/chunk", + "assistant/chunk", + "assistant/chunk", + "assistant/message", + "step/end", + "turn/end" + ], + "notificationMethods": [ + "session.event", + "session.status", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.status" + ], + "session_root": "{{sessions}}" + }, + { + "session_id": "{{session-2}}", + "final_response": "PROCESS_TWO_OK", + "finish_reason": "completed", + "eventTypes": [ + "agent/inbox/spliced", + "turn/start", + "agent/inbox/spliced", + "step/start", + "user/message", + "session/title", + "request/header", + "request/context", + "session-log-deepseek/delivery-accepted", + "assistant/chunk", + "assistant/chunk", + "assistant/chunk", + "assistant/chunk", + "assistant/chunk", + "assistant/message", + "step/end", + "turn/end" + ], + "notificationMethods": [ + "session.event", + "session.status", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.event", + "session.status" + ], + "session_root": "{{sessions}}" + } +] diff --git a/scripts/snapshots/python-sdk-single-exe/restart/session.1.jsonl b/scripts/snapshots/python-sdk-single-exe/restart/session.1.jsonl new file mode 100644 index 0000000000..38e5f98ff6 --- /dev/null +++ b/scripts/snapshots/python-sdk-single-exe/restart/session.1.jsonl @@ -0,0 +1,18 @@ +{"type":"session","version":0,"id":"{{session-1}}","createdAt":0,"cwd":"{{cwd}}","delegationDepth":0} +{"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"inserted":[{"content":[{"type":"text","text":"Complete the first isolated Python SDK process turn."}],"source":{"kind":"user"},"role":"user","id":"{{messageId}}"}]}} +{"type":"turn/start","data":{"turn":1}} +{"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"removedCount":1,"inserted":[]}} +{"type":"step/start","data":{"turn":1,"step":1}} +{"type":"user/message","data":{"content":[{"type":"text","text":"Complete the first isolated Python SDK process turn."}],"source":{"kind":"user"},"role":"user","id":"{{messageId}}"},"surfaceOp":"append"} +{"type":"session/title","data":{"title":"Complete the first isolated Python","messageSeqs":[4],"source":{"kind":"fallback"}}} +{"type":"request/header","data":{"header":{"config":{"provider":"deepseek-official","model":"smoke-model","maxTokens":256000,"reasoningEffort":"high"},"adapterDefaults":{"reasoningEffort":true,"maxTokens":true},"system":"{{system}}","tools":["cordis_define","cordis_inspect_list","cordis_inspect_query","cordis_inspect_self","cordis_run","cordis_stop","cordis_undefine","job_kill","job_list","job_output","run_code","subagent","workflow"]},"reason":"initial"}} +{"type":"request/context","data":{"provider":"deepseek-official","model":"smoke-model","contextWindow":1000000}} +{"type":"session-log-deepseek/delivery-accepted","data":{"sessionId":"{{session-1}}","throughSeq":7}} +{"type":"assistant/chunk","data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"text"}}} +{"type":"assistant/chunk","data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":0,"text":"PROCESS_ONE_OK"}}} +{"type":"assistant/chunk","data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"PROCESS_ONE_OK"}}}} +{"type":"assistant/chunk","data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}}} +{"type":"assistant/chunk","data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"stop"}}}} +{"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"text","text":"PROCESS_ONE_OK"}],"source":{"kind":"model","provider":"deepseek-official","model":"smoke-model"},"id":"{{messageId}}"},"usage":{"inputTokens":3,"outputTokens":3}},"sourceEventSeqs":[9,10,11,12,13],"surfaceOp":"append"} +{"type":"step/end","data":{"turn":1,"step":1}} +{"type":"turn/end","data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/scripts/snapshots/python-sdk-single-exe/restart/session.2.jsonl b/scripts/snapshots/python-sdk-single-exe/restart/session.2.jsonl new file mode 100644 index 0000000000..523f784dfa --- /dev/null +++ b/scripts/snapshots/python-sdk-single-exe/restart/session.2.jsonl @@ -0,0 +1,18 @@ +{"type":"session","version":0,"id":"{{session-2}}","createdAt":0,"cwd":"{{cwd}}","delegationDepth":0} +{"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"inserted":[{"content":[{"type":"text","text":"Complete the second isolated Python SDK process turn."}],"source":{"kind":"user"},"role":"user","id":"{{messageId}}"}]}} +{"type":"turn/start","data":{"turn":1}} +{"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"removedCount":1,"inserted":[]}} +{"type":"step/start","data":{"turn":1,"step":1}} +{"type":"user/message","data":{"content":[{"type":"text","text":"Complete the second isolated Python SDK process turn."}],"source":{"kind":"user"},"role":"user","id":"{{messageId}}"},"surfaceOp":"append"} +{"type":"session/title","data":{"title":"Complete the second isolated Python","messageSeqs":[4],"source":{"kind":"fallback"}}} +{"type":"request/header","data":{"header":{"config":{"provider":"deepseek-official","model":"smoke-model","maxTokens":256000,"reasoningEffort":"high"},"adapterDefaults":{"reasoningEffort":true,"maxTokens":true},"system":"{{system}}","tools":["cordis_define","cordis_inspect_list","cordis_inspect_query","cordis_inspect_self","cordis_run","cordis_stop","cordis_undefine","job_kill","job_list","job_output","run_code","subagent","workflow"]},"reason":"initial"}} +{"type":"request/context","data":{"provider":"deepseek-official","model":"smoke-model","contextWindow":1000000}} +{"type":"session-log-deepseek/delivery-accepted","data":{"sessionId":"{{session-2}}","throughSeq":7}} +{"type":"assistant/chunk","data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"text"}}} +{"type":"assistant/chunk","data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":0,"text":"PROCESS_TWO_OK"}}} +{"type":"assistant/chunk","data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"PROCESS_TWO_OK"}}}} +{"type":"assistant/chunk","data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}}} +{"type":"assistant/chunk","data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"stop"}}}} +{"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"text","text":"PROCESS_TWO_OK"}],"source":{"kind":"model","provider":"deepseek-official","model":"smoke-model"},"id":"{{messageId}}"},"usage":{"inputTokens":3,"outputTokens":3}},"sourceEventSeqs":[9,10,11,12,13],"surfaceOp":"append"} +{"type":"step/end","data":{"turn":1,"step":1}} +{"type":"turn/end","data":{"turn":1,"reason":{"kind":"completed"}}}