diff --git a/python/development.i18n.yaml b/python/development.i18n.yaml index 9828188337..f2527aeebe 100644 --- a/python/development.i18n.yaml +++ b/python/development.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write python/development.md -development.md: 39c47b7d2a86995eea017c77fb147c63f90412b1 -development.zh.md: fe7b4dbb2ca2e341dfb9ceb57959b7d9807f54d2 +development.md: a42a704aea524c142182eb295a53044a43fb7ea8 +development.zh.md: ccb6b262463da282f549ea12bbab3561834a9d67 diff --git a/python/development.md b/python/development.md index 39c47b7d2a..a42a704aea 100644 --- a/python/development.md +++ b/python/development.md @@ -36,7 +36,7 @@ uv run --project python/sdk python scripts/smoke-python-runtime.py \ Three scenarios compare committed expected output under `scripts/snapshots/python-sdk-single-exe/`. `minimal/model-visible.json` pins the Linux/macOS `sdk-minimal` profile's assembled system prompts, advertised tool schemas, and model-visible messages; `minimal/win-x64/model-visible.json` pins its PowerShell counterpart. A plugin that contributes an unintended system section or user message therefore fails the job, and every message the profile emits is compared. `advanced/` pins one complex process's SDK result and parent/child session logs across every target. `restart/` launches two complete SDK runtime processes against one persistence root and snapshots their isolated model histories, high-level results, and separate durable logs across every target. Rerun the owning scenario with `--update-snapshots` and review that diff before committing it. -Trusted pull requests and master pushes also run `--scenario sdk-live --installed-wheel` on each selected native target. That scenario performs two tool-using turns against `https://api.deepseek.com`, verifies the created file externally, and fails when the repository secret is absent instead of self-skipping. Fork and Dependabot pull requests run the complete keyless installed-wheel path but receive no key. +Trusted pull requests and master pushes also run `--scenario sdk-live --installed-wheel` on each selected native target. That scenario performs two tool-using turns against `https://api.deepseek.com`: it checks the created file immediately, replaces its content with a host-only random challenge, and requires the second turn to copy the changed content into a fresh receipt without modifying the source. Both turns must complete with the exact sentinel answer and a model-requested tool call; external byte comparisons check the files. Missing repository secrets fail instead of self-skipping. Fork and Dependabot pull requests run the complete keyless installed-wheel path but receive no key. An interactive smoke test needs `DEEPSEEK_API_KEY` in the environment or repository-root `.env`: diff --git a/python/development.zh.md b/python/development.zh.md index fe7b4dbb2c..ccb6b26246 100644 --- a/python/development.zh.md +++ b/python/development.zh.md @@ -36,7 +36,7 @@ uv run --project python/sdk python scripts/smoke-python-runtime.py \ 其中三个场景会比对 `scripts/snapshots/python-sdk-single-exe/` 下已提交的期望输出。`minimal/model-visible.json` 固定 Linux／macOS `sdk-minimal` profile 所组装的系统提示词、对外公布的工具 schema 与模型可见消息；`minimal/win-x64/model-visible.json` 固定对应的 PowerShell 版本。因此，插件一旦贡献出计划外的系统分段或 user 消息，该任务即失败，且该 profile 发出的每条消息都会参与比对。`advanced/` 跨所有目标固定一个复杂进程的 SDK 结果及父／子会话日志。`restart/` 针对同一持久化根目录启动两个完整 SDK 运行时进程，并跨所有目标固定其彼此隔离的模型历史、高层结果与独立持久日志。重新运行对应场景时加上 `--update-snapshots`，并在提交前审阅该差异。 -可信拉取请求与 master 推送还会在各自选定的原生目标上运行 `--scenario sdk-live --installed-wheel`。该场景面向 `https://api.deepseek.com` 执行两个使用工具的轮次，从外部验证已创建文件，并在仓库密钥缺失时失败而不是自行 skip。Fork 与 Dependabot 拉取请求会运行完整的 keyless 安装后 wheel 路径，但不会获得密钥。 +可信拉取请求与 master 推送还会在各自选定的原生目标上运行 `--scenario sdk-live --installed-wheel`。该场景面向 `https://api.deepseek.com` 执行两个使用工具的轮次：立即检查已创建文件，将其内容替换为仅宿主知道的随机挑战值，并要求第二轮将变更后的内容复制到全新的回执文件，且不修改源文件。两个轮次都必须完成、返回精确的哨兵答案并由模型请求调用工具；文件通过外部逐字节比较验证。仓库密钥缺失时失败，而不是自行 skip。Fork 与 Dependabot 拉取请求会运行完整的 keyless 安装后 wheel 路径，但不会获得密钥。 交互式冒烟测试需要环境变量或仓库根目录 `.env` 中存在 `DEEPSEEK_API_KEY`： diff --git a/python/sdk/tests/test_smoke_model.py b/python/sdk/tests/test_smoke_model.py index a2e1b90be8..a084546b52 100644 --- a/python/sdk/tests/test_smoke_model.py +++ b/python/sdk/tests/test_smoke_model.py @@ -4,14 +4,157 @@ import json import runpy import subprocess from pathlib import Path +from types import SimpleNamespace import pytest +from deepseek_harness import RunResult + ROOT = Path(__file__).resolve().parents[3] SMOKE = runpy.run_path(ROOT / "scripts" / "smoke-python-runtime.py") +def live_result(**overrides: object) -> RunResult: + values = { + "session_id": "installed-wheel-live-api", + "final_response": SMOKE["LIVE_API_SENTINEL"], + "finish_reason": "completed", + "events": [{"type": "tool/call", "data": {"name": "unrelated_tool"}}], + "notifications": [], + } + values.update(overrides) + return RunResult(**values) + + +@pytest.fixture +def live_smoke(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace: + import deepseek_harness + + state = SimpleNamespace( + prompts=[], session_ids=[], challenges=[], checked_logs=[], closed=False, + create_bytes=SMOKE["LIVE_API_SENTINEL"].encode("utf-8"), + create_result=live_result(), verify_result=live_result(), receipt_mode="copy", + ) + globals_ = SMOKE["smoke_sdk_live"].__globals__ + token_hex = globals_["secrets"].token_hex + + def fresh_challenge(size: int) -> str: + assert len(state.prompts) == 1 + value = token_hex(size) + state.challenges.append(value) + return value + + class ScriptedHarness: + def __init__(self, **kwargs: object) -> None: + state.root = Path(kwargs["cwd"]) + assert "toolChoice" not in kwargs + + def __enter__(self) -> ScriptedHarness: + return self + + def __exit__(self, *args: object) -> None: + state.closed = True + + def run(self, prompt: str, *, session_id: str) -> RunResult: + state.prompts.append(prompt) + state.session_ids.append(session_id) + if len(state.prompts) == 1: + state.marker = Path(prompt.splitlines()[-1]) + assert state.marker.parent == state.root + if state.create_bytes is not None: + state.marker.write_bytes(state.create_bytes) + return state.create_result + + assert len(state.prompts) == 2 + assert len(state.challenges) == 1 + challenge = state.challenges[0] + assert all(challenge not in sent for sent in state.prompts) + assert str(state.marker) not in prompt + assert "previous turn" in prompt and "changed externally" in prompt + assert state.marker.read_bytes() == challenge.encode("ascii") + receipt = Path(prompt.splitlines()[-1]) + assert receipt != state.marker and not receipt.exists() + if state.receipt_mode == "copy": + receipt.write_bytes(state.marker.read_bytes()) + elif state.receipt_mode == "stale": + receipt.write_bytes(state.create_bytes) + elif state.receipt_mode == "wrong": + receipt.write_bytes(b"wrong") + elif state.receipt_mode == "newline": + receipt.write_bytes(state.marker.read_bytes() + b"\n") + elif state.receipt_mode == "changed-source": + receipt.write_bytes(state.marker.read_bytes()) + state.marker.write_bytes(b"changed") + elif state.receipt_mode != "missing": + raise AssertionError(state.receipt_mode) + return state.verify_result + + monkeypatch.setenv("DEEPSEEK_API_KEY", "unit-test-key") + monkeypatch.setenv("DEEPSEEK_BASE_URL", "https://api.invalid") + monkeypatch.setattr(deepseek_harness, "DeepSeekHarness", ScriptedHarness) + monkeypatch.setattr(globals_["secrets"], "token_hex", fresh_challenge) + monkeypatch.setitem(globals_, "assert_zstd_session_log", state.checked_logs.append) + return state + + +def test_live_smoke_requires_fresh_external_content(live_smoke: SimpleNamespace) -> None: + SMOKE["smoke_sdk_live"]() + assert len(live_smoke.prompts) == 2 + assert live_smoke.session_ids == ["installed-wheel-live-api"] * 2 + assert len(live_smoke.checked_logs) == 1 + assert live_smoke.closed and not live_smoke.root.exists() + + +@pytest.mark.parametrize("label", ["create", "verify"]) +@pytest.mark.parametrize(("overrides", "message"), [ + ({"finish_reason": "error"}, "turn ended with 'error'"), + ({"events": []}, "turn made no model-requested tool call"), + ({"final_response": "PYTHON_SDK_LIVE_OK extra"}, "turn returned"), +]) +def test_live_smoke_rejects_invalid_turn_before_continuing( + live_smoke: SimpleNamespace, label: str, overrides: dict[str, object], message: str, +) -> None: + setattr(live_smoke, f"{label}_result", live_result(**overrides)) + with pytest.raises(AssertionError, match=f"{label} {message}"): + SMOKE["smoke_sdk_live"]() + assert len(live_smoke.prompts) == (1 if label == "create" else 2) + assert not live_smoke.checked_logs + assert live_smoke.closed + if label == "create": + assert not live_smoke.challenges + + +@pytest.mark.parametrize("content", [None, b"wrong", b"PYTHON_SDK_LIVE_OK\n"]) +def test_live_smoke_rejects_bad_create_before_host_overwrite( + live_smoke: SimpleNamespace, content: bytes | None, +) -> None: + live_smoke.create_bytes = content + with pytest.raises(AssertionError, match="create turn (did not create|wrote unexpected bytes)"): + SMOKE["smoke_sdk_live"]() + assert len(live_smoke.prompts) == 1 + assert not live_smoke.challenges and not live_smoke.checked_logs + assert live_smoke.closed + + +@pytest.mark.parametrize(("mode", "message"), [ + ("missing", "did not create receipt"), + ("stale", "wrote unexpected bytes to receipt"), + ("wrong", "wrote unexpected bytes to receipt"), + ("newline", "wrote unexpected bytes to receipt"), + ("changed-source", "changed source file"), +]) +def test_live_smoke_rejects_unrelated_tool_without_exact_receipt( + live_smoke: SimpleNamespace, mode: str, message: str, +) -> None: + live_smoke.receipt_mode = mode + with pytest.raises(AssertionError, match=f"verify turn {message}"): + SMOKE["smoke_sdk_live"]() + assert len(live_smoke.prompts) == 2 + assert not live_smoke.checked_logs + assert live_smoke.closed + + @pytest.mark.parametrize( ("prompt_name", "expected"), [ diff --git a/scripts/smoke-python-runtime.py b/scripts/smoke-python-runtime.py index a97b5970d7..ab31429c99 100644 --- a/scripts/smoke-python-runtime.py +++ b/scripts/smoke-python-runtime.py @@ -11,6 +11,7 @@ import json import os import queue import re +import secrets import shutil import subprocess import sys @@ -904,12 +905,9 @@ def smoke_sdk_live() -> None: session_id = "installed-wheel-live-api" shell_tool = "pwsh" if IS_WINDOWS else "bash" create_prompt = ( - f"Use the {shell_tool} tool to create the file at the absolute path below with exactly one line " - f"containing {LIVE_API_SENTINEL}. Then reply with exactly {LIVE_API_SENTINEL}.\n{marker}" - ) - verify_prompt = ( - "Use a tool to read the file created in the previous turn. " - f"If its only line is {LIVE_API_SENTINEL}, reply with exactly {LIVE_API_SENTINEL}." + f"Use the {shell_tool} tool to create the file at the absolute path below with exact UTF-8 " + f"content {LIVE_API_SENTINEL}, with no newline or byte-order mark. " + f"Then reply with exactly {LIVE_API_SENTINEL}.\n{marker}" ) with DeepSeekHarness( provider="deepseek-official", @@ -925,34 +923,56 @@ def smoke_sdk_live() -> None: request_timeout_seconds=180, ) as harness: created = harness.run(create_prompt, session_id=session_id) - verified = harness.run(verify_prompt, session_id=session_id) + assert_live_turn("create", created) + if not marker.is_file(): + raise AssertionError(f"create turn did not create {marker}") + if marker.read_bytes() != LIVE_API_SENTINEL.encode("utf-8"): + raise AssertionError(f"create turn wrote unexpected bytes to {marker}") - for label, result in (("create", created), ("verify", verified)): - if result.finish_reason != "completed": - event_types = [event.get("type") for event in result.events] - turn_end_data = next( - (event.get("data") for event in reversed(result.events) if event.get("type") == "turn/end"), - None, + # The challenge is absent from the prior turn and the verification prompt. + challenge = secrets.token_hex(32).encode("ascii") + marker.write_bytes(challenge) + with tempfile.TemporaryDirectory(prefix="receipt-", dir=root) as receipt_directory: + receipt = Path(receipt_directory) / "receipt.txt" + verify_prompt = ( + "The file created in the previous turn has changed externally. " + "Use a tool to read that same file and copy its exact current content to the " + "new receipt path below, without changing the source file. " + "Preserve every byte; do not add a newline or byte-order mark. " + f"Then reply with exactly {LIVE_API_SENTINEL}.\n{receipt}" ) - turn_end = safe_turn_end(turn_end_data) - raise AssertionError( - f"{label} turn ended with {result.finish_reason!r}; " - f"final={result.final_response!r}; turn_end={turn_end!r}; events={event_types}" - ) - if not any(event.get("type") == "tool/call" for event in result.events): - raise AssertionError( - f"{label} turn made no model-requested tool call; " - f"final={result.final_response!r}" - ) - if result.final_response.strip() != LIVE_API_SENTINEL: - raise AssertionError(f"{label} turn returned {result.final_response!r}") - if not marker.is_file(): - raise AssertionError(f"real-model tool turn did not create {marker}") - if marker.read_text(encoding="utf-8").splitlines() != [LIVE_API_SENTINEL]: - raise AssertionError(f"real-model tool turn wrote unexpected text to {marker}") + verified = harness.run(verify_prompt, session_id=session_id) + assert_live_turn("verify", verified) + if not receipt.is_file(): + raise AssertionError(f"verify turn did not create receipt {receipt}") + if receipt.read_bytes() != challenge: + raise AssertionError(f"verify turn wrote unexpected bytes to receipt {receipt}") + if not marker.is_file() or marker.read_bytes() != challenge: + raise AssertionError(f"verify turn changed source file {marker}") assert_zstd_session_log(sessions) +def assert_live_turn(label: str, result: RunResult) -> None: + """Require completed model tool use and the exact smoke answer for each live turn.""" + if result.finish_reason != "completed": + event_types = [event.get("type") for event in result.events] + turn_end_data = next( + (event.get("data") for event in reversed(result.events) if event.get("type") == "turn/end"), + None, + ) + turn_end = safe_turn_end(turn_end_data) + raise AssertionError( + f"{label} turn ended with {result.finish_reason!r}; " + f"final={result.final_response!r}; turn_end={turn_end!r}; events={event_types}" + ) + if not any(event.get("type") == "tool/call" for event in result.events): + raise AssertionError( + f"{label} turn made no model-requested tool call; " + f"final={result.final_response!r}" + ) + if result.final_response.strip() != LIVE_API_SENTINEL: + raise AssertionError(f"{label} turn returned {result.final_response!r}") + def safe_turn_end(value: object) -> object: """Project a live-provider failure without retaining credential-bearing text.""" if not isinstance(value, dict):