mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-09-09 04:02:35 +00:00
fix(python): challenge fresh file state in live SDK smoke
This commit is contained in:
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write python/development.md
|
||||
development.md: 39c47b7d2a86995eea017c77fb147c63f90412b1
|
||||
development.zh.md: fe7b4dbb2ca2e341dfb9ceb57959b7d9807f54d2
|
||||
development.md: a42a704aea524c142182eb295a53044a43fb7ea8
|
||||
development.zh.md: ccb6b262463da282f549ea12bbab3561834a9d67
|
||||
|
||||
@@ -36,7 +36,7 @@ uv run --project python/sdk python scripts/smoke-python-runtime.py \
|
||||
|
||||
Three scenarios compare committed expected output under `scripts/snapshots/python-sdk-single-exe/`. `minimal/model-visible.json` pins the Linux/macOS `sdk-minimal` profile's assembled system prompts, advertised tool schemas, and model-visible messages; `minimal/win-x64/model-visible.json` pins its PowerShell counterpart. A plugin that contributes an unintended system section or user message therefore fails the job, and every message the profile emits is compared. `advanced/` pins one complex process's SDK result and parent/child session logs across every target. `restart/` launches two complete SDK runtime processes against one persistence root and snapshots their isolated model histories, high-level results, and separate durable logs across every target. Rerun the owning scenario with `--update-snapshots` and review that diff before committing it.
|
||||
|
||||
Trusted pull requests and master pushes also run `--scenario sdk-live --installed-wheel` on each selected native target. That scenario performs two tool-using turns against `https://api.deepseek.com`, verifies the created file externally, and fails when the repository secret is absent instead of self-skipping. Fork and Dependabot pull requests run the complete keyless installed-wheel path but receive no key.
|
||||
Trusted pull requests and master pushes also run `--scenario sdk-live --installed-wheel` on each selected native target. That scenario performs two tool-using turns against `https://api.deepseek.com`: it checks the created file immediately, replaces its content with a host-only random challenge, and requires the second turn to copy the changed content into a fresh receipt without modifying the source. Both turns must complete with the exact sentinel answer and a model-requested tool call; external byte comparisons check the files. Missing repository secrets fail instead of self-skipping. Fork and Dependabot pull requests run the complete keyless installed-wheel path but receive no key.
|
||||
|
||||
An interactive smoke test needs `DEEPSEEK_API_KEY` in the environment or repository-root `.env`:
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ uv run --project python/sdk python scripts/smoke-python-runtime.py \
|
||||
|
||||
其中三个场景会比对 `scripts/snapshots/python-sdk-single-exe/` 下已提交的期望输出。`minimal/model-visible.json` 固定 Linux/macOS `sdk-minimal` profile 所组装的系统提示词、对外公布的工具 schema 与模型可见消息;`minimal/win-x64/model-visible.json` 固定对应的 PowerShell 版本。因此,插件一旦贡献出计划外的系统分段或 user 消息,该任务即失败,且该 profile 发出的每条消息都会参与比对。`advanced/` 跨所有目标固定一个复杂进程的 SDK 结果及父/子会话日志。`restart/` 针对同一持久化根目录启动两个完整 SDK 运行时进程,并跨所有目标固定其彼此隔离的模型历史、高层结果与独立持久日志。重新运行对应场景时加上 `--update-snapshots`,并在提交前审阅该差异。
|
||||
|
||||
可信拉取请求与 master 推送还会在各自选定的原生目标上运行 `--scenario sdk-live --installed-wheel`。该场景面向 `https://api.deepseek.com` 执行两个使用工具的轮次,从外部验证已创建文件,并在仓库密钥缺失时失败而不是自行 skip。Fork 与 Dependabot 拉取请求会运行完整的 keyless 安装后 wheel 路径,但不会获得密钥。
|
||||
可信拉取请求与 master 推送还会在各自选定的原生目标上运行 `--scenario sdk-live --installed-wheel`。该场景面向 `https://api.deepseek.com` 执行两个使用工具的轮次:立即检查已创建文件,将其内容替换为仅宿主知道的随机挑战值,并要求第二轮将变更后的内容复制到全新的回执文件,且不修改源文件。两个轮次都必须完成、返回精确的哨兵答案并由模型请求调用工具;文件通过外部逐字节比较验证。仓库密钥缺失时失败,而不是自行 skip。Fork 与 Dependabot 拉取请求会运行完整的 keyless 安装后 wheel 路径,但不会获得密钥。
|
||||
|
||||
交互式冒烟测试需要环境变量或仓库根目录 `.env` 中存在 `DEEPSEEK_API_KEY`:
|
||||
|
||||
|
||||
@@ -4,14 +4,157 @@ import json
|
||||
import runpy
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from deepseek_harness import RunResult
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SMOKE = runpy.run_path(ROOT / "scripts" / "smoke-python-runtime.py")
|
||||
|
||||
|
||||
def live_result(**overrides: object) -> RunResult:
|
||||
values = {
|
||||
"session_id": "installed-wheel-live-api",
|
||||
"final_response": SMOKE["LIVE_API_SENTINEL"],
|
||||
"finish_reason": "completed",
|
||||
"events": [{"type": "tool/call", "data": {"name": "unrelated_tool"}}],
|
||||
"notifications": [],
|
||||
}
|
||||
values.update(overrides)
|
||||
return RunResult(**values)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_smoke(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
|
||||
import deepseek_harness
|
||||
|
||||
state = SimpleNamespace(
|
||||
prompts=[], session_ids=[], challenges=[], checked_logs=[], closed=False,
|
||||
create_bytes=SMOKE["LIVE_API_SENTINEL"].encode("utf-8"),
|
||||
create_result=live_result(), verify_result=live_result(), receipt_mode="copy",
|
||||
)
|
||||
globals_ = SMOKE["smoke_sdk_live"].__globals__
|
||||
token_hex = globals_["secrets"].token_hex
|
||||
|
||||
def fresh_challenge(size: int) -> str:
|
||||
assert len(state.prompts) == 1
|
||||
value = token_hex(size)
|
||||
state.challenges.append(value)
|
||||
return value
|
||||
|
||||
class ScriptedHarness:
|
||||
def __init__(self, **kwargs: object) -> None:
|
||||
state.root = Path(kwargs["cwd"])
|
||||
assert "toolChoice" not in kwargs
|
||||
|
||||
def __enter__(self) -> ScriptedHarness:
|
||||
return self
|
||||
|
||||
def __exit__(self, *args: object) -> None:
|
||||
state.closed = True
|
||||
|
||||
def run(self, prompt: str, *, session_id: str) -> RunResult:
|
||||
state.prompts.append(prompt)
|
||||
state.session_ids.append(session_id)
|
||||
if len(state.prompts) == 1:
|
||||
state.marker = Path(prompt.splitlines()[-1])
|
||||
assert state.marker.parent == state.root
|
||||
if state.create_bytes is not None:
|
||||
state.marker.write_bytes(state.create_bytes)
|
||||
return state.create_result
|
||||
|
||||
assert len(state.prompts) == 2
|
||||
assert len(state.challenges) == 1
|
||||
challenge = state.challenges[0]
|
||||
assert all(challenge not in sent for sent in state.prompts)
|
||||
assert str(state.marker) not in prompt
|
||||
assert "previous turn" in prompt and "changed externally" in prompt
|
||||
assert state.marker.read_bytes() == challenge.encode("ascii")
|
||||
receipt = Path(prompt.splitlines()[-1])
|
||||
assert receipt != state.marker and not receipt.exists()
|
||||
if state.receipt_mode == "copy":
|
||||
receipt.write_bytes(state.marker.read_bytes())
|
||||
elif state.receipt_mode == "stale":
|
||||
receipt.write_bytes(state.create_bytes)
|
||||
elif state.receipt_mode == "wrong":
|
||||
receipt.write_bytes(b"wrong")
|
||||
elif state.receipt_mode == "newline":
|
||||
receipt.write_bytes(state.marker.read_bytes() + b"\n")
|
||||
elif state.receipt_mode == "changed-source":
|
||||
receipt.write_bytes(state.marker.read_bytes())
|
||||
state.marker.write_bytes(b"changed")
|
||||
elif state.receipt_mode != "missing":
|
||||
raise AssertionError(state.receipt_mode)
|
||||
return state.verify_result
|
||||
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", "unit-test-key")
|
||||
monkeypatch.setenv("DEEPSEEK_BASE_URL", "https://api.invalid")
|
||||
monkeypatch.setattr(deepseek_harness, "DeepSeekHarness", ScriptedHarness)
|
||||
monkeypatch.setattr(globals_["secrets"], "token_hex", fresh_challenge)
|
||||
monkeypatch.setitem(globals_, "assert_zstd_session_log", state.checked_logs.append)
|
||||
return state
|
||||
|
||||
|
||||
def test_live_smoke_requires_fresh_external_content(live_smoke: SimpleNamespace) -> None:
|
||||
SMOKE["smoke_sdk_live"]()
|
||||
assert len(live_smoke.prompts) == 2
|
||||
assert live_smoke.session_ids == ["installed-wheel-live-api"] * 2
|
||||
assert len(live_smoke.checked_logs) == 1
|
||||
assert live_smoke.closed and not live_smoke.root.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("label", ["create", "verify"])
|
||||
@pytest.mark.parametrize(("overrides", "message"), [
|
||||
({"finish_reason": "error"}, "turn ended with 'error'"),
|
||||
({"events": []}, "turn made no model-requested tool call"),
|
||||
({"final_response": "PYTHON_SDK_LIVE_OK extra"}, "turn returned"),
|
||||
])
|
||||
def test_live_smoke_rejects_invalid_turn_before_continuing(
|
||||
live_smoke: SimpleNamespace, label: str, overrides: dict[str, object], message: str,
|
||||
) -> None:
|
||||
setattr(live_smoke, f"{label}_result", live_result(**overrides))
|
||||
with pytest.raises(AssertionError, match=f"{label} {message}"):
|
||||
SMOKE["smoke_sdk_live"]()
|
||||
assert len(live_smoke.prompts) == (1 if label == "create" else 2)
|
||||
assert not live_smoke.checked_logs
|
||||
assert live_smoke.closed
|
||||
if label == "create":
|
||||
assert not live_smoke.challenges
|
||||
|
||||
|
||||
@pytest.mark.parametrize("content", [None, b"wrong", b"PYTHON_SDK_LIVE_OK\n"])
|
||||
def test_live_smoke_rejects_bad_create_before_host_overwrite(
|
||||
live_smoke: SimpleNamespace, content: bytes | None,
|
||||
) -> None:
|
||||
live_smoke.create_bytes = content
|
||||
with pytest.raises(AssertionError, match="create turn (did not create|wrote unexpected bytes)"):
|
||||
SMOKE["smoke_sdk_live"]()
|
||||
assert len(live_smoke.prompts) == 1
|
||||
assert not live_smoke.challenges and not live_smoke.checked_logs
|
||||
assert live_smoke.closed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("mode", "message"), [
|
||||
("missing", "did not create receipt"),
|
||||
("stale", "wrote unexpected bytes to receipt"),
|
||||
("wrong", "wrote unexpected bytes to receipt"),
|
||||
("newline", "wrote unexpected bytes to receipt"),
|
||||
("changed-source", "changed source file"),
|
||||
])
|
||||
def test_live_smoke_rejects_unrelated_tool_without_exact_receipt(
|
||||
live_smoke: SimpleNamespace, mode: str, message: str,
|
||||
) -> None:
|
||||
live_smoke.receipt_mode = mode
|
||||
with pytest.raises(AssertionError, match=f"verify turn {message}"):
|
||||
SMOKE["smoke_sdk_live"]()
|
||||
assert len(live_smoke.prompts) == 2
|
||||
assert not live_smoke.checked_logs
|
||||
assert live_smoke.closed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt_name", "expected"),
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user