mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-09-09 04:02:35 +00:00
363 lines
12 KiB
Python
363 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import runpy
|
|
import subprocess
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
SMOKE = runpy.run_path(ROOT / "scripts" / "smoke-python-runtime.py")
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("behavior", "error"),
|
|
[
|
|
("read-current", None),
|
|
("no-verify-tool", "verify turn made no model-requested tool call"),
|
|
("no-create-tool", "create turn made no model-requested tool call"),
|
|
("create-error", "create turn ended with.*AUTH.*401"),
|
|
("stale-answer", "verify turn returned"),
|
|
("missing-create", "real-model tool turn did not create"),
|
|
("wrong-create", "real-model tool turn wrote unexpected text"),
|
|
("modify-verify", "real-model tool turn wrote unexpected text"),
|
|
],
|
|
)
|
|
def test_live_smoke_requires_fresh_file_observation(
|
|
monkeypatch: pytest.MonkeyPatch, behavior: str, error: str | None,
|
|
) -> None:
|
|
import deepseek_harness
|
|
|
|
smoke_live = SMOKE["smoke_sdk_live"]
|
|
sentinel = SMOKE["LIVE_API_SENTINEL"]
|
|
prompts: list[str] = []
|
|
session_ids: list[str] = []
|
|
log_checks: list[Path] = []
|
|
|
|
class ScriptedHarness:
|
|
def __init__(self, *, cwd: str, **_kwargs: object) -> None:
|
|
self.marker = Path(cwd) / "live-api-marker.txt"
|
|
|
|
def __enter__(self) -> ScriptedHarness:
|
|
return self
|
|
|
|
def __exit__(self, *_args: object) -> None:
|
|
pass
|
|
|
|
def run(self, prompt: str, *, session_id: str) -> SimpleNamespace:
|
|
prompts.append(prompt)
|
|
session_ids.append(session_id)
|
|
events = [{"type": "tool/call"}]
|
|
if len(prompts) == 1:
|
|
assert not self.marker.exists()
|
|
if behavior == "create-error":
|
|
return SimpleNamespace(finish_reason="error", final_response="", events=[{
|
|
"type": "turn/end",
|
|
"data": {"turn": 1, "reason": {
|
|
"kind": "error", "error": {"code": "AUTH", "status": 401},
|
|
}},
|
|
}])
|
|
if behavior == "no-create-tool":
|
|
events = []
|
|
if behavior != "missing-create":
|
|
self.marker.write_text(
|
|
("wrong" if behavior == "wrong-create" else sentinel) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
response = sentinel
|
|
else:
|
|
current = self.marker.read_text(encoding="utf-8").strip()
|
|
assert current != sentinel, "verification must require new world state"
|
|
assert all(current not in text for text in prompts), "prompts must not reveal the answer"
|
|
response = sentinel if behavior == "stale-answer" else current
|
|
if behavior == "no-verify-tool":
|
|
events = []
|
|
if behavior == "modify-verify":
|
|
self.marker.write_text("changed\n", encoding="utf-8")
|
|
return SimpleNamespace(finish_reason="completed", final_response=response, events=events)
|
|
|
|
monkeypatch.setenv("DEEPSEEK_API_KEY", "fixture-key")
|
|
monkeypatch.setenv("DEEPSEEK_BASE_URL", "https://fixture.invalid")
|
|
monkeypatch.setattr(deepseek_harness, "DeepSeekHarness", ScriptedHarness)
|
|
monkeypatch.setitem(smoke_live.__globals__, "assert_zstd_session_log", log_checks.append)
|
|
|
|
if error is None:
|
|
smoke_live()
|
|
assert len(prompts) == 2
|
|
assert session_ids[0] == session_ids[1]
|
|
assert len(log_checks) == 1
|
|
else:
|
|
with pytest.raises(AssertionError, match=error):
|
|
smoke_live()
|
|
assert not log_checks
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("prompt_name", "expected"),
|
|
[
|
|
("SNAPSHOT_DIRECT_CHILD_PROMPT", "DIRECT_CHILD_OK"),
|
|
("SNAPSHOT_WORKFLOW_CHILD_PROMPT", "WORKFLOW_CHILD_OK"),
|
|
],
|
|
)
|
|
def test_child_prompt_precedes_runtime_context(prompt_name: str, expected: str) -> None:
|
|
chunks = SMOKE["completion_chunks"]({
|
|
"messages": [
|
|
{"role": "user", "content": SMOKE[prompt_name]},
|
|
{"role": "user", "content": "Current runtime context"},
|
|
],
|
|
})
|
|
|
|
assert any(
|
|
choice.get("delta", {}).get("content") == expected
|
|
for chunk in chunks
|
|
for choice in chunk.get("choices", [])
|
|
)
|
|
|
|
|
|
def test_mcp_smoke_requests_the_discovered_tool() -> None:
|
|
chunks = SMOKE["completion_chunks"]({
|
|
"messages": [{"role": "user", "content": SMOKE["MCP_PROMPT"]}],
|
|
"tools": [{"type": "function", "function": {"name": "mcp__fixture__add"}}],
|
|
})
|
|
|
|
calls = [
|
|
call
|
|
for chunk in chunks
|
|
for choice in chunk.get("choices", [])
|
|
for call in choice.get("delta", {}).get("tool_calls", [])
|
|
]
|
|
assert calls[0]["function"] == {
|
|
"name": "mcp__fixture__add",
|
|
"arguments": '{"a": 19, "b": 23}',
|
|
}
|
|
|
|
|
|
def test_mcp_smoke_accepts_the_external_server_result() -> None:
|
|
chunks = SMOKE["completion_chunks"]({
|
|
"messages": [
|
|
{"role": "user", "content": SMOKE["MCP_PROMPT"]},
|
|
{
|
|
"role": "assistant",
|
|
"tool_calls": [{
|
|
"id": "mcp-add",
|
|
"type": "function",
|
|
"function": {"name": "mcp__fixture__add", "arguments": '{}'},
|
|
}],
|
|
},
|
|
{"role": "tool", "tool_call_id": "mcp-add", "content": "42"},
|
|
],
|
|
})
|
|
|
|
assert any(
|
|
choice.get("delta", {}).get("content") == SMOKE["MCP_TEXT"]
|
|
for chunk in chunks
|
|
for choice in chunk.get("choices", [])
|
|
)
|
|
|
|
|
|
def test_snapshot_comparison_normalizes_only_session_generation_provenance() -> None:
|
|
normalize = SMOKE["normalize_session_format_comparison"]
|
|
expected = {
|
|
"header": {"type": "session", "version": 0, "otherVersion": 7},
|
|
"accepted": {
|
|
"type": "session-log-deepseek/delivery-accepted",
|
|
"data": {"sessionId": "s", "throughSeq": 4},
|
|
},
|
|
"source": {
|
|
"kind": "session-reference",
|
|
"references": [{"sessionId": "other", "capturedThroughSeq": 8}],
|
|
},
|
|
}
|
|
actual = {
|
|
"header": {"type": "session", "version": 1, "otherVersion": 7},
|
|
"accepted": {
|
|
"type": "session-log-deepseek/delivery-accepted",
|
|
"data": {"sessionId": "s", "sessionFormatVersion": 1, "throughSeq": 4},
|
|
},
|
|
"source": {
|
|
"kind": "session-reference",
|
|
"references": [{
|
|
"sessionId": "other",
|
|
"capturedFormatVersion": 1,
|
|
"capturedThroughSeq": 8,
|
|
}],
|
|
},
|
|
}
|
|
|
|
assert normalize(expected) == normalize(actual)
|
|
assert normalize(expected)["header"]["otherVersion"] == 7
|
|
|
|
|
|
def test_snapshot_value_normalizes_embedded_assistant_stream_timing() -> None:
|
|
normalize = SMOKE["normalize_snapshot_value"]
|
|
event = {
|
|
"type": "assistant/message",
|
|
"seq": 4,
|
|
"time": 100,
|
|
"data": {
|
|
"stream": [
|
|
{"type": "chunk", "time": 101, "chunk": {"type": "finish"}},
|
|
{"type": "text-chunks", "time0": 102, "dt": [1, 2], "texts": ["a", "b", "c"]},
|
|
],
|
|
},
|
|
}
|
|
|
|
normalized = normalize(event, [])
|
|
|
|
assert normalized["time"] == 0
|
|
assert normalized["data"]["stream"] == [
|
|
{"type": "chunk", "time": 0, "chunk": {"type": "finish"}},
|
|
{"type": "text-chunks", "time0": 0, "dt": [0, 0], "texts": ["a", "b", "c"]},
|
|
]
|
|
|
|
|
|
def test_snapshot_comparison_expands_embedded_assistant_streams() -> None:
|
|
normalize = SMOKE["normalize_session_format_comparison"]
|
|
expected = [
|
|
{
|
|
"type": "assistant/chunk",
|
|
"seq": 4,
|
|
"time": 0,
|
|
"data": {"turn": 1, "step": 1, "chunk": {
|
|
"type": "text-delta", "index": 0, "text": "done",
|
|
}},
|
|
},
|
|
{
|
|
"type": "assistant/message",
|
|
"seq": 5,
|
|
"time": 0,
|
|
"data": {"turn": 1, "step": 1, "message": {"role": "assistant"}},
|
|
"sourceEventSeqs": [4],
|
|
"surfaceOp": "append",
|
|
},
|
|
]
|
|
actual = [{
|
|
"type": "assistant/message",
|
|
"seq": 4,
|
|
"time": 0,
|
|
"data": {
|
|
"turn": 1,
|
|
"step": 1,
|
|
"message": {"role": "assistant"},
|
|
"stream": [{
|
|
"type": "text-chunks", "time0": 0, "index": 0, "dt": [], "texts": ["done"],
|
|
}],
|
|
},
|
|
"surfaceOp": "append",
|
|
}]
|
|
|
|
assert normalize(actual, 2) == normalize(expected, 1)
|
|
|
|
tool_result = {
|
|
"type": "tool/result",
|
|
"data": {"turn": 1, "step": 1},
|
|
"sourceEventSeqs": [4],
|
|
}
|
|
assert normalize(tool_result, 1)["sourceEventSeqs"] == [4]
|
|
assert normalize(tool_result, 2)["sourceEventSeqs"] == [4]
|
|
|
|
|
|
def test_snapshot_stream_expands_reasoning_and_tool_call_records() -> None:
|
|
expand = SMOKE["expand_snapshot_stream_member"]
|
|
|
|
assert expand({
|
|
"type": "reasoning-chunks", "time0": 0, "index": 1,
|
|
"dt": [], "texts": ["think"],
|
|
}) == [{"type": "reasoning-delta", "index": 1, "text": "think"}]
|
|
assert expand({
|
|
"type": "tool-call-chunks", "time0": 0, "index": 2,
|
|
"id": "call-1", "name": "read", "dt": [1], "args": ["{", "}"],
|
|
}) == [
|
|
{"type": "tool-call-delta", "index": 2, "id": "call-1", "name": "read", "argumentsDelta": "{"},
|
|
{"type": "tool-call-delta", "index": 2, "id": "call-1", "name": "read", "argumentsDelta": "}"},
|
|
]
|
|
|
|
|
|
def test_snapshot_file_builder_order_is_checked_outside_update_mode(tmp_path: Path) -> None:
|
|
compare = SMOKE["compare_snapshot_files"]
|
|
|
|
with pytest.raises(AssertionError, match="snapshot builder produced"):
|
|
compare({}, False, tmp_path, ("result.json",))
|
|
|
|
|
|
def test_snapshot_comparison_expands_sdk_wrapped_attempts() -> None:
|
|
normalize = SMOKE["normalize_session_format_comparison"]
|
|
actual = [{
|
|
"method": "session.event",
|
|
"payload": {
|
|
"sessionId": "s",
|
|
"event": {
|
|
"type": "assistant/attempt",
|
|
"seq": 7,
|
|
"time": 0,
|
|
"data": {
|
|
"turn": 1,
|
|
"step": 1,
|
|
"stream": [{"type": "chunk", "time": 0, "chunk": {"type": "finish"}}],
|
|
},
|
|
},
|
|
},
|
|
}]
|
|
|
|
assert normalize(actual) == [{
|
|
"method": "session.event",
|
|
"payload": {
|
|
"sessionId": "s",
|
|
"event": {
|
|
"type": "assistant/chunk",
|
|
"data": {"turn": 1, "step": 1, "chunk": {"type": "finish"}},
|
|
},
|
|
},
|
|
}]
|
|
|
|
|
|
def test_snapshot_generation_names_select_highest_role_without_double_counting(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
render = SMOKE["snapshot_session_filename"]
|
|
select = SMOKE["selected_snapshot_session_files"]
|
|
assert render(0, 0) == "session.jsonl"
|
|
assert render(0, 2) == "session.v2.jsonl"
|
|
assert render(3, 0) == "session.3.jsonl"
|
|
assert render(3, 2) == "session.3.v2.jsonl"
|
|
|
|
(tmp_path / "session.jsonl").write_text(
|
|
'{"type":"session","version":0}\n', encoding="utf-8",
|
|
)
|
|
(tmp_path / "session.v1.jsonl").write_text(
|
|
'{"type":"session","version":1}\n', encoding="utf-8",
|
|
)
|
|
(tmp_path / "session.1.jsonl").write_text(
|
|
'{"type":"session","version":0}\n', encoding="utf-8",
|
|
)
|
|
|
|
assert {index: path.name for index, path in select(tmp_path).items()} == {
|
|
0: "session.v1.jsonl",
|
|
1: "session.1.jsonl",
|
|
}
|
|
|
|
|
|
def test_snapshot_generation_filename_must_match_header(tmp_path: Path) -> None:
|
|
(tmp_path / "session.v1.jsonl").write_text(
|
|
'{"type":"session","version":0}\n', encoding="utf-8",
|
|
)
|
|
|
|
with pytest.raises(AssertionError, match="filename declares Session format v1"):
|
|
SMOKE["selected_snapshot_session_files"](tmp_path)
|
|
|
|
|
|
@pytest.mark.parametrize("returncode", [1, -1073741819, 3221225477])
|
|
def test_profile_plugin_failure_reports_native_exit_status(monkeypatch: pytest.MonkeyPatch, returncode: int) -> None:
|
|
def failed_install(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.CompletedProcess(args=[], returncode=returncode, stdout="", stderr="")
|
|
|
|
monkeypatch.setattr(subprocess, "run", failed_install)
|
|
with pytest.raises(AssertionError) as error:
|
|
SMOKE["smoke_sdk_profile_plugin"]("http://127.0.0.1:1")
|
|
message = str(error.value)
|
|
assert f"returncode={returncode}" in message
|
|
assert f"0x{returncode & 0xffffffff:08x}" in message
|
|
assert "stdout='' stderr=''" in message
|