mengzhx/deepseek-harness
1
0
Fork 0
mirror of https://github.com/deepseek-ai/deepseek-harness.git synced 2026-09-12 04:01:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge origin/master into feat/remove-str-replace-editor-from-minimal

Browse Source
This commit is contained in:
fz
2026-09-08 23:22:25 +08:00
parent 1f1537914f 96a66ba478
commit 94ff82f57b
1567 changed files with 28869 additions and 7319 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/smoke-python-runtime.py
+179 -58
View File
@@ -126,7 +126,7 @@ ADVANCED_SNAPSHOT_DIRECTORY = (
Path(__file__).resolve().parent / "snapshots" / "python-sdk-single-exe" / "advanced"
)
ADVANCED_SNAPSHOT_FILENAMES = (
"result.json", "session.v2.jsonl", "session.1.v2.jsonl", "session.2.v2.jsonl",
"result.json", "session.v3.jsonl", "session.1.v3.jsonl", "session.2.v3.jsonl",
)
MINIMAL_SNAPSHOT_DIRECTORY = (
Path(__file__).resolve().parent / "snapshots" / "python-sdk-single-exe" / "minimal"
@@ -134,11 +134,15 @@ MINIMAL_SNAPSHOT_DIRECTORY = (
if IS_WINDOWS:
MINIMAL_SNAPSHOT_DIRECTORY /= "win-x64"
MINIMAL_SNAPSHOT_FILENAMES = ("model-visible.json",)
IN_HISTORY_SNAPSHOT_DIRECTORY = (
Path(__file__).resolve().parent / "snapshots" / "python-sdk-single-exe" / "minimal-in-history"
)
IN_HISTORY_SNAPSHOT_FILENAMES = ("prompt-history.json",)
RESTART_SNAPSHOT_DIRECTORY = (
Path(__file__).resolve().parent / "snapshots" / "python-sdk-single-exe" / "restart"
)
RESTART_SNAPSHOT_FILENAMES = (
"result.json", "requests.json", "session.1.v2.jsonl", "session.2.v2.jsonl",
"result.json", "requests.json", "session.1.v3.jsonl", "session.2.v3.jsonl",
)
MCP_SERVER_SCRIPT = """\
import json
@@ -313,7 +317,8 @@ def completion_chunks(body: dict[str, object]) -> list[dict[str, object]]:
messages = body.get("messages")
if not isinstance(messages, list) or not messages:
raise AssertionError(f"model request has no messages: {body}")
latest = messages[-1]
# A system prompt update may follow the tool result without replacing it.
latest = next(message for message in reversed(messages) if message.get("role") != "system")
if not isinstance(latest, dict):
raise AssertionError(f"model request has an invalid latest message: {body}")
@@ -734,7 +739,7 @@ def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--scenario",
choices=("all", "sdk-default", "sdk-custom", "sdk-minimal", "sdk-fs-search", "sdk-spawn-node", "sdk-mcp", "sdk-snapshot", "sdk-restart", "sdk-profile-plugin", "sdk-live", "runner", "direct"),
choices=("all", "sdk-default", "sdk-custom", "sdk-minimal", "sdk-minimal-in-history", "sdk-fs-search", "sdk-spawn-node", "sdk-mcp", "sdk-snapshot", "sdk-restart", "sdk-profile-plugin", "sdk-live", "runner", "direct"),
default="all",
)
parser.add_argument("--exe", type=Path)
@@ -753,10 +758,10 @@ def main() -> None:
parser.error("--scenario sdk-profile-plugin requires --installed-wheel")
if args.installed_wheel:
args.exe = assert_installed_wheel_environment()
if args.scenario in {"all", "sdk-custom", "sdk-minimal", "sdk-fs-search", "sdk-spawn-node", "sdk-snapshot", "sdk-restart", "runner", "direct"} and args.exe is None:
if args.scenario in {"all", "sdk-custom", "sdk-minimal", "sdk-minimal-in-history", "sdk-fs-search", "sdk-spawn-node", "sdk-snapshot", "sdk-restart", "runner", "direct"} and args.exe is None:
parser.error("--exe is required for custom, minimal, fs-search, spawn-node, snapshot, restart, runner, and direct scenarios")
if args.update_snapshots and args.scenario not in {"all", "sdk-minimal", "sdk-snapshot", "sdk-restart"}:
parser.error("--update-snapshots requires --scenario sdk-minimal, sdk-snapshot, sdk-restart, or all")
if args.update_snapshots and args.scenario not in {"all", "sdk-minimal", "sdk-minimal-in-history", "sdk-snapshot", "sdk-restart"}:
parser.error("--update-snapshots requires --scenario sdk-minimal, sdk-minimal-in-history, sdk-snapshot, sdk-restart, or all")
if args.exe is not None and not args.exe.is_file():
parser.error(f"runtime executable does not exist: {args.exe}")
@@ -781,6 +786,9 @@ def main() -> None:
if args.scenario in {"all", "sdk-minimal"}:
assert args.exe is not None
smoke_sdk_minimal(model.url, args.exe.resolve(), args.update_snapshots)
if args.scenario in {"all", "sdk-minimal-in-history"}:
assert args.exe is not None
smoke_sdk_minimal(model.url, args.exe.resolve(), args.update_snapshots, in_history=True)
if args.scenario in {"all", "sdk-fs-search"}:
assert args.exe is not None
smoke_sdk_fs_search(model.url, args.exe.resolve())
@@ -876,13 +884,9 @@ def smoke_sdk_live() -> None:
session_id = "installed-wheel-live-api"
shell_tool = "pwsh" if IS_WINDOWS else "bash"
create_prompt = (
f"Use the {shell_tool} tool to create the file at the absolute path below with exactly one line "
f"containing {LIVE_API_SENTINEL}. Then reply with exactly {LIVE_API_SENTINEL}.\n{marker}"
)
verify_prompt = (
"The file created in the previous turn now contains a new value written by the test. "
"Use a tool to read its current contents without modifying it. "
"Reply with exactly its current single line, not the value from the previous turn."
f"Use the {shell_tool} tool to create the file at the absolute path below with exact UTF-8 "
f"content {LIVE_API_SENTINEL}, with no newline or byte-order mark. "
f"Then reply with exactly {LIVE_API_SENTINEL}.\n{marker}"
)
with DeepSeekHarness(
provider="deepseek-official",
@@ -898,26 +902,37 @@ def smoke_sdk_live() -> None:
request_timeout_seconds=180,
) as harness:
created = harness.run(create_prompt, session_id=session_id)
assert_live_turn("create", created, LIVE_API_SENTINEL)
assert_live_turn("create", created)
if not marker.is_file():
raise AssertionError(f"real-model tool turn did not create {marker}")
if marker.read_text(encoding="utf-8").splitlines() != [LIVE_API_SENTINEL]:
raise AssertionError(f"real-model tool turn wrote unexpected text to {marker}")
# Only a fresh filesystem observation can reveal the verification answer.
verification_value = secrets.token_hex(16)
marker.write_text(verification_value + "\n", encoding="utf-8")
verified = harness.run(verify_prompt, session_id=session_id)
assert_live_turn("verify", verified, verification_value)
raise AssertionError(f"create turn did not create {marker}")
if marker.read_bytes() != LIVE_API_SENTINEL.encode("utf-8"):
raise AssertionError(f"create turn wrote unexpected bytes to {marker}")
if not marker.is_file():
raise AssertionError(f"real-model tool turn did not create {marker}")
if marker.read_text(encoding="utf-8").splitlines() != [verification_value]:
raise AssertionError(f"real-model tool turn wrote unexpected text to {marker}")
# The challenge is absent from the prior turn and the verification prompt.
challenge = secrets.token_hex(32).encode("ascii")
marker.write_bytes(challenge)
with tempfile.TemporaryDirectory(prefix="receipt-", dir=root) as receipt_directory:
receipt = Path(receipt_directory) / "receipt.txt"
verify_prompt = (
"The file created in the previous turn has changed externally. "
"Use a tool to read that same file and copy its exact current content to the "
"new receipt path below, without changing the source file. "
"Preserve every byte; do not add a newline or byte-order mark. "
f"Then reply with exactly {LIVE_API_SENTINEL}.\n{receipt}"
)
verified = harness.run(verify_prompt, session_id=session_id)
assert_live_turn("verify", verified)
if not receipt.is_file():
raise AssertionError(f"verify turn did not create receipt {receipt}")
if receipt.read_bytes() != challenge:
raise AssertionError(f"verify turn wrote unexpected bytes to receipt {receipt}")
if not marker.is_file() or marker.read_bytes() != challenge:
raise AssertionError(f"verify turn changed source file {marker}")
assert_zstd_session_log(sessions)
def assert_live_turn(label: str, result: RunResult, expected: str) -> None:
"""Require a completed model-requested tool turn with the expected response."""
def assert_live_turn(label: str, result: RunResult) -> None:
"""Require completed model tool use and the exact smoke answer for each live turn."""
if result.finish_reason != "completed":
event_types = [event.get("type") for event in result.events]
turn_end_data = next(
@@ -934,10 +949,9 @@ def assert_live_turn(label: str, result: RunResult, expected: str) -> None:
f"{label} turn made no model-requested tool call; "
f"final={result.final_response!r}"
)
if result.final_response.strip() != expected:
if result.final_response.strip() != LIVE_API_SENTINEL:
raise AssertionError(f"{label} turn returned {result.final_response!r}")
def safe_turn_end(value: object) -> object:
"""Project a live-provider failure without retaining credential-bearing text."""
if not isinstance(value, dict):
@@ -1023,7 +1037,9 @@ def smoke_sdk_custom(base_url: str, executable: Path) -> None:
assert_session_log(sessions, root, EXPECTED_TEXT, CODE_WORKER_TEXT, WORKFLOW_WORKER_TEXT)
def smoke_sdk_minimal(base_url: str, executable: Path, update_snapshots: bool) -> None:
def smoke_sdk_minimal(
base_url: str, executable: Path, update_snapshots: bool, *, in_history: bool = False,
) -> None:
"""Exercise the shipped standalone minimal profile through the packaged executable."""
from deepseek_harness import DeepSeekHarness
@@ -1033,6 +1049,19 @@ def smoke_sdk_minimal(base_url: str, executable: Path, update_snapshots: bool) -
root = Path(temporary).resolve()
dsh_home = root / "home"
sessions = dsh_home / "sessions"
patches = ()
if in_history:
patch = root / "in-history.patch.yml"
patch.write_text(json.dumps([
{"id": "llm-deepseek", "config": {"models": [
{"id": "smoke-model", "systemPromptUpdate": "in-history"},
]}},
{"insert": [{
"id": "in-history-prompt",
"name": (Path(__file__).resolve().parent / "fixtures/python-sdk-in-history-prompt.mjs").as_uri(),
}]},
]))
patches = (str(patch),)
with DeepSeekHarness(
provider="deepseek-official",
model="smoke-model",
@@ -1040,6 +1069,7 @@ def smoke_sdk_minimal(base_url: str, executable: Path, update_snapshots: bool) -
dsh_bin=str(executable),
dsh_home=str(dsh_home),
profile="sdk-minimal",
patches=patches,
api_key="sk-keyless-smoke",
base_url=base_url,
request_timeout_seconds=60,
@@ -1051,10 +1081,18 @@ def smoke_sdk_minimal(base_url: str, executable: Path, update_snapshots: bool) -
raise AssertionError(f"minimal agent run emitted no final response: {result.events}")
assert_session_log(sessions, root, MINIMAL_TEXT, "COUNT=1", "COUNT=2")
files = build_minimal_snapshot_files(MockModelHandler.requests[first_request:], root)
compare_snapshot_files(
files, update_snapshots, MINIMAL_SNAPSHOT_DIRECTORY, MINIMAL_SNAPSHOT_FILENAMES,
)
requests = MockModelHandler.requests[first_request:]
if in_history:
logs = read_session_logs(sessions)
files = build_in_history_snapshot_files(result, requests, logs[result.session_id])
compare_snapshot_files(
files, update_snapshots, IN_HISTORY_SNAPSHOT_DIRECTORY, IN_HISTORY_SNAPSHOT_FILENAMES,
)
else:
files = build_minimal_snapshot_files(requests, root)
compare_snapshot_files(
files, update_snapshots, MINIMAL_SNAPSHOT_DIRECTORY, MINIMAL_SNAPSHOT_FILENAMES,
)
def smoke_sdk_fs_search(base_url: str, executable: Path) -> None:
@@ -1288,8 +1326,15 @@ def smoke_sdk_snapshot(base_url: str, executable: Path, update_snapshots: bool)
methods = [notification.method for notification in result.notifications]
if methods.count("subagent.started") != 2 or methods.count("subagent.finished") != 2:
raise AssertionError(f"advanced snapshot emitted unexpected subagent lifecycle: {methods}")
if not any(event.get("type") == "tool/code-dispatch" for event in result.events):
raise AssertionError("advanced snapshot emitted no tool/code-dispatch event")
ptc_events = [event for event in result.events
if event.get("type") in ("tool/ptc-dispatch-start", "tool/ptc-dispatch")]
if [event["type"] for event in ptc_events] != ["tool/ptc-dispatch-start", "tool/ptc-dispatch"]:
raise AssertionError(f"advanced snapshot emitted unexpected PTC dispatch events: {ptc_events}")
for event in ptc_events:
data = event["data"]
identity = (data.get("rootCallId"), data.get("parentCallId"), data.get("subCallId"))
if identity != ("advanced-code", "advanced-code", "advanced-code:ptc:1"):
raise AssertionError(f"advanced snapshot emitted unexpected PTC dispatch identity: {identity}")
logs = read_session_logs(sessions)
child_ids = snapshot_child_ids(result)
@@ -1627,8 +1672,25 @@ def session_header_version(content: str, label: str) -> int:
return version
def assert_current_session_version(version: int, label: str) -> None:
"""Require generated logs to use the source writer generation, independent of goldens."""
source = Path(__file__).resolve().parents[1] / "packages/core/session/src/types.ts"
declarations = re.findall(
r"^export const SESSION_FORMAT_VERSION = ([0-9]+)$",
source.read_text(encoding="utf-8"),
re.MULTILINE,
)
if len(declarations) != 1:
raise AssertionError(f"{source}: expected one literal SESSION_FORMAT_VERSION declaration")
current_version = int(declarations[0])
if version != current_version:
raise AssertionError(
f"{label}: expected current Session format v{current_version}, got v{version}",
)
def assert_persisted_session_version(path: Path, content: str) -> int:
"""Require a raw persistence basename and header to name the same generation."""
"""Require generated persistence filenames and headers to use the current generation."""
filename_version = persisted_session_filename_version(path)
if filename_version is None:
raise AssertionError(f"non-canonical Session persistence filename: {path.name}")
@@ -1638,6 +1700,7 @@ def assert_persisted_session_version(path: Path, content: str) -> int:
f"{path.name}: filename declares Session format v{filename_version}, "
f"header declares v{header_version}",
)
assert_current_session_version(header_version, path.name)
return header_version
@@ -1745,6 +1808,56 @@ def snapshot_child_ids(result: "RunResult") -> list[str]:
return child_ids
def build_in_history_snapshot_files(
result: "RunResult",
requests: list[dict[str, object]],
log: list[dict[str, object]],
) -> dict[str, str]:
"""Assert live requests, SDK subscriptions, and persistence retain both prompts."""
systems = [event for event in result.events if event.get("type") == "system/message"]
assert len(systems) == 2, systems
assert [event.get("surfaceOp") for event in systems] == ["append", "append"], systems
prompts = [message_text(event["data"]["message"]["content"]) for event in systems]
assert "Python SDK prompt version 1." in prompts[0], prompts
assert "Python SDK prompt version 2." not in prompts[0], prompts
assert "Python SDK prompt version 2." in prompts[1], prompts
assert "Python SDK prompt version 1." not in prompts[1], prompts
assert prompts[0] != prompts[1], prompts
assert [event for event in log if event.get("type") == "system/message"] == systems
subscribed = [
notification.payload["event"]
for notification in result.notifications
if notification.method == "session.event"
and notification.payload.get("event", {}).get("type") == "system/message"
]
assert subscribed == systems, subscribed
contexts = [event["data"] for event in result.events if event.get("type") == "request/context"]
assert contexts and all(context.get("systemPromptUpdate") == "in-history" for context in contexts), contexts
assert len([event for event in result.events if event.get("type") == "request/header"]) == 1
assert all(event.get("surfaceOp") in (None, "append") for event in result.events)
first_tool = next(index for index, event in enumerate(result.events) if event.get("type") == "tool/result")
assert result.events.index(systems[1]) > first_tool
assert len(requests) == 4, requests
request_prompts = []
for index, request in enumerate(requests):
messages = request["messages"]
assert messages[0]["role"] == "system" and message_text(messages[0]["content"]) == prompts[0]
assert request["tools"] == requests[0]["tools"], "prompt update changed tool schemas"
positions = [position for position, message in enumerate(messages) if message["role"] == "system"]
texts = [message_text(messages[position]["content"]) for position in positions]
assert texts == (prompts[:1] if index == 0 else prompts), texts
if index > 0:
assert messages[positions[1] - 1]["role"] == "tool", messages
request_prompts.append(texts)
evidence = {
"requestSystemPrompts": request_prompts,
"systemMessageOperations": [event["surfaceOp"] for event in systems],
"subscribedSystemPrompts": prompts,
"requestContexts": contexts,
}
return {"prompt-history.json": json.dumps(evidence, indent=2, ensure_ascii=False) + "\n"}
def build_minimal_snapshot_files(
requests: list[dict[str, object]],
cwd: Path,
@@ -2021,7 +2134,7 @@ def normalize_snapshot_value(
dt = member.get("dt")
if isinstance(dt, list):
member["dt"] = [0] * len(dt)
if isinstance(normalized.get("id"), str) and normalized.get("role") in ("assistant", "user"):
if isinstance(normalized.get("id"), str) and normalized.get("role") in ("assistant", "system", "user"):
if not normalized["id"].startswith("{{message:"):
normalized["id"] = "{{messageId}}"
if normalized.get("type") in ("feedback/message-put", "feedback/message-delete"):
@@ -2033,11 +2146,12 @@ def normalize_snapshot_value(
item["createdAt"] = 0
item["updatedAt"] = 0
scrub_snapshot_header(normalized)
scrub_snapshot_system_message(normalized)
return normalized
def scrub_snapshot_header(value: dict[object, object]) -> None:
"""Tokenize full request-header bulk while retaining tool names."""
"""Tokenize full request-header tool schemas while retaining tool names."""
data = value.get("data")
if not isinstance(data, dict):
return
@@ -2045,8 +2159,6 @@ def scrub_snapshot_header(value: dict[object, object]) -> None:
header = data.get("header")
if not isinstance(header, dict):
return
if "system" in header:
header["system"] = "{{system}}"
tools = header.get("tools")
if isinstance(tools, list):
header["tools"] = [
@@ -2055,6 +2167,20 @@ def scrub_snapshot_header(value: dict[object, object]) -> None:
]
def scrub_snapshot_system_message(value: dict[object, object]) -> None:
"""Tokenize the rendered prompt text of a `system/message` surface node."""
if value.get("type") != "system/message":
return
data = value.get("data")
message = data.get("message") if isinstance(data, dict) else None
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
return
for block in content:
if isinstance(block, dict) and block.get("type") == "text":
block["text"] = "{{system}}"
def render_jsonl(records: list[object]) -> str:
"""Render parsed JSON values as compact, newline-terminated JSONL."""
return "".join(
@@ -2192,19 +2318,6 @@ def normalize_session_format_comparison(
normalized.pop("time", None)
if source_session_version == 1 and normalized.get("type") == "assistant/message":
normalized.pop("sourceEventSeqs", None)
if normalized.get("type") == "session-log-deepseek/delivery-accepted":
data = normalized.get("data")
if isinstance(data, dict):
data.pop("throughSeq", None)
data.pop("sessionFormatVersion", None)
data["sessionFormatVersion"] = SESSION_FORMAT_PROVENANCE
if normalized.get("kind") == "session-reference":
references = normalized.get("references")
if isinstance(references, list):
for reference in references:
if isinstance(reference, dict):
reference.pop("capturedFormatVersion", None)
reference["capturedFormatVersion"] = SESSION_FORMAT_PROVENANCE
return normalized
@@ -2237,10 +2350,18 @@ def compare_snapshot_files(
directory: Path,
filenames: tuple[str, ...],
) -> None:
"""Write or exactly compare one scenario's expected snapshot files."""
"""Compare ordered artifact roles and Session content across generations, or write generated filenames."""
scenario = directory.name
if tuple(files) != filenames:
def role_name(name: str) -> str:
parsed = parse_snapshot_session_filename(name)
return name if parsed is None else snapshot_session_filename(parsed[0], 0)
if tuple(map(role_name, files)) != tuple(map(role_name, filenames)):
raise AssertionError(f"{scenario} snapshot builder produced {tuple(files)}, expected {filenames}")
for name, content in files.items():
if parse_snapshot_session_filename(name) is not None:
assert_current_session_version(session_header_version(content, name), name)
if update:
directory.mkdir(parents=True, exist_ok=True)
for name, content in files.items():