diff --git a/.agents/notes/implemented/testing/2026-09-06-pr-ci-runner-temporary-storage.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-pr-ci-runner-temporary-storage.i18n.yaml index eeab915207..a5ddda57d2 100644 --- a/.agents/notes/implemented/testing/2026-09-06-pr-ci-runner-temporary-storage.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-pr-ci-runner-temporary-storage.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-06-pr-ci-runner-temporary-storage.md -2026-09-06-pr-ci-runner-temporary-storage.md: 29c4b52afdd366ba3b58b70371c10ab261a50b67 -2026-09-06-pr-ci-runner-temporary-storage.zh.md: b653c3b5bc17b9fadd3e1515b13c0f23b106e531 +2026-09-06-pr-ci-runner-temporary-storage.md: 31fe62dd23fac4b95f5c7f5ddaf93552bc8ea5a6 +2026-09-06-pr-ci-runner-temporary-storage.zh.md: 5e7863e44f2cbba22c56a7658d6c89aa938c8c6f diff --git a/.agents/notes/implemented/testing/2026-09-06-pr-ci-runner-temporary-storage.md b/.agents/notes/implemented/testing/2026-09-06-pr-ci-runner-temporary-storage.md index 29c4b52afd..31fe62dd23 100644 --- a/.agents/notes/implemented/testing/2026-09-06-pr-ci-runner-temporary-storage.md +++ b/.agents/notes/implemented/testing/2026-09-06-pr-ci-runner-temporary-storage.md @@ -24,6 +24,10 @@ The [ACP diagnostic scenario](../../../../snapshots/session/subagent-acp-diagnos The headless `session-sandbox-root` fixture declares `workspace.parent: outside-temp`, not a home-filesystem dependency. Its allocator uses a sibling of the canonical platform temp root where the parent is writable and avoids system temporary grants, otherwise home, and rejects a cwd already covered by automatic temporary write grants. On the failover runner this keeps the test on the data volume without making its write succeed through a temporary-directory exemption. The filesystem-sandbox containment tests use the same allocator for their workspace and denied sibling; they register cleanup immediately after successful acquisition. Atomic workspace allocation, recorded Session bytes, and the independent expected file remain unchanged. +## Retry-exposed fixture synchronization + +The Inspector console integration test waits for a Client `Runtime.evaluate` round trip after enablement before issuing its separate fixture log command. Worker-side context announcements alone do not prove that the client has consumed its console-enable message. The installed-wheel live SDK test externally replaces the created file with a fresh host-only challenge before asking the model to verify it; the verification prompt does not reveal that value. Both turns must still contain model-requested tool calls, and the verifier compares the returned value and actual file bytes. + ## Alternatives considered **Delete shared temporary files from a PR job.** Another runner may still own those files. Repository jobs must not reclaim a shared directory by pathname or age. diff --git a/.agents/notes/implemented/testing/2026-09-06-pr-ci-runner-temporary-storage.zh.md b/.agents/notes/implemented/testing/2026-09-06-pr-ci-runner-temporary-storage.zh.md index b653c3b5bc..5e7863e44f 100644 --- a/.agents/notes/implemented/testing/2026-09-06-pr-ci-runner-temporary-storage.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-pr-ci-runner-temporary-storage.zh.md @@ -24,6 +24,10 @@ Linux 故障切换池在同一台虚拟机上运行多个 runner 实例。PR 覆 Headless 的 `session-sandbox-root` 夹具声明 `workspace.parent: outside-temp`，而不是依赖 home 所在文件系统。分配器在父目录可写且避开系统临时授权时选择规范化平台临时根目录的同级目录，否则使用 home，并拒绝已被自动临时写授权覆盖的 cwd。在故障切换 runner 上，这让测试留在数据卷中，同时不会让写入借助临时目录豁免而成功。文件系统沙箱的包含关系测试使用同一分配器创建 workspace 及被拒绝的同级目录，并在成功获取目录后立即注册清理。原子 workspace 分配、录制的 Session 字节以及独立预期文件保持不变。 +## 重跑暴露的夹具同步问题 + +Inspector console 集成测试在启用后等待 Client 的 `Runtime.evaluate` 往返，再发出独立的夹具日志命令。仅有 Worker 侧的 context 公告不能证明客户端已消费 console-enable 消息。已安装 wheel 的在线 SDK 测试在要求模型验证前，由外部将创建的文件替换为新的、仅主机知道的挑战值；验证提示不暴露该值。两个 turn 仍必须包含模型请求的工具调用，验证器同时比较返回值及真实文件字节。 + ## 考虑过的替代方案 **由 PR 作业删除共享临时文件。** 其他 runner 可能仍在使用这些文件。仓库作业不得按路径或文件年龄回收共享目录。 diff --git a/packages/experimental/inspector/tests/integration.host.spec.ts b/packages/experimental/inspector/tests/integration.host.spec.ts index b50ea4aced..3219b9ce93 100644 --- a/packages/experimental/inspector/tests/integration.host.spec.ts +++ b/packages/experimental/inspector/tests/integration.host.spec.ts @@ -370,6 +370,13 @@ describe('experimental Inspector real Worker', () => { await Promise.all([cdp.call('Runtime.enable'), secondCdp.call('Runtime.enable')]) const firstContext = await clientContext(cdp) const secondContext = await clientContext(secondCdp) + // Context announcements do not acknowledge Client consumption of console-enable frames. + // Round-trip the ordered ingest socket before logging over the fixture's separate MessagePort. + for (const [connection, contextId] of [[cdp, firstContext], [secondCdp, secondContext]] as const) { + await expect(connection.call('Runtime.evaluate', { contextId, expression: '0' })).resolves.toMatchObject({ + result: { result: { type: 'number', value: 0 } }, + }) + } const value = { owner: 'client-console' } const marker = 'client-console-event' await client.log(value, marker) diff --git a/python/sdk/tests/test_smoke_model.py b/python/sdk/tests/test_smoke_model.py index 55570501a9..f2d2a4cb11 100644 --- a/python/sdk/tests/test_smoke_model.py +++ b/python/sdk/tests/test_smoke_model.py @@ -3,6 +3,7 @@ from __future__ import annotations import runpy import subprocess from pathlib import Path +from types import SimpleNamespace import pytest @@ -11,6 +12,88 @@ ROOT = Path(__file__).resolve().parents[3] SMOKE = runpy.run_path(ROOT / "scripts" / "smoke-python-runtime.py") +@pytest.mark.parametrize( + ("behavior", "error"), + [ + ("read-current", None), + ("no-verify-tool", "verify turn made no model-requested tool call"), + ("no-create-tool", "create turn made no model-requested tool call"), + ("create-error", "create turn ended with.*AUTH.*401"), + ("stale-answer", "verify turn returned"), + ("missing-create", "real-model tool turn did not create"), + ("wrong-create", "real-model tool turn wrote unexpected text"), + ("modify-verify", "real-model tool turn wrote unexpected text"), + ], +) +def test_live_smoke_requires_fresh_file_observation( + monkeypatch: pytest.MonkeyPatch, behavior: str, error: str | None, +) -> None: + import deepseek_harness + + smoke_live = SMOKE["smoke_sdk_live"] + sentinel = SMOKE["LIVE_API_SENTINEL"] + prompts: list[str] = [] + session_ids: list[str] = [] + log_checks: list[Path] = [] + + class ScriptedHarness: + def __init__(self, *, cwd: str, **_kwargs: object) -> None: + self.marker = Path(cwd) / "live-api-marker.txt" + + def __enter__(self) -> ScriptedHarness: + return self + + def __exit__(self, *_args: object) -> None: + pass + + def run(self, prompt: str, *, session_id: str) -> SimpleNamespace: + prompts.append(prompt) + session_ids.append(session_id) + events = [{"type": "tool/call"}] + if len(prompts) == 1: + assert not self.marker.exists() + if behavior == "create-error": + return SimpleNamespace(finish_reason="error", final_response="", events=[{ + "type": "turn/end", + "data": {"turn": 1, "reason": { + "kind": "error", "error": {"code": "AUTH", "status": 401}, + }}, + }]) + if behavior == "no-create-tool": + events = [] + if behavior != "missing-create": + self.marker.write_text( + ("wrong" if behavior == "wrong-create" else sentinel) + "\n", + encoding="utf-8", + ) + response = sentinel + else: + current = self.marker.read_text(encoding="utf-8").strip() + assert current != sentinel, "verification must require new world state" + assert all(current not in text for text in prompts), "prompts must not reveal the answer" + response = sentinel if behavior == "stale-answer" else current + if behavior == "no-verify-tool": + events = [] + if behavior == "modify-verify": + self.marker.write_text("changed\n", encoding="utf-8") + return SimpleNamespace(finish_reason="completed", final_response=response, events=events) + + monkeypatch.setenv("DEEPSEEK_API_KEY", "fixture-key") + monkeypatch.setenv("DEEPSEEK_BASE_URL", "https://fixture.invalid") + monkeypatch.setattr(deepseek_harness, "DeepSeekHarness", ScriptedHarness) + monkeypatch.setitem(smoke_live.__globals__, "assert_zstd_session_log", log_checks.append) + + if error is None: + smoke_live() + assert len(prompts) == 2 + assert session_ids[0] == session_ids[1] + assert len(log_checks) == 1 + else: + with pytest.raises(AssertionError, match=error): + smoke_live() + assert not log_checks + + @pytest.mark.parametrize( ("prompt_name", "expected"), [ diff --git a/scripts/smoke-python-runtime.py b/scripts/smoke-python-runtime.py index 0a1331d718..30c8334571 100644 --- a/scripts/smoke-python-runtime.py +++ b/scripts/smoke-python-runtime.py @@ -11,6 +11,7 @@ import json import os import queue import re +import secrets import shutil import subprocess import sys @@ -908,8 +909,9 @@ def smoke_sdk_live() -> None: f"containing {LIVE_API_SENTINEL}. Then reply with exactly {LIVE_API_SENTINEL}.\n{marker}" ) verify_prompt = ( - "Use a tool to read the file created in the previous turn. " - f"If its only line is {LIVE_API_SENTINEL}, reply with exactly {LIVE_API_SENTINEL}." + "The file created in the previous turn now contains a new value written by the test. " + "Use a tool to read its current contents without modifying it. " + "Reply with exactly its current single line, not the value from the previous turn." ) with DeepSeekHarness( provider="deepseek-official", @@ -925,34 +927,46 @@ def smoke_sdk_live() -> None: request_timeout_seconds=180, ) as harness: created = harness.run(create_prompt, session_id=session_id) + assert_live_turn("create", created, LIVE_API_SENTINEL) + if not marker.is_file(): + raise AssertionError(f"real-model tool turn did not create {marker}") + if marker.read_text(encoding="utf-8").splitlines() != [LIVE_API_SENTINEL]: + raise AssertionError(f"real-model tool turn wrote unexpected text to {marker}") + # Only a fresh filesystem observation can reveal the verification answer. + verification_value = secrets.token_hex(16) + marker.write_text(verification_value + "\n", encoding="utf-8") verified = harness.run(verify_prompt, session_id=session_id) + assert_live_turn("verify", verified, verification_value) - for label, result in (("create", created), ("verify", verified)): - if result.finish_reason != "completed": - event_types = [event.get("type") for event in result.events] - turn_end_data = next( - (event.get("data") for event in reversed(result.events) if event.get("type") == "turn/end"), - None, - ) - turn_end = safe_turn_end(turn_end_data) - raise AssertionError( - f"{label} turn ended with {result.finish_reason!r}; " - f"final={result.final_response!r}; turn_end={turn_end!r}; events={event_types}" - ) - if not any(event.get("type") == "tool/call" for event in result.events): - raise AssertionError( - f"{label} turn made no model-requested tool call; " - f"final={result.final_response!r}" - ) - if result.final_response.strip() != LIVE_API_SENTINEL: - raise AssertionError(f"{label} turn returned {result.final_response!r}") if not marker.is_file(): raise AssertionError(f"real-model tool turn did not create {marker}") - if marker.read_text(encoding="utf-8").splitlines() != [LIVE_API_SENTINEL]: + if marker.read_text(encoding="utf-8").splitlines() != [verification_value]: raise AssertionError(f"real-model tool turn wrote unexpected text to {marker}") assert_zstd_session_log(sessions) +def assert_live_turn(label: str, result: RunResult, expected: str) -> None: + """Require a completed model-requested tool turn with the expected response.""" + if result.finish_reason != "completed": + event_types = [event.get("type") for event in result.events] + turn_end_data = next( + (event.get("data") for event in reversed(result.events) if event.get("type") == "turn/end"), + None, + ) + turn_end = safe_turn_end(turn_end_data) + raise AssertionError( + f"{label} turn ended with {result.finish_reason!r}; " + f"final={result.final_response!r}; turn_end={turn_end!r}; events={event_types}" + ) + if not any(event.get("type") == "tool/call" for event in result.events): + raise AssertionError( + f"{label} turn made no model-requested tool call; " + f"final={result.final_response!r}" + ) + if result.final_response.strip() != expected: + raise AssertionError(f"{label} turn returned {result.final_response!r}") + + def safe_turn_end(value: object) -> object: """Project a live-provider failure without retaining credential-bearing text.""" if not isinstance(value, dict):