mengzhx/deepseek-harness
1
0
Fork 0
mirror of https://github.com/deepseek-ai/deepseek-harness.git synced 2026-08-29 04:26:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(web): snapshot blocked loopback fetch

Browse Source
This commit is contained in:
Dudu-0223
2026-08-24 11:57:27 +08:00
parent b2219bba63
commit 9d5fa7a593
6 changed files with 12 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/acp-agent/tests/acp.snapshot.ts
+4 -5
View File
@@ -350,11 +350,10 @@ const SCENARIOS: Scenario[] = [
prepareWorkspace: prepareEditingCordisSkillWorkspace,
},
{ name: 'lsp-definition', hasModelTurn: true, recorded: false, pinsHeader: true, headerClass: 'lsp', configPath: LSP_CONFIG },
// web_fetch markdown rendering end to end: the overlay's loopback fixture
// server supplies deterministic HTML (entities, a GFM table, nesting), the
// REAL local fetch provider retrieves it, and the tool result pins the
// turndown conversion. The fetched URL (fixed port) is part of the recorded
// transcript; replay re-executes the real fetch against the same fixture.
// web_fetch non-public-address rejection end to end: the real provider
// resolves the recorded loopback target and the result pins the failed tool
// call. The fixed URL is part of the recorded transcript; replay re-executes
// the real network policy without opening a connection.
{ name: 'web-fetch', hasModelTurn: true, recorded: true, pinsHeader: true, headerClass: 'web', configPath: WEB_CONFIG },
{
name: 'workspace-edit',
examples/acp-agent/tests/snapshots/web-fetch/session.jsonl
+1 -1
View File
@@ -21,7 +21,7 @@
{"type":"assistant/chunk","data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}
{"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"reasoning","text":"The user wants me to use the web_fetch tool exactly once to fetch http://127.0.0.1:43117/menu.html, then reply with exactly \"DONE\". Let me do that."},{"type":"tool-call","id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","arguments":"{\"url\": \"http://127.0.0.1:43117/menu.html\"}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-pro"},"id":"63b78628-921c-4d56-aaa3-ea8e61c54da2"},"usage":{"inputTokens":5405,"outputTokens":103,"cacheReadTokens":0,"reasoningTokens":44}},"sourceEventSeqs":[12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85],"surfaceOp":"append"}
{"type":"tool/call","data":{"turn":1,"step":1,"callId":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","arguments":"{\"url\": \"http://127.0.0.1:43117/menu.html\"}"}}
{"type":"tool/result","data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_00_sxjOyfDYN07koiE7jiIa5326"},"content":[{"type":"tool-result","toolCallId":"call_00_sxjOyfDYN07koiE7jiIa5326","content":[{"type":"text","text":"Fetched http://127.0.0.1:43117/menu.html (HTTP 200)\n\nMenu\n\n# Café menu\n\nPrices include **service & _tax_** — updated daily.\n\n- Espresso\n- Flat white\n\n| Drink | Price |\n| --- | --- |\n| Espresso | €2 |\n| Flat white | €3 |\n\nSee [today’s specials](https://fixture.invalid/specials)."}],"isError":false}],"role":"user","id":"f78dd40c-94c1-4007-b3c2-a8bd3729c43f"},"meta":{"url":"http://127.0.0.1:43117/menu.html","statusCode":200,"truncated":false}},"sourceEventSeqs":[87],"surfaceOp":"append"}
{"type":"tool/result","data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_00_sxjOyfDYN07koiE7jiIa5326"},"content":[{"type":"tool-result","toolCallId":"call_00_sxjOyfDYN07koiE7jiIa5326","content":[{"type":"text","text":"Error: URL hostname \"127.0.0.1\" resolves to a non-public IP address"}],"isError":true}],"role":"user","id":"fa26e713-d7f8-4db9-aed3-fc13c74f90f7"},"error":{"name":"WebError","code":"WEB_BLOCKED_URL"}},"sourceEventSeqs":[87],"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":1}}
{"type":"step/start","data":{"turn":1,"step":2}}
{"type":"assistant/chunk","data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}}
examples/acp-agent/tests/snapshots/web-fetch/stdout.expected.jsonl
+1 -1
View File
@@ -2,7 +2,7 @@
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}","configOptions":[{"id":"model","name":"Model","category":"model","type":"select","currentValue":"[\"deepseek-official\",\"deepseek-v4-pro\"]","options":[{"group":"deepseek-official","name":"DeepSeek","options":[{"value":"[\"deepseek-official\",\"deepseek-v4-flash\"]","name":"deepseek-v4-flash"},{"value":"[\"deepseek-official\",\"deepseek-v4-pro\"]","name":"deepseek-v4-pro"}]}]}]}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"The user wants me to use the web_fetch tool exactly once to fetch http://127.0.0.1:43117/menu.html, then reply with exactly \"DONE\". Let me do that."}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_00_sxjOyfDYN07koiE7jiIa5326","title":"web_fetch","kind":"other","status":"in_progress","rawInput":{"url":"http://127.0.0.1:43117/menu.html"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_00_sxjOyfDYN07koiE7jiIa5326","status":"completed","content":[{"type":"content","content":{"type":"text","text":"Fetched http://127.0.0.1:43117/menu.html (HTTP 200)\n\nMenu\n\n# Café menu\n\nPrices include **service & _tax_** — updated daily.\n\n- Espresso\n- Flat white\n\n| Drink | Price |\n| --- | --- |\n| Espresso | €2 |\n| Flat white | €3 |\n\nSee [today’s specials](https://fixture.invalid/specials)."}}]}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_00_sxjOyfDYN07koiE7jiIa5326","status":"failed","content":[{"type":"content","content":{"type":"text","text":"Error: URL hostname \"127.0.0.1\" resolves to a non-public IP address"}}]}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"The user asked me to fetch the URL, then reply with exactly \"DONE\". I've fetched it. Now I just reply with \"DONE\"."}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"DONE"}}}}
{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}
examples/acp-agent/web-fetch-fixture-server.mjs
-55
View File
@@ -1,55 +0,0 @@
/**
* Deterministic loopback HTTP fixture for the web-fetch snapshot scenario: a
* small HTML page (headings, named entities, a GFM table, nested formatting)
* on a fixed port, so recording and keyless replay drive the REAL
* `dsh-web-fetch-http` transport and `dsh-tool-web` markdown rendering
* without external network. The port is fixed because the fetched URL is part
* of the recorded model transcript.
*/
import { createServer } from 'node:http'
/** Fixed loopback port the scenario prompt points `web_fetch` at. */
const PORT = 43117
const PAGE = `<!doctype html>
<html><head><title>Menu</title><style>.x{color:red}</style><script>ignored()</script></head>
<body>
<h1>Caf&eacute; menu</h1>
<p>Prices include <strong>service &amp; <em>tax</em></strong> &mdash; updated daily.</p>
<ul><li>Espresso</li><li>Flat white</li></ul>
<table><thead><tr><th>Drink</th><th>Price</th></tr></thead><tbody><tr><td>Espresso</td><td>&euro;2</td></tr><tr><td>Flat white</td><td>&euro;3</td></tr></tbody></table>
<p>See <a href="https://fixture.invalid/specials">today&rsquo;s specials</a>.</p>
</body></html>
`
/** Cordis plugin name. */
export const name = 'web-fetch-fixture-server'
/**
* Start the fixture server on 127.0.0.1 and register its shutdown.
* @param ctx - Cordis context; the effect disposes the server with the fiber.
*/
export async function apply(ctx) {
const server = createServer((req, res) => {
if (req.url === '/menu.html') {
res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' })
res.end(PAGE)
return
}
res.writeHead(404, { 'content-type': 'text/plain; charset=utf-8' })
res.end('not found')
})
await new Promise((resolve, reject) => {
server.once('error', reject)
server.listen(PORT, '127.0.0.1', () => resolve(undefined))
})
// The fixture must never hold the process open past protocol shutdown.
server.unref()
ctx.effect(() => async () => {
await new Promise((resolve, reject) => {
server.close(error => error ? reject(error) : resolve(undefined))
// Stop accepting first so a connection cannot arrive after the forced close.
server.closeAllConnections()
})
}, 'web-fetch-fixture-server')
}
examples/acp-agent/web.cordis.snapshot.yml
+2 -5
View File
@@ -1,6 +1,5 @@
# Keyless replay counterpart to web.cordis.yml: the web stack and loopback
# fixture server stay real (the tool call re-executes the actual HTTP fetch and
# markdown rendering); only the model adapter is replaced by replay.
# Keyless replay counterpart to web.cordis.yml: the real provider rejects the
# recorded loopback target; only the model adapter is replaced by replay.
- id: llm-deepseek
name: '@deepseek-ai/dsh-llm-deepseek'
disabled: true
@@ -8,8 +7,6 @@
- insert:
- id: web-fetch-http
name: '@deepseek-ai/dsh-web-fetch-http'
- id: web-fetch-fixture
name: './web-fetch-fixture-server.mjs'
- id: llm-replay
name: '@deepseek-ai/dsh-llm-replay'
config:
examples/acp-agent/web.cordis.yml
+4 -6
View File
@@ -1,13 +1,11 @@
# Web-fetch composition for the web-fetch snapshot scenario: the web seam, the
# real local HTTP fetch provider, the model-facing web tools (fetch only, so
# the pinned header carries exactly the surface under test), and the loopback
# fixture server the scenario prompt fetches — deterministic content, no
# external network, in recording and replay alike.
# real local HTTP fetch provider, and the model-facing web tools (fetch only,
# so the pinned header carries exactly the surface under test). The recorded
# loopback target exercises the provider's non-public-address rejection without
# opening a network connection.
- insert:
- id: web-fetch-http
name: '@deepseek-ai/dsh-web-fetch-http'
- id: web-fetch-fixture
name: './web-fetch-fixture-server.mjs'
- id: web
name: '@deepseek-ai/dsh-web'