mengzhx/deepseek-harness
1
0
Fork 0
mirror of https://github.com/deepseek-ai/deepseek-harness.git synced 2026-09-11 04:00:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3313 from deepseek-harness/worktree-llme2e

Browse Source 
test(e2e): use Flash for all live model coverage
This commit is contained in:
imccyu
2026-08-29 17:36:05 +08:00
committed by GitHub
parent 1404ded8b1 08bfeda7e8
commit d5e2f22b08
6 changed files with 51 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/llm/llm-deepseek/tests/adapter.e2e.ts
+19 -25
View File
@@ -28,14 +28,13 @@ import type { Config } from '@deepseek-ai/dsh-llm-deepseek'
import { assemble, type AssembledResult } from './assemble.ts'
/**
* Real-API e2e for the direct-fetch adapter: V4 Flash + V4 Pro across
* thinking modes and all official effort levels. The suite skips entirely
* without $DEEPSEEK_API_KEY; the pre-release vision smoke additionally
* Real-API e2e for the direct-fetch adapter: V4 Flash across thinking modes
* and a max-effort tool round trip with reasoning passback. The suite skips
* entirely without $DEEPSEEK_API_KEY; the pre-release vision smoke additionally
* requires $DEEPSEEK_VISION_E2E=1 (see vitest.e2e.config.ts).
*/
const FLASH = 'deepseek-v4-flash'
const PRO = 'deepseek-v4-pro'
const VISION = 'deepseek-v4-flash-vision-exp'
const VISION_E2E_ENABLED = process.env.DEEPSEEK_VISION_E2E === '1'
const TEST_PNG = Uint8Array.from(readFileSync(
@@ -266,20 +265,23 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('llm-deepseek e2e (real API)', ()
expect(withThinking.usage?.reasoningTokens).toBeGreaterThan(0)
})
it.each(['high', 'max'] as const)(
'pro + thinking enabled (effort %s): tool-call round trip with reasoning passback',
async (effort) => {
const ctx = await harness(PRO, { thinking: 'enabled' })
it(
'flash + thinking enabled (effort max): tool-call round trip with reasoning passback',
async () => {
const ctx = await harness(FLASH, { thinking: 'enabled' })
// Turn 1: the model must call the tool (and think before it).
const first = await assemble(ctx,{
model: PRO,
reasoningEffort: ReasoningEffortId(effort),
model: FLASH,
reasoningEffort: ReasoningEffortId('max'),
messages: ask('What is the weather in Paris right now? Use the get_weather tool.'),
tools: [weatherTool],
maxTokens: 2000,
})
expect(first.finish.kind).toBe('tool-calls')
expect(
first.finish.kind,
`DeepSeek Flash tool-call turn finished as ${JSON.stringify(first.finish)}`,
).toBe('tool-calls')
const call = first.message.content.find(block => block.type === 'tool-call')
expect(call).toBeDefined()
expect(call!.name).toBe('get_weather')
@@ -288,8 +290,8 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('llm-deepseek e2e (real API)', ()
// Turn 2: send the tool result back WITH the assistant's reasoning
// block in history (the official thinking+tools passback rule).
const second = await assemble(ctx,{
model: PRO,
reasoningEffort: ReasoningEffortId(effort),
model: FLASH,
reasoningEffort: ReasoningEffortId('max'),
messages: [
...ask('What is the weather in Paris right now? Use the get_weather tool.'),
createMessage({
@@ -308,22 +310,14 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('llm-deepseek e2e (real API)', ()
tools: [weatherTool],
maxTokens: 2000,
})
expect(second.finish.kind).toBe('stop')
expect(
second.finish.kind,
`DeepSeek Flash tool-result turn finished as ${JSON.stringify(second.finish)}`,
).toBe('stop')
expect(textOf(second).toLowerCase()).toMatch(/sunny|22/)
},
)
it('pro + thinking disabled: plain generation without reasoning blocks', async () => {
const ctx = await harness(PRO, { thinking: 'disabled' })
const result = await assemble(ctx,{
model: PRO,
messages: ask('Reply with exactly the word: pong'),
maxTokens: 50,
})
expect(result.finish.kind).toBe('stop')
expect(result.message.content.some(block => block.type === 'reasoning')).toBe(false)
})
it('streams raw chunks in protocol order', async () => {
const ctx = await harness(FLASH, { thinking: 'disabled' })
const kinds: string[] = []
packages/llm/llm-deepseek/tests/adapter.spec.ts
+2 -2
View File
@@ -338,7 +338,7 @@ describe('DeepSeekAdapter against a mock server', () => {
const ctx = await harness(server.url)
const result = await assemble(ctx, {
model: 'deepseek-v4-flash',
model: 'deepseek-v4-pro',
messages: [createUserMessage({
content: [{ type: 'text', text: 'hi' }],
source: { kind: 'plugin', plugin: 'test' },
@@ -350,7 +350,7 @@ describe('DeepSeekAdapter against a mock server', () => {
// The wire request carried the auth header contents we configured.
expect(server.requests[0]).toMatchObject({
model: 'deepseek-v4-flash',
model: 'deepseek-v4-pro',
max_tokens: 256_000,
reasoning_effort: 'high',
stream: true,
packages/llm/llm-pi-ai/tests/adapter.e2e.ts
+21 -17
View File
@@ -8,14 +8,12 @@ import * as LlmDeepSeek from '@deepseek-ai/dsh-llm-deepseek'
import { assemble, type AssembledResult } from './assemble.ts'
/**
* Real-API e2e for the pi-ai-backed adapter: V4 Flash + V4 Pro with provider
* defaults and representative off/high/max reasoning. Mirrors the native
* adapter's StreamChunk contract and exercises a replayed tool follow-up.
* Key-gated.
* Real-API e2e for the pi-ai-backed adapter: V4 Flash defaults and
* off/high/max reasoning. Mirrors the native adapter's StreamChunk contract
* and exercises a replayed tool follow-up. Key-gated.
*/
const FLASH = 'deepseek-v4-flash'
const PRO = 'deepseek-v4-pro'
const contexts: Context[] = []
async function harness(_model: string, config: Partial<PiAiProviderProfile> = {}) {
@@ -67,10 +65,10 @@ const weatherTool: ToolSchema = {
}
describe.skipIf(!process.env.DEEPSEEK_API_KEY)('llm-pi-ai e2e (real API)', () => {
it.each([FLASH, PRO])('%s + provider-default reasoning: plain text generation', async (model) => {
const ctx = await harness(model)
it(`${FLASH} + provider-default reasoning: plain text generation`, async () => {
const ctx = await harness(FLASH)
const result = await assemble(ctx,{
model,
model: FLASH,
messages: ask('Reply with exactly the word: pong'),
maxTokens: 50,
})
@@ -91,10 +89,10 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('llm-pi-ai e2e (real API)', () =>
expect(textOf(result).toLowerCase()).toContain('pong')
})
it.each([FLASH, PRO])('%s + reasoning high: reasoning blocks present', async (model) => {
const ctx = await harness(model)
it(`${FLASH} + reasoning high: reasoning blocks present`, async () => {
const ctx = await harness(FLASH)
const result = await assemble(ctx,{
model,
model: FLASH,
reasoningEffort: ReasoningEffortId('high'),
messages: ask('Which is larger, 9.11 or 9.8? Answer with just the number.'),
maxTokens: 2000,
@@ -104,24 +102,27 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('llm-pi-ai e2e (real API)', () =>
expect(textOf(result)).toContain('9.8')
})
it('pro + reasoning max: tool-call round trip', async () => {
const ctx = await harness(PRO)
it('flash + reasoning max: tool-call round trip', async () => {
const ctx = await harness(FLASH)
const first = await assemble(ctx,{
model: PRO,
model: FLASH,
reasoningEffort: ReasoningEffortId('max'),
messages: ask('What is the weather in Paris right now? Use the get_weather tool.'),
tools: [weatherTool],
maxTokens: 2000,
})
expect(first.finish.kind).toBe('tool-calls')
expect(
first.finish.kind,
`pi-ai Flash tool-call turn finished as ${JSON.stringify(first.finish)}`,
).toBe('tool-calls')
const call = first.message.content.find(block => block.type === 'tool-call')
expect(call).toBeDefined()
expect(call!.name).toBe('get_weather')
expect(JSON.parse(call!.arguments)).toMatchObject({ city: expect.stringMatching(/paris/i) as string })
const second = await assemble(ctx,{
model: PRO,
model: FLASH,
reasoningEffort: ReasoningEffortId('max'),
messages: [
...ask('What is the weather in Paris right now? Use the get_weather tool.'),
@@ -138,7 +139,10 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('llm-pi-ai e2e (real API)', () =>
tools: [weatherTool],
maxTokens: 2000,
})
expect(second.finish.kind).toBe('stop')
expect(
second.finish.kind,
`pi-ai Flash tool-result turn finished as ${JSON.stringify(second.finish)}`,
).toBe('stop')
expect(textOf(second).toLowerCase()).toMatch(/sunny|22/)
})
packages/llm/llm-pi-ai/tests/adapter.spec.ts
+2 -2
View File
@@ -135,14 +135,14 @@ describe('PiAiAdapter provider routing', () => {
thinkingBudgets: { high: 2048 },
})
await assemble(ctx, {
model: 'deepseek-v4-flash',
model: 'deepseek-v4-pro',
messages: [],
temperature: 0.2,
maxTokens: 77,
sessionId: 'session-for-pi' as never,
})
expect(server.requests[0]).toMatchObject({
model: 'deepseek-v4-flash',
model: 'deepseek-v4-pro',
temperature: 0.2,
max_tokens: 77,
thinking: { type: 'enabled' },
packages/subagent/subagent-claude-code/tests/real-deepseek.e2e.ts
+6 -5
View File
@@ -21,6 +21,7 @@ import * as claudeCode from '../src/index.ts'
const execFileAsync = promisify(execFile)
const OFFICIAL_DEEPSEEK_BASE_URL = 'https://api.deepseek.com'
const DEEPSEEK_MODEL = 'deepseek-v4-flash'
const sdkRoot = dirname(fileURLToPath(
import.meta.resolve('@anthropic-ai/claude-agent-sdk'),
))
@@ -90,11 +91,11 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)(
const env = {
ANTHROPIC_AUTH_TOKEN: apiKey,
ANTHROPIC_BASE_URL: `${deepSeekBaseUrl()}/anthropic`,
ANTHROPIC_MODEL: 'deepseek-v4-pro[1m]',
ANTHROPIC_DEFAULT_OPUS_MODEL: 'deepseek-v4-pro[1m]',
ANTHROPIC_DEFAULT_SONNET_MODEL: 'deepseek-v4-pro[1m]',
ANTHROPIC_DEFAULT_HAIKU_MODEL: 'deepseek-v4-flash',
CLAUDE_CODE_SUBAGENT_MODEL: 'deepseek-v4-flash',
ANTHROPIC_MODEL: DEEPSEEK_MODEL,
ANTHROPIC_DEFAULT_OPUS_MODEL: DEEPSEEK_MODEL,
ANTHROPIC_DEFAULT_SONNET_MODEL: DEEPSEEK_MODEL,
ANTHROPIC_DEFAULT_HAIKU_MODEL: DEEPSEEK_MODEL,
CLAUDE_CODE_SUBAGENT_MODEL: DEEPSEEK_MODEL,
CLAUDE_CODE_EFFORT_LEVEL: 'max',
CLAUDE_CONFIG_DIR: claudeConfig,
HOME: root,
snapshots/acp/escalation-approved/cordis.yml
+1 -1
View File
@@ -34,7 +34,7 @@
name: '@deepseek-ai/dsh-acp'
config:
provider: deepseek-official
model: deepseek-v4-pro
model: deepseek-v4-flash
- id: system-prompt
name: '@deepseek-ai/dsh-system-prompt'