/** * T4 — Error contract verification for the MCP HTTP server. * * Sends intentionally invalid calls to the live server at :3917 and captures * the structured response (code + message, or tool payload isError). Each case * is logged with a verdict: PASS (expectation matched), WARN (acceptable but * different shape), or FAIL (wrong behaviour / bug). * * Run: * bun packages/mcp/test-reports/t4-errors/run.ts */ import { Client } from '@modelcontextprotocol/sdk/client/index.js' import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js' import { McpError } from '@modelcontextprotocol/sdk/types.js' const SERVER_URL = new URL('http://localhost:3917/') type Verdict = 'PASS' | 'WARN' | 'FAIL' type CaseResult = { id: string tool: string description: string input: unknown expected: string actual: { kind: 'mcp_error' | 'tool_error' | 'unexpected_success' | 'client_error' code?: number message?: string data?: unknown structuredContent?: unknown rawContent?: unknown } verdict: Verdict note?: string } const results: CaseResult[] = [] let client: Client | null = null /** * Call a tool and normalise the outcome into one of three shapes: * - mcp_error: server threw McpError (protocol-level JSON-RPC error). * - tool_error: tool returned `{ isError: true, content: [...] }`. * - unexpected_success: tool returned a normal payload. * Anything else (timeout, transport crash) becomes `client_error`. */ async function callTool( name: string, args: Record, ): Promise { try { const result = await client!.callTool({ name, arguments: args }) if (result.isError) { const rawContent = (result.content ?? []) as Array<{ type: string; text?: string }> const textBlock = rawContent.find((b) => b.type === 'text') return { kind: 'tool_error', message: textBlock?.text ?? JSON.stringify(rawContent), rawContent: result.content, structuredContent: result.structuredContent, } } return { kind: 'unexpected_success', message: 'tool returned successfully with no isError flag', structuredContent: result.structuredContent, rawContent: result.content, } } catch (err) { if (err instanceof McpError) { return { kind: 'mcp_error', code: err.code, message: err.message, data: err.data, } } return { kind: 'client_error', message: err instanceof Error ? err.message : String(err), } } } function record( id: string, tool: string, description: string, input: unknown, expected: string, actual: CaseResult['actual'], verdict: Verdict, note?: string, ): void { results.push({ id, tool, description, input, expected, actual, verdict, note }) const icon = verdict === 'PASS' ? '[PASS]' : verdict === 'WARN' ? '[WARN]' : '[FAIL]' const line = `${icon} ${id} (${tool}): ${description}` console.log(line) if (actual.kind === 'mcp_error') { console.log(` -> McpError code=${actual.code} msg="${actual.message}"`) } else if (actual.kind === 'tool_error') { console.log(` -> tool_error msg="${actual.message}"`) } else if (actual.kind === 'unexpected_success') { console.log(` -> SUCCESS payload=${JSON.stringify(actual.structuredContent)}`) } else { console.log(` -> client_error msg="${actual.message}"`) } if (note) console.log(` note: ${note}`) } /** Verify actual matches expectation of "any structured error surface". */ function classifyRejection( actual: CaseResult['actual'], allowMcp = true, allowTool = true, ): Verdict { if (actual.kind === 'mcp_error' && allowMcp) return 'PASS' if (actual.kind === 'tool_error' && allowTool) return 'PASS' if (actual.kind === 'unexpected_success') return 'FAIL' if (actual.kind === 'client_error') return 'FAIL' return 'WARN' } async function main(): Promise { // --- Connect --- const transport = new StreamableHTTPClientTransport(SERVER_URL) client = new Client({ name: 't4-error-tester', version: '0.0.0' }) await client.connect(transport) console.log(`connected to ${SERVER_URL.href}`) // --- Baseline: scene snapshot --- const scene0 = await client.callTool({ name: 'get_scene', arguments: {} }) const nodes0 = (scene0.structuredContent as { nodes?: Record })?.nodes ?? {} const nodeCount0 = Object.keys(nodes0).length console.log(`baseline node count = ${nodeCount0}`) // Discover real IDs for positive-side assertions (e.g. real wall, real level). const scenePayload = scene0.structuredContent as { nodes: Record } const allNodes = Object.values(scenePayload.nodes ?? {}) as Array<{ id: string type: string children?: string[] }> const findFirst = (t: string) => allNodes.find((n) => n.type === t) const realSite = findFirst('site') const realBuilding = findFirst('building') const realLevel = findFirst('level') const realWall = findFirst('wall') // Try to locate a node with children (for delete_node cascade=false case). let nodeWithChildren = allNodes.find((n) => (n.children?.length ?? 0) > 0) // Fallback to site/building/level if they have children. if (!nodeWithChildren) nodeWithChildren = realSite ?? realBuilding ?? realLevel console.log( `discovered: site=${realSite?.id} building=${realBuilding?.id} level=${realLevel?.id} wall=${realWall?.id}`, ) console.log( `node-with-children=${nodeWithChildren?.id} type=${nodeWithChildren?.type} children=${nodeWithChildren?.children?.length ?? 0}`, ) // ========================================================================== // TESTS // ========================================================================== // 1. get_node — nonexistent id { const input = { id: 'node_doesnotexist_xyz' } const actual = await callTool('get_node', input) record( 'T4-01', 'get_node', 'nonexistent id', input, 'McpError InvalidParams (-32602) "Node not found" OR structured tool error', actual, classifyRejection(actual), ) } // 2. describe_node — nonexistent id { const input = { id: 'node_missing_123' } const actual = await callTool('describe_node', input) record( 'T4-02', 'describe_node', 'nonexistent id', input, 'McpError InvalidParams (-32602) "Node not found"', actual, classifyRejection(actual), ) } // 3. find_nodes — invalid type enum { const input = { type: 'hamster' } const actual = await callTool('find_nodes', input) // Zod validation should fail before handler runs → MCP error. const isZod = actual.kind === 'mcp_error' && (actual.message?.toLowerCase().includes('invalid') || actual.message?.toLowerCase().includes('enum') || actual.message?.toLowerCase().includes('hamster')) record( 'T4-03', 'find_nodes', 'invalid type enum "hamster"', input, 'Zod validation error (MCP InvalidParams -32602)', actual, isZod ? 'PASS' : classifyRejection(actual), ) } // 4. measure — nonexistent fromId { const input = { fromId: 'node_nosuch_f', toId: realSite?.id ?? 'x' } const actual = await callTool('measure', input) record( 'T4-04', 'measure', 'nonexistent fromId', input, 'McpError InvalidParams "Node not found"', actual, classifyRejection(actual), ) } // 5. apply_patch — patch with invalid node (missing type field) // The schema accepts `node: z.record(z.string(), z.unknown())` so missing // `type` slips past Zod; the bridge's core validator catches it and // apply-patch converts that into an McpError via its try/catch. { const input = { patches: [ { op: 'create', node: { foo: 'bar' /* no type */ }, parentId: realLevel?.id ?? 'missing', }, ], } const actual = await callTool('apply_patch', input) record( 'T4-05', 'apply_patch', 'patches with one invalid node (missing type)', input, 'McpError InvalidParams, all-or-nothing rollback (no partial state change)', actual, classifyRejection(actual), ) } // 6. apply_patch — delete nonexistent id { const input = { patches: [{ op: 'delete', id: 'node_nonexistent_delete_xyz' }], } const actual = await callTool('apply_patch', input) record( 'T4-06', 'apply_patch', 'delete nonexistent id', input, 'McpError InvalidParams, no state change', actual, classifyRejection(actual), ) } // 7. create_level — buildingId that isn't a building (feed it a site/wall/level) { const notBuildingId = realWall?.id ?? realLevel?.id ?? realSite?.id ?? 'missing' const input = { buildingId: notBuildingId, elevation: 0 } const actual = await callTool('create_level', input) record( 'T4-07', 'create_level', 'buildingId is not a building (passed a wall/level/site id)', input, 'McpError InvalidParams "expected building"', actual, classifyRejection(actual), ) } // 8. create_wall — levelId that doesn't exist { const input = { levelId: 'level_nosuch_999', start: [0, 0], end: [5, 0], } const actual = await callTool('create_wall', input) record( 'T4-08', 'create_wall', "levelId doesn't exist", input, 'McpError InvalidParams "Level not found"', actual, classifyRejection(actual), ) } // 9. create_wall — start/end not tuples { const input = { levelId: realLevel?.id ?? 'x', start: 'not-a-tuple', end: [5, 0], } const actual = await callTool('create_wall', input) record( 'T4-09', 'create_wall', 'start not a tuple', input, 'Zod validation error (MCP InvalidParams -32602)', actual, classifyRejection(actual), ) } // 10. place_item — targetNodeId doesn't exist { const input = { catalogItemId: 'chair-1', targetNodeId: 'node_nosuch_target', position: [0, 0, 0], } const actual = await callTool('place_item', input) record( 'T4-10', 'place_item', "targetNodeId doesn't exist", input, 'McpError InvalidParams "Target node not found"', actual, classifyRejection(actual), ) } // 11. cut_opening — wallId isn't a wall (pass a site/building/level) { const notWallId = realSite?.id ?? realBuilding?.id ?? realLevel?.id ?? 'missing' const input = { wallId: notWallId, type: 'door', position: 0.5, width: 0.8, height: 2.0, } const actual = await callTool('cut_opening', input) record( 'T4-11', 'cut_opening', 'wallId is not a wall', input, 'McpError InvalidParams "expected wall"', actual, classifyRejection(actual), ) } // 12. cut_opening — position out of [0,1] { const input = { wallId: realWall?.id ?? 'missing_wall', type: 'door', position: 2.5, width: 0.8, height: 2.0, } const actual = await callTool('cut_opening', input) record( 'T4-12', 'cut_opening', 'position out of [0,1]', input, 'Zod validation error (MCP InvalidParams) — position must be <= 1', actual, classifyRejection(actual), ) } // 13. set_zone — polygon with < 3 points { const input = { levelId: realLevel?.id ?? 'missing', polygon: [ [0, 0], [5, 0], ], label: 'Tiny', } const actual = await callTool('set_zone', input) record( 'T4-13', 'set_zone', 'polygon with < 3 points', input, 'Zod validation error (MCP InvalidParams) — polygon must have >= 3 points', actual, classifyRejection(actual), ) } // 14. duplicate_level — levelId isn't a level (pass a wall/site/building) { const notLevelId = realWall?.id ?? realSite?.id ?? realBuilding?.id ?? 'missing' const input = { levelId: notLevelId } const actual = await callTool('duplicate_level', input) record( 'T4-14', 'duplicate_level', 'levelId is not a level', input, 'McpError InvalidParams "expected level"', actual, classifyRejection(actual), ) } // 15. delete_node — id with children, cascade=false (happy-path for the rejection) // NOTE: this one mutates state if it succeeds. Since we expect it to REJECT // when cascade=false, there should be no state change. We'll verify below // via validate_scene. { const target = nodeWithChildren if (!target) { record( 'T4-15', 'delete_node', 'cascade=false with children', { id: 'no-candidate-found' }, 'McpError "node has children"', { kind: 'client_error', message: 'no node with children available in scene' }, 'WARN', 'skipped: no node with children in the default scene', ) } else { const input = { id: target.id, cascade: false } const actual = await callTool('delete_node', input) record( 'T4-15', 'delete_node', `cascade=false with children (target ${target.type} ${target.id} children=${target.children?.length ?? 0})`, input, 'McpError InvalidRequest "node has children" (no delete)', actual, classifyRejection(actual), ) } } // 16. undo — negative steps { const input = { steps: -1 } const actual = await callTool('undo', input) record( 'T4-16a', 'undo', 'negative steps', input, 'Zod validation error (MCP InvalidParams) — steps must be positive int', actual, classifyRejection(actual), ) } // 16b. redo — negative steps { const input = { steps: -2 } const actual = await callTool('redo', input) record( 'T4-16b', 'redo', 'negative steps', input, 'Zod validation error (MCP InvalidParams) — steps must be positive int', actual, classifyRejection(actual), ) } // 17. export_json — prettify is string 'yes' instead of bool { const input = { pretty: 'yes' } const actual = await callTool('export_json', input) record( 'T4-17', 'export_json', "pretty='yes' (string not bool)", input, 'Zod validation error (MCP InvalidParams) — pretty must be boolean', actual, classifyRejection(actual), ) } // 18. check_collisions — levelId doesn't exist (spec: empty result OR graceful error) { const input = { levelId: 'level_nosuch_zzz' } const actual = await callTool('check_collisions', input) // Spec allows either. Success with empty collisions is the friendly path. let verdict: Verdict = 'WARN' let note: string | undefined if (actual.kind === 'unexpected_success') { const sc = actual.structuredContent as { collisions?: unknown[] } | undefined const empty = Array.isArray(sc?.collisions) && sc.collisions.length === 0 verdict = empty ? 'PASS' : 'WARN' note = empty ? 'returned empty collisions (graceful)' : 'returned non-empty result' } else if (actual.kind === 'mcp_error' || actual.kind === 'tool_error') { verdict = 'PASS' note = 'structured error (also acceptable per spec)' } else { verdict = 'FAIL' } record( 'T4-18', 'check_collisions', "levelId doesn't exist", input, 'Empty collisions result OR graceful error', actual, verdict, note, ) } // 19. validate_scene — no args → should succeed (baseline, not an error test) { const input = {} const actual = await callTool('validate_scene', input) const ok = actual.kind === 'unexpected_success' record( 'T4-19', 'validate_scene', 'baseline: no args', input, 'Success — structured { valid, errors[] }', actual, ok ? 'PASS' : 'FAIL', ok ? 'baseline passed' : 'baseline failed', ) } // 20. analyze_floorplan_image — image: '' { const input = { image: '' } const actual = await callTool('analyze_floorplan_image', input) // Expected: Zod string.min rule (we have no min, so it accepts empty), // falling through to sampling_unavailable (no client caps on HTTP). // Either is acceptable — this is a validation/sampling-guard test. let verdict = classifyRejection(actual) let note: string | undefined if (actual.kind === 'mcp_error') { if (actual.message?.includes('sampling_unavailable')) { note = 'sampling_unavailable (no client capabilities) — acceptable' verdict = 'PASS' } else { note = 'structured error' } } record( 'T4-20a', 'analyze_floorplan_image', "image: '' (empty string)", input, 'Validation error OR sampling_unavailable', actual, verdict, note, ) } // 20b. analyze_floorplan_image — image: 'not-a-url-or-base64' { const input = { image: 'not-a-url-or-base64' } const actual = await callTool('analyze_floorplan_image', input) let verdict = classifyRejection(actual) let note: string | undefined if (actual.kind === 'mcp_error' && actual.message?.includes('sampling_unavailable')) { note = 'sampling_unavailable (no client capabilities)' verdict = 'PASS' } record( 'T4-20b', 'analyze_floorplan_image', "image: 'not-a-url-or-base64'", input, 'Validation error OR sampling_unavailable', actual, verdict, note, ) } // 21. analyze_room_photo — image: '' { const input = { image: '' } const actual = await callTool('analyze_room_photo', input) let verdict = classifyRejection(actual) let note: string | undefined if (actual.kind === 'mcp_error' && actual.message?.includes('sampling_unavailable')) { note = 'sampling_unavailable (no client capabilities)' verdict = 'PASS' } record( 'T4-21a', 'analyze_room_photo', "image: '' (empty string)", input, 'Validation error OR sampling_unavailable', actual, verdict, note, ) } // 21b. analyze_room_photo — image: 'not-a-url-or-base64' { const input = { image: 'not-a-url-or-base64' } const actual = await callTool('analyze_room_photo', input) let verdict = classifyRejection(actual) let note: string | undefined if (actual.kind === 'mcp_error' && actual.message?.includes('sampling_unavailable')) { note = 'sampling_unavailable (no client capabilities)' verdict = 'PASS' } record( 'T4-21b', 'analyze_room_photo', "image: 'not-a-url-or-base64'", input, 'Validation error OR sampling_unavailable', actual, verdict, note, ) } // --- Post-check: scene count should be unchanged (all error paths) --- const scene1 = await client.callTool({ name: 'get_scene', arguments: {} }) const nodes1 = (scene1.structuredContent as { nodes?: Record })?.nodes ?? {} const nodeCount1 = Object.keys(nodes1).length console.log(`\nfinal node count = ${nodeCount1} (baseline ${nodeCount0})`) const delta = nodeCount1 - nodeCount0 if (delta !== 0) { console.log(`WARN: node count changed by ${delta}`) } const validationFinal = await client.callTool({ name: 'validate_scene', arguments: {}, }) const vf = validationFinal.structuredContent as { valid: boolean; errors: unknown[] } console.log(`final validation: valid=${vf.valid} errors=${vf.errors.length}`) // --- Emit report --- await writeReport(nodeCount0, nodeCount1, vf) await client.close() } async function writeReport( nodeCountBefore: number, nodeCountAfter: number, finalValidation: { valid: boolean; errors: unknown[] }, ): Promise { const pass = results.filter((r) => r.verdict === 'PASS').length const warn = results.filter((r) => r.verdict === 'WARN').length const fail = results.filter((r) => r.verdict === 'FAIL').length const lines: string[] = [] lines.push('# T4 — Error Contract Verification Report') lines.push('') lines.push(`Server: \`${SERVER_URL.href}\``) lines.push(`Run date: ${new Date().toISOString()}`) lines.push('') lines.push('## Summary') lines.push('') lines.push(`- PASS: ${pass}`) lines.push(`- WARN: ${warn}`) lines.push(`- FAIL: ${fail}`) lines.push(`- Total cases: ${results.length}`) lines.push('') lines.push(`Baseline node count: ${nodeCountBefore}`) lines.push(`Final node count: ${nodeCountAfter} (delta=${nodeCountAfter - nodeCountBefore})`) lines.push( `Final validation: valid=${finalValidation.valid}, errors=${finalValidation.errors.length}`, ) lines.push('') if (fail > 0) { lines.push('## Failures (real bugs)') lines.push('') for (const r of results.filter((x) => x.verdict === 'FAIL')) { lines.push(`- **${r.id}** \`${r.tool}\`: ${r.description}`) lines.push(` - Expected: ${r.expected}`) lines.push(` - Actual kind: \`${r.actual.kind}\``) if (r.actual.code !== undefined) lines.push(` - Code: \`${r.actual.code}\``) if (r.actual.message) lines.push(` - Message: \`${r.actual.message}\``) } lines.push('') } lines.push('## Cases') lines.push('') for (const r of results) { const icon = r.verdict === 'PASS' ? '✅' : r.verdict === 'WARN' ? '⚠️' : '❌' lines.push(`### ${r.id} — \`${r.tool}\` — ${r.description}`) lines.push('') lines.push(`**Verdict:** ${icon} ${r.verdict}`) lines.push('') lines.push('**Input:**') lines.push('```json') lines.push(JSON.stringify(r.input, null, 2)) lines.push('```') lines.push('') lines.push(`**Expected:** ${r.expected}`) lines.push('') lines.push('**Actual:**') lines.push('```json') lines.push(JSON.stringify(r.actual, null, 2)) lines.push('```') if (r.note) { lines.push('') lines.push(`**Note:** ${r.note}`) } lines.push('') } const { writeFile } = await import('node:fs/promises') const reportPath = new URL('./REPORT.md', import.meta.url) await writeFile(reportPath, lines.join('\n'), 'utf8') console.log(`\nwrote report: ${reportPath.pathname}`) console.log(`summary: PASS=${pass} WARN=${warn} FAIL=${fail}`) } main().catch((err) => { console.error('[t4] fatal:', err) process.exit(1) })