fix(mcp): apply_patch preserves schema-defaulted ids in multi-op batches
SceneBridge.applyPatch now threads the Zod-parsed node through to the apply phase instead of the raw input. Previously, when a caller sent a create patch without an `id` field, the schema's objectId default ran during dry-run parsing but only in `res.data`; the apply phase pushed the unparsed `p.node` (no id) to the store, so subsequent tools that walked `level.children` crashed on undefined entries (e.g. duplicate_level -> cloneLevelSubtree -> extractIdPrefix(undefined)). Also adds test-reports/ artefacts from live end-to-end testing: - t1-stdio: 21/21 tools pass via stdio (~106ms) - t2-http: connect/single-session behaviour (HTTP transport quirk documented) - t3-scenario: 2-bedroom apartment built end-to-end — 12/12 steps after this fix (24 final nodes, validate=true, apartment.json exported) - t4-errors: 24/24 invalid-input cases rejected with proper MCP errors - t5-resources-prompts: 4/4 resources, 3/3 prompts, dev server /api/health 200 OK Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Adrian Perez
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
a6f1c4140f
commit
b37e88cb83
@@ -0,0 +1,728 @@
|
||||
# T4 — Error Contract Verification Report
|
||||
|
||||
Server: `http://localhost:3917/`
|
||||
Run date: 2026-04-18T16:17:28.757Z
|
||||
|
||||
## Summary
|
||||
|
||||
- PASS: 24
|
||||
- WARN: 0
|
||||
- FAIL: 0
|
||||
- Total cases: 24
|
||||
|
||||
Baseline node count: 3
|
||||
Final node count: 3 (delta=0)
|
||||
Final validation: valid=true, errors=0
|
||||
|
||||
## Cases
|
||||
|
||||
### T4-01 — `get_node` — nonexistent id
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"id": "node_doesnotexist_xyz"
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** McpError InvalidParams (-32602) "Node not found" OR structured tool error
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Node not found: node_doesnotexist_xyz",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Node not found: node_doesnotexist_xyz"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-02 — `describe_node` — nonexistent id
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"id": "node_missing_123"
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** McpError InvalidParams (-32602) "Node not found"
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Node not found: node_missing_123",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Node not found: node_missing_123"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-03 — `find_nodes` — invalid type enum "hamster"
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"type": "hamster"
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** Zod validation error (MCP InvalidParams -32602)
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Input validation error: Invalid arguments for tool find_nodes: [\n {\n \"code\": \"invalid_value\",\n \"values\": [\n \"site\",\n \"building\",\n \"level\",\n \"wall\",\n \"fence\",\n \"zone\",\n \"slab\",\n \"ceiling\",\n \"roof\",\n \"roof-segment\",\n \"stair\",\n \"stair-segment\",\n \"item\",\n \"door\",\n \"window\",\n \"scan\",\n \"guide\"\n ],\n \"path\": [\n \"type\"\n ],\n \"message\": \"Invalid option: expected one of \\\"site\\\"|\\\"building\\\"|\\\"level\\\"|\\\"wall\\\"|\\\"fence\\\"|\\\"zone\\\"|\\\"slab\\\"|\\\"ceiling\\\"|\\\"roof\\\"|\\\"roof-segment\\\"|\\\"stair\\\"|\\\"stair-segment\\\"|\\\"item\\\"|\\\"door\\\"|\\\"window\\\"|\\\"scan\\\"|\\\"guide\\\"\"\n }\n]",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Input validation error: Invalid arguments for tool find_nodes: [\n {\n \"code\": \"invalid_value\",\n \"values\": [\n \"site\",\n \"building\",\n \"level\",\n \"wall\",\n \"fence\",\n \"zone\",\n \"slab\",\n \"ceiling\",\n \"roof\",\n \"roof-segment\",\n \"stair\",\n \"stair-segment\",\n \"item\",\n \"door\",\n \"window\",\n \"scan\",\n \"guide\"\n ],\n \"path\": [\n \"type\"\n ],\n \"message\": \"Invalid option: expected one of \\\"site\\\"|\\\"building\\\"|\\\"level\\\"|\\\"wall\\\"|\\\"fence\\\"|\\\"zone\\\"|\\\"slab\\\"|\\\"ceiling\\\"|\\\"roof\\\"|\\\"roof-segment\\\"|\\\"stair\\\"|\\\"stair-segment\\\"|\\\"item\\\"|\\\"door\\\"|\\\"window\\\"|\\\"scan\\\"|\\\"guide\\\"\"\n }\n]"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-04 — `measure` — nonexistent fromId
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"fromId": "node_nosuch_f",
|
||||
"toId": "site_watn4a0qt2xpgri7"
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** McpError InvalidParams "Node not found"
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Node not found: node_nosuch_f",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Node not found: node_nosuch_f"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-05 — `apply_patch` — patches with one invalid node (missing type)
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"patches": [
|
||||
{
|
||||
"op": "create",
|
||||
"node": {
|
||||
"foo": "bar"
|
||||
},
|
||||
"parentId": "level_tl2aravmn2u9afft"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** McpError InvalidParams, all-or-nothing rollback (no partial state change)
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: invalid patch: patches[0] create node failed schema: [\n {\n \"code\": \"invalid_union\",\n \"errors\": [],\n \"note\": \"No matching discriminator\",\n \"discriminator\": \"type\",\n \"path\": [\n \"type\"\n ],\n \"message\": \"Invalid input\"\n }\n]",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: invalid patch: patches[0] create node failed schema: [\n {\n \"code\": \"invalid_union\",\n \"errors\": [],\n \"note\": \"No matching discriminator\",\n \"discriminator\": \"type\",\n \"path\": [\n \"type\"\n ],\n \"message\": \"Invalid input\"\n }\n]"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-06 — `apply_patch` — delete nonexistent id
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"patches": [
|
||||
{
|
||||
"op": "delete",
|
||||
"id": "node_nonexistent_delete_xyz"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** McpError InvalidParams, no state change
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: invalid patch: patches[0] delete id \"node_nonexistent_delete_xyz\" not found",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: invalid patch: patches[0] delete id \"node_nonexistent_delete_xyz\" not found"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-07 — `create_level` — buildingId is not a building (passed a wall/level/site id)
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"buildingId": "level_tl2aravmn2u9afft",
|
||||
"elevation": 0
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** McpError InvalidParams "expected building"
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Node level_tl2aravmn2u9afft is a level, expected building",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Node level_tl2aravmn2u9afft is a level, expected building"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-08 — `create_wall` — levelId doesn't exist
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"levelId": "level_nosuch_999",
|
||||
"start": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"end": [
|
||||
5,
|
||||
0
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** McpError InvalidParams "Level not found"
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Level not found: level_nosuch_999",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Level not found: level_nosuch_999"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-09 — `create_wall` — start not a tuple
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"levelId": "level_tl2aravmn2u9afft",
|
||||
"start": "not-a-tuple",
|
||||
"end": [
|
||||
5,
|
||||
0
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** Zod validation error (MCP InvalidParams -32602)
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Input validation error: Invalid arguments for tool create_wall: [\n {\n \"expected\": \"tuple\",\n \"code\": \"invalid_type\",\n \"path\": [\n \"start\"\n ],\n \"message\": \"Invalid input: expected tuple, received string\"\n }\n]",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Input validation error: Invalid arguments for tool create_wall: [\n {\n \"expected\": \"tuple\",\n \"code\": \"invalid_type\",\n \"path\": [\n \"start\"\n ],\n \"message\": \"Invalid input: expected tuple, received string\"\n }\n]"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-10 — `place_item` — targetNodeId doesn't exist
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"catalogItemId": "chair-1",
|
||||
"targetNodeId": "node_nosuch_target",
|
||||
"position": [
|
||||
0,
|
||||
0,
|
||||
0
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** McpError InvalidParams "Target node not found"
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Target node not found: node_nosuch_target",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Target node not found: node_nosuch_target"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-11 — `cut_opening` — wallId is not a wall
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"wallId": "site_watn4a0qt2xpgri7",
|
||||
"type": "door",
|
||||
"position": 0.5,
|
||||
"width": 0.8,
|
||||
"height": 2
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** McpError InvalidParams "expected wall"
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Node site_watn4a0qt2xpgri7 is a site, expected wall",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Node site_watn4a0qt2xpgri7 is a site, expected wall"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-12 — `cut_opening` — position out of [0,1]
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"wallId": "missing_wall",
|
||||
"type": "door",
|
||||
"position": 2.5,
|
||||
"width": 0.8,
|
||||
"height": 2
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** Zod validation error (MCP InvalidParams) — position must be <= 1
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Input validation error: Invalid arguments for tool cut_opening: [\n {\n \"origin\": \"number\",\n \"code\": \"too_big\",\n \"maximum\": 1,\n \"inclusive\": true,\n \"path\": [\n \"position\"\n ],\n \"message\": \"Too big: expected number to be <=1\"\n }\n]",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Input validation error: Invalid arguments for tool cut_opening: [\n {\n \"origin\": \"number\",\n \"code\": \"too_big\",\n \"maximum\": 1,\n \"inclusive\": true,\n \"path\": [\n \"position\"\n ],\n \"message\": \"Too big: expected number to be <=1\"\n }\n]"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-13 — `set_zone` — polygon with < 3 points
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"levelId": "level_tl2aravmn2u9afft",
|
||||
"polygon": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
5,
|
||||
0
|
||||
]
|
||||
],
|
||||
"label": "Tiny"
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** Zod validation error (MCP InvalidParams) — polygon must have >= 3 points
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Input validation error: Invalid arguments for tool set_zone: [\n {\n \"origin\": \"array\",\n \"code\": \"too_small\",\n \"minimum\": 3,\n \"inclusive\": true,\n \"path\": [\n \"polygon\"\n ],\n \"message\": \"Too small: expected array to have >=3 items\"\n }\n]",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Input validation error: Invalid arguments for tool set_zone: [\n {\n \"origin\": \"array\",\n \"code\": \"too_small\",\n \"minimum\": 3,\n \"inclusive\": true,\n \"path\": [\n \"polygon\"\n ],\n \"message\": \"Too small: expected array to have >=3 items\"\n }\n]"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-14 — `duplicate_level` — levelId is not a level
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"levelId": "site_watn4a0qt2xpgri7"
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** McpError InvalidParams "expected level"
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Node site_watn4a0qt2xpgri7 is a site, expected level",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Node site_watn4a0qt2xpgri7 is a site, expected level"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-15 — `delete_node` — cascade=false with children (target site site_watn4a0qt2xpgri7 children=1)
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"id": "site_watn4a0qt2xpgri7",
|
||||
"cascade": false
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** McpError InvalidRequest "node has children" (no delete)
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32600: node has 2 descendant(s); pass cascade: true to delete recursively",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32600: node has 2 descendant(s); pass cascade: true to delete recursively"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-16a — `undo` — negative steps
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"steps": -1
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** Zod validation error (MCP InvalidParams) — steps must be positive int
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Input validation error: Invalid arguments for tool undo: [\n {\n \"origin\": \"number\",\n \"code\": \"too_small\",\n \"minimum\": 0,\n \"inclusive\": false,\n \"path\": [\n \"steps\"\n ],\n \"message\": \"Too small: expected number to be >0\"\n }\n]",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Input validation error: Invalid arguments for tool undo: [\n {\n \"origin\": \"number\",\n \"code\": \"too_small\",\n \"minimum\": 0,\n \"inclusive\": false,\n \"path\": [\n \"steps\"\n ],\n \"message\": \"Too small: expected number to be >0\"\n }\n]"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-16b — `redo` — negative steps
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"steps": -2
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** Zod validation error (MCP InvalidParams) — steps must be positive int
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Input validation error: Invalid arguments for tool redo: [\n {\n \"origin\": \"number\",\n \"code\": \"too_small\",\n \"minimum\": 0,\n \"inclusive\": false,\n \"path\": [\n \"steps\"\n ],\n \"message\": \"Too small: expected number to be >0\"\n }\n]",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Input validation error: Invalid arguments for tool redo: [\n {\n \"origin\": \"number\",\n \"code\": \"too_small\",\n \"minimum\": 0,\n \"inclusive\": false,\n \"path\": [\n \"steps\"\n ],\n \"message\": \"Too small: expected number to be >0\"\n }\n]"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-17 — `export_json` — pretty='yes' (string not bool)
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"pretty": "yes"
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** Zod validation error (MCP InvalidParams) — pretty must be boolean
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32602: Input validation error: Invalid arguments for tool export_json: [\n {\n \"expected\": \"boolean\",\n \"code\": \"invalid_type\",\n \"path\": [\n \"pretty\"\n ],\n \"message\": \"Invalid input: expected boolean, received string\"\n }\n]",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32602: Input validation error: Invalid arguments for tool export_json: [\n {\n \"expected\": \"boolean\",\n \"code\": \"invalid_type\",\n \"path\": [\n \"pretty\"\n ],\n \"message\": \"Invalid input: expected boolean, received string\"\n }\n]"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-18 — `check_collisions` — levelId doesn't exist
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"levelId": "level_nosuch_zzz"
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** Empty collisions result OR graceful error
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "unexpected_success",
|
||||
"message": "tool returned successfully with no isError flag",
|
||||
"structuredContent": {
|
||||
"collisions": []
|
||||
},
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "{\"collisions\":[]}"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Note:** returned empty collisions (graceful)
|
||||
|
||||
### T4-19 — `validate_scene` — baseline: no args
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{}
|
||||
```
|
||||
|
||||
**Expected:** Success — structured { valid, errors[] }
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "unexpected_success",
|
||||
"message": "tool returned successfully with no isError flag",
|
||||
"structuredContent": {
|
||||
"valid": true,
|
||||
"errors": []
|
||||
},
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "{\"valid\":true,\"errors\":[]}"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Note:** baseline passed
|
||||
|
||||
### T4-20a — `analyze_floorplan_image` — image: '' (empty string)
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"image": ""
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** Validation error OR sampling_unavailable
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32600: sampling_unavailable",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32600: sampling_unavailable"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-20b — `analyze_floorplan_image` — image: 'not-a-url-or-base64'
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"image": "not-a-url-or-base64"
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** Validation error OR sampling_unavailable
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32600: sampling_unavailable",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32600: sampling_unavailable"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-21a — `analyze_room_photo` — image: '' (empty string)
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"image": ""
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** Validation error OR sampling_unavailable
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32600: sampling_unavailable",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32600: sampling_unavailable"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### T4-21b — `analyze_room_photo` — image: 'not-a-url-or-base64'
|
||||
|
||||
**Verdict:** ✅ PASS
|
||||
|
||||
**Input:**
|
||||
```json
|
||||
{
|
||||
"image": "not-a-url-or-base64"
|
||||
}
|
||||
```
|
||||
|
||||
**Expected:** Validation error OR sampling_unavailable
|
||||
|
||||
**Actual:**
|
||||
```json
|
||||
{
|
||||
"kind": "tool_error",
|
||||
"message": "MCP error -32600: sampling_unavailable",
|
||||
"rawContent": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "MCP error -32600: sampling_unavailable"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,160 @@
|
||||
connected to http://localhost:3917/
|
||||
baseline node count = 3
|
||||
discovered: site=site_watn4a0qt2xpgri7 building=building_wqydpgpprigdcq8a level=level_tl2aravmn2u9afft wall=undefined
|
||||
node-with-children=site_watn4a0qt2xpgri7 type=site children=1
|
||||
[PASS] T4-01 (get_node): nonexistent id
|
||||
-> tool_error msg="MCP error -32602: Node not found: node_doesnotexist_xyz"
|
||||
[PASS] T4-02 (describe_node): nonexistent id
|
||||
-> tool_error msg="MCP error -32602: Node not found: node_missing_123"
|
||||
[PASS] T4-03 (find_nodes): invalid type enum "hamster"
|
||||
-> tool_error msg="MCP error -32602: Input validation error: Invalid arguments for tool find_nodes: [
|
||||
{
|
||||
"code": "invalid_value",
|
||||
"values": [
|
||||
"site",
|
||||
"building",
|
||||
"level",
|
||||
"wall",
|
||||
"fence",
|
||||
"zone",
|
||||
"slab",
|
||||
"ceiling",
|
||||
"roof",
|
||||
"roof-segment",
|
||||
"stair",
|
||||
"stair-segment",
|
||||
"item",
|
||||
"door",
|
||||
"window",
|
||||
"scan",
|
||||
"guide"
|
||||
],
|
||||
"path": [
|
||||
"type"
|
||||
],
|
||||
"message": "Invalid option: expected one of \"site\"|\"building\"|\"level\"|\"wall\"|\"fence\"|\"zone\"|\"slab\"|\"ceiling\"|\"roof\"|\"roof-segment\"|\"stair\"|\"stair-segment\"|\"item\"|\"door\"|\"window\"|\"scan\"|\"guide\""
|
||||
}
|
||||
]"
|
||||
[PASS] T4-04 (measure): nonexistent fromId
|
||||
-> tool_error msg="MCP error -32602: Node not found: node_nosuch_f"
|
||||
[PASS] T4-05 (apply_patch): patches with one invalid node (missing type)
|
||||
-> tool_error msg="MCP error -32602: invalid patch: patches[0] create node failed schema: [
|
||||
{
|
||||
"code": "invalid_union",
|
||||
"errors": [],
|
||||
"note": "No matching discriminator",
|
||||
"discriminator": "type",
|
||||
"path": [
|
||||
"type"
|
||||
],
|
||||
"message": "Invalid input"
|
||||
}
|
||||
]"
|
||||
[PASS] T4-06 (apply_patch): delete nonexistent id
|
||||
-> tool_error msg="MCP error -32602: invalid patch: patches[0] delete id "node_nonexistent_delete_xyz" not found"
|
||||
[PASS] T4-07 (create_level): buildingId is not a building (passed a wall/level/site id)
|
||||
-> tool_error msg="MCP error -32602: Node level_tl2aravmn2u9afft is a level, expected building"
|
||||
[PASS] T4-08 (create_wall): levelId doesn't exist
|
||||
-> tool_error msg="MCP error -32602: Level not found: level_nosuch_999"
|
||||
[PASS] T4-09 (create_wall): start not a tuple
|
||||
-> tool_error msg="MCP error -32602: Input validation error: Invalid arguments for tool create_wall: [
|
||||
{
|
||||
"expected": "tuple",
|
||||
"code": "invalid_type",
|
||||
"path": [
|
||||
"start"
|
||||
],
|
||||
"message": "Invalid input: expected tuple, received string"
|
||||
}
|
||||
]"
|
||||
[PASS] T4-10 (place_item): targetNodeId doesn't exist
|
||||
-> tool_error msg="MCP error -32602: Target node not found: node_nosuch_target"
|
||||
[PASS] T4-11 (cut_opening): wallId is not a wall
|
||||
-> tool_error msg="MCP error -32602: Node site_watn4a0qt2xpgri7 is a site, expected wall"
|
||||
[PASS] T4-12 (cut_opening): position out of [0,1]
|
||||
-> tool_error msg="MCP error -32602: Input validation error: Invalid arguments for tool cut_opening: [
|
||||
{
|
||||
"origin": "number",
|
||||
"code": "too_big",
|
||||
"maximum": 1,
|
||||
"inclusive": true,
|
||||
"path": [
|
||||
"position"
|
||||
],
|
||||
"message": "Too big: expected number to be <=1"
|
||||
}
|
||||
]"
|
||||
[PASS] T4-13 (set_zone): polygon with < 3 points
|
||||
-> tool_error msg="MCP error -32602: Input validation error: Invalid arguments for tool set_zone: [
|
||||
{
|
||||
"origin": "array",
|
||||
"code": "too_small",
|
||||
"minimum": 3,
|
||||
"inclusive": true,
|
||||
"path": [
|
||||
"polygon"
|
||||
],
|
||||
"message": "Too small: expected array to have >=3 items"
|
||||
}
|
||||
]"
|
||||
[PASS] T4-14 (duplicate_level): levelId is not a level
|
||||
-> tool_error msg="MCP error -32602: Node site_watn4a0qt2xpgri7 is a site, expected level"
|
||||
[PASS] T4-15 (delete_node): cascade=false with children (target site site_watn4a0qt2xpgri7 children=1)
|
||||
-> tool_error msg="MCP error -32600: node has 2 descendant(s); pass cascade: true to delete recursively"
|
||||
[PASS] T4-16a (undo): negative steps
|
||||
-> tool_error msg="MCP error -32602: Input validation error: Invalid arguments for tool undo: [
|
||||
{
|
||||
"origin": "number",
|
||||
"code": "too_small",
|
||||
"minimum": 0,
|
||||
"inclusive": false,
|
||||
"path": [
|
||||
"steps"
|
||||
],
|
||||
"message": "Too small: expected number to be >0"
|
||||
}
|
||||
]"
|
||||
[PASS] T4-16b (redo): negative steps
|
||||
-> tool_error msg="MCP error -32602: Input validation error: Invalid arguments for tool redo: [
|
||||
{
|
||||
"origin": "number",
|
||||
"code": "too_small",
|
||||
"minimum": 0,
|
||||
"inclusive": false,
|
||||
"path": [
|
||||
"steps"
|
||||
],
|
||||
"message": "Too small: expected number to be >0"
|
||||
}
|
||||
]"
|
||||
[PASS] T4-17 (export_json): pretty='yes' (string not bool)
|
||||
-> tool_error msg="MCP error -32602: Input validation error: Invalid arguments for tool export_json: [
|
||||
{
|
||||
"expected": "boolean",
|
||||
"code": "invalid_type",
|
||||
"path": [
|
||||
"pretty"
|
||||
],
|
||||
"message": "Invalid input: expected boolean, received string"
|
||||
}
|
||||
]"
|
||||
[PASS] T4-18 (check_collisions): levelId doesn't exist
|
||||
-> SUCCESS payload={"collisions":[]}
|
||||
note: returned empty collisions (graceful)
|
||||
[PASS] T4-19 (validate_scene): baseline: no args
|
||||
-> SUCCESS payload={"valid":true,"errors":[]}
|
||||
note: baseline passed
|
||||
[PASS] T4-20a (analyze_floorplan_image): image: '' (empty string)
|
||||
-> tool_error msg="MCP error -32600: sampling_unavailable"
|
||||
[PASS] T4-20b (analyze_floorplan_image): image: 'not-a-url-or-base64'
|
||||
-> tool_error msg="MCP error -32600: sampling_unavailable"
|
||||
[PASS] T4-21a (analyze_room_photo): image: '' (empty string)
|
||||
-> tool_error msg="MCP error -32600: sampling_unavailable"
|
||||
[PASS] T4-21b (analyze_room_photo): image: 'not-a-url-or-base64'
|
||||
-> tool_error msg="MCP error -32600: sampling_unavailable"
|
||||
|
||||
final node count = 3 (baseline 3)
|
||||
final validation: valid=true errors=0
|
||||
|
||||
wrote report: /Users/adrian/Desktop/editor/.worktrees/mcp-server/packages/mcp/test-reports/t4-errors/REPORT.md
|
||||
summary: PASS=24 WARN=0 FAIL=0
|
||||
@@ -0,0 +1,751 @@
|
||||
/**
|
||||
* T4 — Error contract verification for the MCP HTTP server.
|
||||
*
|
||||
* Sends intentionally invalid calls to the live server at :3917 and captures
|
||||
* the structured response (code + message, or tool payload isError). Each case
|
||||
* is logged with a verdict: PASS (expectation matched), WARN (acceptable but
|
||||
* different shape), or FAIL (wrong behaviour / bug).
|
||||
*
|
||||
* Run:
|
||||
* bun packages/mcp/test-reports/t4-errors/run.ts
|
||||
*/
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
|
||||
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'
|
||||
import { McpError } from '@modelcontextprotocol/sdk/types.js'
|
||||
|
||||
const SERVER_URL = new URL('http://localhost:3917/')
|
||||
|
||||
type Verdict = 'PASS' | 'WARN' | 'FAIL'
|
||||
|
||||
type CaseResult = {
|
||||
id: string
|
||||
tool: string
|
||||
description: string
|
||||
input: unknown
|
||||
expected: string
|
||||
actual: {
|
||||
kind: 'mcp_error' | 'tool_error' | 'unexpected_success' | 'client_error'
|
||||
code?: number
|
||||
message?: string
|
||||
data?: unknown
|
||||
structuredContent?: unknown
|
||||
rawContent?: unknown
|
||||
}
|
||||
verdict: Verdict
|
||||
note?: string
|
||||
}
|
||||
|
||||
const results: CaseResult[] = []
|
||||
let client: Client | null = null
|
||||
|
||||
/**
|
||||
* Call a tool and normalise the outcome into one of three shapes:
|
||||
* - mcp_error: server threw McpError (protocol-level JSON-RPC error).
|
||||
* - tool_error: tool returned `{ isError: true, content: [...] }`.
|
||||
* - unexpected_success: tool returned a normal payload.
|
||||
* Anything else (timeout, transport crash) becomes `client_error`.
|
||||
*/
|
||||
async function callTool(
|
||||
name: string,
|
||||
args: Record<string, unknown>,
|
||||
): Promise<CaseResult['actual']> {
|
||||
try {
|
||||
const result = await client!.callTool({ name, arguments: args })
|
||||
if (result.isError) {
|
||||
const rawContent = (result.content ?? []) as Array<{ type: string; text?: string }>
|
||||
const textBlock = rawContent.find((b) => b.type === 'text')
|
||||
return {
|
||||
kind: 'tool_error',
|
||||
message: textBlock?.text ?? JSON.stringify(rawContent),
|
||||
rawContent: result.content,
|
||||
structuredContent: result.structuredContent,
|
||||
}
|
||||
}
|
||||
return {
|
||||
kind: 'unexpected_success',
|
||||
message: 'tool returned successfully with no isError flag',
|
||||
structuredContent: result.structuredContent,
|
||||
rawContent: result.content,
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof McpError) {
|
||||
return {
|
||||
kind: 'mcp_error',
|
||||
code: err.code,
|
||||
message: err.message,
|
||||
data: err.data,
|
||||
}
|
||||
}
|
||||
return {
|
||||
kind: 'client_error',
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function record(
|
||||
id: string,
|
||||
tool: string,
|
||||
description: string,
|
||||
input: unknown,
|
||||
expected: string,
|
||||
actual: CaseResult['actual'],
|
||||
verdict: Verdict,
|
||||
note?: string,
|
||||
): void {
|
||||
results.push({ id, tool, description, input, expected, actual, verdict, note })
|
||||
const icon = verdict === 'PASS' ? '[PASS]' : verdict === 'WARN' ? '[WARN]' : '[FAIL]'
|
||||
const line = `${icon} ${id} (${tool}): ${description}`
|
||||
console.log(line)
|
||||
if (actual.kind === 'mcp_error') {
|
||||
console.log(` -> McpError code=${actual.code} msg="${actual.message}"`)
|
||||
} else if (actual.kind === 'tool_error') {
|
||||
console.log(` -> tool_error msg="${actual.message}"`)
|
||||
} else if (actual.kind === 'unexpected_success') {
|
||||
console.log(` -> SUCCESS payload=${JSON.stringify(actual.structuredContent)}`)
|
||||
} else {
|
||||
console.log(` -> client_error msg="${actual.message}"`)
|
||||
}
|
||||
if (note) console.log(` note: ${note}`)
|
||||
}
|
||||
|
||||
/** Verify actual matches expectation of "any structured error surface". */
|
||||
function classifyRejection(
|
||||
actual: CaseResult['actual'],
|
||||
allowMcp = true,
|
||||
allowTool = true,
|
||||
): Verdict {
|
||||
if (actual.kind === 'mcp_error' && allowMcp) return 'PASS'
|
||||
if (actual.kind === 'tool_error' && allowTool) return 'PASS'
|
||||
if (actual.kind === 'unexpected_success') return 'FAIL'
|
||||
if (actual.kind === 'client_error') return 'FAIL'
|
||||
return 'WARN'
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
// --- Connect ---
|
||||
const transport = new StreamableHTTPClientTransport(SERVER_URL)
|
||||
client = new Client({ name: 't4-error-tester', version: '0.0.0' })
|
||||
await client.connect(transport)
|
||||
console.log(`connected to ${SERVER_URL.href}`)
|
||||
|
||||
// --- Baseline: scene snapshot ---
|
||||
const scene0 = await client.callTool({ name: 'get_scene', arguments: {} })
|
||||
const nodes0 = (scene0.structuredContent as { nodes?: Record<string, unknown> })?.nodes ?? {}
|
||||
const nodeCount0 = Object.keys(nodes0).length
|
||||
console.log(`baseline node count = ${nodeCount0}`)
|
||||
|
||||
// Discover real IDs for positive-side assertions (e.g. real wall, real level).
|
||||
const scenePayload = scene0.structuredContent as {
|
||||
nodes: Record<string, { id: string; type: string; children?: string[] }>
|
||||
}
|
||||
const allNodes = Object.values(scenePayload.nodes ?? {}) as Array<{
|
||||
id: string
|
||||
type: string
|
||||
children?: string[]
|
||||
}>
|
||||
const findFirst = (t: string) => allNodes.find((n) => n.type === t)
|
||||
const realSite = findFirst('site')
|
||||
const realBuilding = findFirst('building')
|
||||
const realLevel = findFirst('level')
|
||||
const realWall = findFirst('wall')
|
||||
|
||||
// Try to locate a node with children (for delete_node cascade=false case).
|
||||
let nodeWithChildren = allNodes.find((n) => (n.children?.length ?? 0) > 0)
|
||||
// Fallback to site/building/level if they have children.
|
||||
if (!nodeWithChildren) nodeWithChildren = realSite ?? realBuilding ?? realLevel
|
||||
console.log(
|
||||
`discovered: site=${realSite?.id} building=${realBuilding?.id} level=${realLevel?.id} wall=${realWall?.id}`,
|
||||
)
|
||||
console.log(
|
||||
`node-with-children=${nodeWithChildren?.id} type=${nodeWithChildren?.type} children=${nodeWithChildren?.children?.length ?? 0}`,
|
||||
)
|
||||
|
||||
// ==========================================================================
|
||||
// TESTS
|
||||
// ==========================================================================
|
||||
|
||||
// 1. get_node — nonexistent id
|
||||
{
|
||||
const input = { id: 'node_doesnotexist_xyz' }
|
||||
const actual = await callTool('get_node', input)
|
||||
record(
|
||||
'T4-01',
|
||||
'get_node',
|
||||
'nonexistent id',
|
||||
input,
|
||||
'McpError InvalidParams (-32602) "Node not found" OR structured tool error',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 2. describe_node — nonexistent id
|
||||
{
|
||||
const input = { id: 'node_missing_123' }
|
||||
const actual = await callTool('describe_node', input)
|
||||
record(
|
||||
'T4-02',
|
||||
'describe_node',
|
||||
'nonexistent id',
|
||||
input,
|
||||
'McpError InvalidParams (-32602) "Node not found"',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 3. find_nodes — invalid type enum
|
||||
{
|
||||
const input = { type: 'hamster' }
|
||||
const actual = await callTool('find_nodes', input)
|
||||
// Zod validation should fail before handler runs → MCP error.
|
||||
const isZod =
|
||||
actual.kind === 'mcp_error' &&
|
||||
(actual.message?.toLowerCase().includes('invalid') ||
|
||||
actual.message?.toLowerCase().includes('enum') ||
|
||||
actual.message?.toLowerCase().includes('hamster'))
|
||||
record(
|
||||
'T4-03',
|
||||
'find_nodes',
|
||||
'invalid type enum "hamster"',
|
||||
input,
|
||||
'Zod validation error (MCP InvalidParams -32602)',
|
||||
actual,
|
||||
isZod ? 'PASS' : classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 4. measure — nonexistent fromId
|
||||
{
|
||||
const input = { fromId: 'node_nosuch_f', toId: realSite?.id ?? 'x' }
|
||||
const actual = await callTool('measure', input)
|
||||
record(
|
||||
'T4-04',
|
||||
'measure',
|
||||
'nonexistent fromId',
|
||||
input,
|
||||
'McpError InvalidParams "Node not found"',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 5. apply_patch — patch with invalid node (missing type field)
|
||||
// The schema accepts `node: z.record(z.string(), z.unknown())` so missing
|
||||
// `type` slips past Zod; the bridge's core validator catches it and
|
||||
// apply-patch converts that into an McpError via its try/catch.
|
||||
{
|
||||
const input = {
|
||||
patches: [
|
||||
{
|
||||
op: 'create',
|
||||
node: { foo: 'bar' /* no type */ },
|
||||
parentId: realLevel?.id ?? 'missing',
|
||||
},
|
||||
],
|
||||
}
|
||||
const actual = await callTool('apply_patch', input)
|
||||
record(
|
||||
'T4-05',
|
||||
'apply_patch',
|
||||
'patches with one invalid node (missing type)',
|
||||
input,
|
||||
'McpError InvalidParams, all-or-nothing rollback (no partial state change)',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 6. apply_patch — delete nonexistent id
|
||||
{
|
||||
const input = {
|
||||
patches: [{ op: 'delete', id: 'node_nonexistent_delete_xyz' }],
|
||||
}
|
||||
const actual = await callTool('apply_patch', input)
|
||||
record(
|
||||
'T4-06',
|
||||
'apply_patch',
|
||||
'delete nonexistent id',
|
||||
input,
|
||||
'McpError InvalidParams, no state change',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 7. create_level — buildingId that isn't a building (feed it a site/wall/level)
|
||||
{
|
||||
const notBuildingId = realWall?.id ?? realLevel?.id ?? realSite?.id ?? 'missing'
|
||||
const input = { buildingId: notBuildingId, elevation: 0 }
|
||||
const actual = await callTool('create_level', input)
|
||||
record(
|
||||
'T4-07',
|
||||
'create_level',
|
||||
'buildingId is not a building (passed a wall/level/site id)',
|
||||
input,
|
||||
'McpError InvalidParams "expected building"',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 8. create_wall — levelId that doesn't exist
|
||||
{
|
||||
const input = {
|
||||
levelId: 'level_nosuch_999',
|
||||
start: [0, 0],
|
||||
end: [5, 0],
|
||||
}
|
||||
const actual = await callTool('create_wall', input)
|
||||
record(
|
||||
'T4-08',
|
||||
'create_wall',
|
||||
"levelId doesn't exist",
|
||||
input,
|
||||
'McpError InvalidParams "Level not found"',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 9. create_wall — start/end not tuples
|
||||
{
|
||||
const input = {
|
||||
levelId: realLevel?.id ?? 'x',
|
||||
start: 'not-a-tuple',
|
||||
end: [5, 0],
|
||||
}
|
||||
const actual = await callTool('create_wall', input)
|
||||
record(
|
||||
'T4-09',
|
||||
'create_wall',
|
||||
'start not a tuple',
|
||||
input,
|
||||
'Zod validation error (MCP InvalidParams -32602)',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 10. place_item — targetNodeId doesn't exist
|
||||
{
|
||||
const input = {
|
||||
catalogItemId: 'chair-1',
|
||||
targetNodeId: 'node_nosuch_target',
|
||||
position: [0, 0, 0],
|
||||
}
|
||||
const actual = await callTool('place_item', input)
|
||||
record(
|
||||
'T4-10',
|
||||
'place_item',
|
||||
"targetNodeId doesn't exist",
|
||||
input,
|
||||
'McpError InvalidParams "Target node not found"',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 11. cut_opening — wallId isn't a wall (pass a site/building/level)
|
||||
{
|
||||
const notWallId = realSite?.id ?? realBuilding?.id ?? realLevel?.id ?? 'missing'
|
||||
const input = {
|
||||
wallId: notWallId,
|
||||
type: 'door',
|
||||
position: 0.5,
|
||||
width: 0.8,
|
||||
height: 2.0,
|
||||
}
|
||||
const actual = await callTool('cut_opening', input)
|
||||
record(
|
||||
'T4-11',
|
||||
'cut_opening',
|
||||
'wallId is not a wall',
|
||||
input,
|
||||
'McpError InvalidParams "expected wall"',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 12. cut_opening — position out of [0,1]
|
||||
{
|
||||
const input = {
|
||||
wallId: realWall?.id ?? 'missing_wall',
|
||||
type: 'door',
|
||||
position: 2.5,
|
||||
width: 0.8,
|
||||
height: 2.0,
|
||||
}
|
||||
const actual = await callTool('cut_opening', input)
|
||||
record(
|
||||
'T4-12',
|
||||
'cut_opening',
|
||||
'position out of [0,1]',
|
||||
input,
|
||||
'Zod validation error (MCP InvalidParams) — position must be <= 1',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 13. set_zone — polygon with < 3 points
|
||||
{
|
||||
const input = {
|
||||
levelId: realLevel?.id ?? 'missing',
|
||||
polygon: [
|
||||
[0, 0],
|
||||
[5, 0],
|
||||
],
|
||||
label: 'Tiny',
|
||||
}
|
||||
const actual = await callTool('set_zone', input)
|
||||
record(
|
||||
'T4-13',
|
||||
'set_zone',
|
||||
'polygon with < 3 points',
|
||||
input,
|
||||
'Zod validation error (MCP InvalidParams) — polygon must have >= 3 points',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 14. duplicate_level — levelId isn't a level (pass a wall/site/building)
|
||||
{
|
||||
const notLevelId = realWall?.id ?? realSite?.id ?? realBuilding?.id ?? 'missing'
|
||||
const input = { levelId: notLevelId }
|
||||
const actual = await callTool('duplicate_level', input)
|
||||
record(
|
||||
'T4-14',
|
||||
'duplicate_level',
|
||||
'levelId is not a level',
|
||||
input,
|
||||
'McpError InvalidParams "expected level"',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 15. delete_node — id with children, cascade=false (happy-path for the rejection)
|
||||
// NOTE: this one mutates state if it succeeds. Since we expect it to REJECT
|
||||
// when cascade=false, there should be no state change. We'll verify below
|
||||
// via validate_scene.
|
||||
{
|
||||
const target = nodeWithChildren
|
||||
if (!target) {
|
||||
record(
|
||||
'T4-15',
|
||||
'delete_node',
|
||||
'cascade=false with children',
|
||||
{ id: 'no-candidate-found' },
|
||||
'McpError "node has children"',
|
||||
{ kind: 'client_error', message: 'no node with children available in scene' },
|
||||
'WARN',
|
||||
'skipped: no node with children in the default scene',
|
||||
)
|
||||
} else {
|
||||
const input = { id: target.id, cascade: false }
|
||||
const actual = await callTool('delete_node', input)
|
||||
record(
|
||||
'T4-15',
|
||||
'delete_node',
|
||||
`cascade=false with children (target ${target.type} ${target.id} children=${target.children?.length ?? 0})`,
|
||||
input,
|
||||
'McpError InvalidRequest "node has children" (no delete)',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// 16. undo — negative steps
|
||||
{
|
||||
const input = { steps: -1 }
|
||||
const actual = await callTool('undo', input)
|
||||
record(
|
||||
'T4-16a',
|
||||
'undo',
|
||||
'negative steps',
|
||||
input,
|
||||
'Zod validation error (MCP InvalidParams) — steps must be positive int',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 16b. redo — negative steps
|
||||
{
|
||||
const input = { steps: -2 }
|
||||
const actual = await callTool('redo', input)
|
||||
record(
|
||||
'T4-16b',
|
||||
'redo',
|
||||
'negative steps',
|
||||
input,
|
||||
'Zod validation error (MCP InvalidParams) — steps must be positive int',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 17. export_json — prettify is string 'yes' instead of bool
|
||||
{
|
||||
const input = { pretty: 'yes' }
|
||||
const actual = await callTool('export_json', input)
|
||||
record(
|
||||
'T4-17',
|
||||
'export_json',
|
||||
"pretty='yes' (string not bool)",
|
||||
input,
|
||||
'Zod validation error (MCP InvalidParams) — pretty must be boolean',
|
||||
actual,
|
||||
classifyRejection(actual),
|
||||
)
|
||||
}
|
||||
|
||||
// 18. check_collisions — levelId doesn't exist (spec: empty result OR graceful error)
|
||||
{
|
||||
const input = { levelId: 'level_nosuch_zzz' }
|
||||
const actual = await callTool('check_collisions', input)
|
||||
// Spec allows either. Success with empty collisions is the friendly path.
|
||||
let verdict: Verdict = 'WARN'
|
||||
let note: string | undefined
|
||||
if (actual.kind === 'unexpected_success') {
|
||||
const sc = actual.structuredContent as { collisions?: unknown[] } | undefined
|
||||
const empty = Array.isArray(sc?.collisions) && sc.collisions.length === 0
|
||||
verdict = empty ? 'PASS' : 'WARN'
|
||||
note = empty ? 'returned empty collisions (graceful)' : 'returned non-empty result'
|
||||
} else if (actual.kind === 'mcp_error' || actual.kind === 'tool_error') {
|
||||
verdict = 'PASS'
|
||||
note = 'structured error (also acceptable per spec)'
|
||||
} else {
|
||||
verdict = 'FAIL'
|
||||
}
|
||||
record(
|
||||
'T4-18',
|
||||
'check_collisions',
|
||||
"levelId doesn't exist",
|
||||
input,
|
||||
'Empty collisions result OR graceful error',
|
||||
actual,
|
||||
verdict,
|
||||
note,
|
||||
)
|
||||
}
|
||||
|
||||
// 19. validate_scene — no args → should succeed (baseline, not an error test)
|
||||
{
|
||||
const input = {}
|
||||
const actual = await callTool('validate_scene', input)
|
||||
const ok = actual.kind === 'unexpected_success'
|
||||
record(
|
||||
'T4-19',
|
||||
'validate_scene',
|
||||
'baseline: no args',
|
||||
input,
|
||||
'Success — structured { valid, errors[] }',
|
||||
actual,
|
||||
ok ? 'PASS' : 'FAIL',
|
||||
ok ? 'baseline passed' : 'baseline failed',
|
||||
)
|
||||
}
|
||||
|
||||
// 20. analyze_floorplan_image — image: ''
|
||||
{
|
||||
const input = { image: '' }
|
||||
const actual = await callTool('analyze_floorplan_image', input)
|
||||
// Expected: Zod string.min rule (we have no min, so it accepts empty),
|
||||
// falling through to sampling_unavailable (no client caps on HTTP).
|
||||
// Either is acceptable — this is a validation/sampling-guard test.
|
||||
let verdict = classifyRejection(actual)
|
||||
let note: string | undefined
|
||||
if (actual.kind === 'mcp_error') {
|
||||
if (actual.message?.includes('sampling_unavailable')) {
|
||||
note = 'sampling_unavailable (no client capabilities) — acceptable'
|
||||
verdict = 'PASS'
|
||||
} else {
|
||||
note = 'structured error'
|
||||
}
|
||||
}
|
||||
record(
|
||||
'T4-20a',
|
||||
'analyze_floorplan_image',
|
||||
"image: '' (empty string)",
|
||||
input,
|
||||
'Validation error OR sampling_unavailable',
|
||||
actual,
|
||||
verdict,
|
||||
note,
|
||||
)
|
||||
}
|
||||
|
||||
// 20b. analyze_floorplan_image — image: 'not-a-url-or-base64'
|
||||
{
|
||||
const input = { image: 'not-a-url-or-base64' }
|
||||
const actual = await callTool('analyze_floorplan_image', input)
|
||||
let verdict = classifyRejection(actual)
|
||||
let note: string | undefined
|
||||
if (actual.kind === 'mcp_error' && actual.message?.includes('sampling_unavailable')) {
|
||||
note = 'sampling_unavailable (no client capabilities)'
|
||||
verdict = 'PASS'
|
||||
}
|
||||
record(
|
||||
'T4-20b',
|
||||
'analyze_floorplan_image',
|
||||
"image: 'not-a-url-or-base64'",
|
||||
input,
|
||||
'Validation error OR sampling_unavailable',
|
||||
actual,
|
||||
verdict,
|
||||
note,
|
||||
)
|
||||
}
|
||||
|
||||
// 21. analyze_room_photo — image: ''
|
||||
{
|
||||
const input = { image: '' }
|
||||
const actual = await callTool('analyze_room_photo', input)
|
||||
let verdict = classifyRejection(actual)
|
||||
let note: string | undefined
|
||||
if (actual.kind === 'mcp_error' && actual.message?.includes('sampling_unavailable')) {
|
||||
note = 'sampling_unavailable (no client capabilities)'
|
||||
verdict = 'PASS'
|
||||
}
|
||||
record(
|
||||
'T4-21a',
|
||||
'analyze_room_photo',
|
||||
"image: '' (empty string)",
|
||||
input,
|
||||
'Validation error OR sampling_unavailable',
|
||||
actual,
|
||||
verdict,
|
||||
note,
|
||||
)
|
||||
}
|
||||
|
||||
// 21b. analyze_room_photo — image: 'not-a-url-or-base64'
|
||||
{
|
||||
const input = { image: 'not-a-url-or-base64' }
|
||||
const actual = await callTool('analyze_room_photo', input)
|
||||
let verdict = classifyRejection(actual)
|
||||
let note: string | undefined
|
||||
if (actual.kind === 'mcp_error' && actual.message?.includes('sampling_unavailable')) {
|
||||
note = 'sampling_unavailable (no client capabilities)'
|
||||
verdict = 'PASS'
|
||||
}
|
||||
record(
|
||||
'T4-21b',
|
||||
'analyze_room_photo',
|
||||
"image: 'not-a-url-or-base64'",
|
||||
input,
|
||||
'Validation error OR sampling_unavailable',
|
||||
actual,
|
||||
verdict,
|
||||
note,
|
||||
)
|
||||
}
|
||||
|
||||
// --- Post-check: scene count should be unchanged (all error paths) ---
|
||||
const scene1 = await client.callTool({ name: 'get_scene', arguments: {} })
|
||||
const nodes1 = (scene1.structuredContent as { nodes?: Record<string, unknown> })?.nodes ?? {}
|
||||
const nodeCount1 = Object.keys(nodes1).length
|
||||
console.log(`\nfinal node count = ${nodeCount1} (baseline ${nodeCount0})`)
|
||||
const delta = nodeCount1 - nodeCount0
|
||||
if (delta !== 0) {
|
||||
console.log(`WARN: node count changed by ${delta}`)
|
||||
}
|
||||
|
||||
const validationFinal = await client.callTool({
|
||||
name: 'validate_scene',
|
||||
arguments: {},
|
||||
})
|
||||
const vf = validationFinal.structuredContent as { valid: boolean; errors: unknown[] }
|
||||
console.log(`final validation: valid=${vf.valid} errors=${vf.errors.length}`)
|
||||
|
||||
// --- Emit report ---
|
||||
await writeReport(nodeCount0, nodeCount1, vf)
|
||||
|
||||
await client.close()
|
||||
}
|
||||
|
||||
async function writeReport(
|
||||
nodeCountBefore: number,
|
||||
nodeCountAfter: number,
|
||||
finalValidation: { valid: boolean; errors: unknown[] },
|
||||
): Promise<void> {
|
||||
const pass = results.filter((r) => r.verdict === 'PASS').length
|
||||
const warn = results.filter((r) => r.verdict === 'WARN').length
|
||||
const fail = results.filter((r) => r.verdict === 'FAIL').length
|
||||
|
||||
const lines: string[] = []
|
||||
lines.push('# T4 — Error Contract Verification Report')
|
||||
lines.push('')
|
||||
lines.push(`Server: \`${SERVER_URL.href}\``)
|
||||
lines.push(`Run date: ${new Date().toISOString()}`)
|
||||
lines.push('')
|
||||
lines.push('## Summary')
|
||||
lines.push('')
|
||||
lines.push(`- PASS: ${pass}`)
|
||||
lines.push(`- WARN: ${warn}`)
|
||||
lines.push(`- FAIL: ${fail}`)
|
||||
lines.push(`- Total cases: ${results.length}`)
|
||||
lines.push('')
|
||||
lines.push(`Baseline node count: ${nodeCountBefore}`)
|
||||
lines.push(`Final node count: ${nodeCountAfter} (delta=${nodeCountAfter - nodeCountBefore})`)
|
||||
lines.push(
|
||||
`Final validation: valid=${finalValidation.valid}, errors=${finalValidation.errors.length}`,
|
||||
)
|
||||
lines.push('')
|
||||
|
||||
if (fail > 0) {
|
||||
lines.push('## Failures (real bugs)')
|
||||
lines.push('')
|
||||
for (const r of results.filter((x) => x.verdict === 'FAIL')) {
|
||||
lines.push(`- **${r.id}** \`${r.tool}\`: ${r.description}`)
|
||||
lines.push(` - Expected: ${r.expected}`)
|
||||
lines.push(` - Actual kind: \`${r.actual.kind}\``)
|
||||
if (r.actual.code !== undefined) lines.push(` - Code: \`${r.actual.code}\``)
|
||||
if (r.actual.message) lines.push(` - Message: \`${r.actual.message}\``)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
lines.push('## Cases')
|
||||
lines.push('')
|
||||
for (const r of results) {
|
||||
const icon = r.verdict === 'PASS' ? '✅' : r.verdict === 'WARN' ? '⚠️' : '❌'
|
||||
lines.push(`### ${r.id} — \`${r.tool}\` — ${r.description}`)
|
||||
lines.push('')
|
||||
lines.push(`**Verdict:** ${icon} ${r.verdict}`)
|
||||
lines.push('')
|
||||
lines.push('**Input:**')
|
||||
lines.push('```json')
|
||||
lines.push(JSON.stringify(r.input, null, 2))
|
||||
lines.push('```')
|
||||
lines.push('')
|
||||
lines.push(`**Expected:** ${r.expected}`)
|
||||
lines.push('')
|
||||
lines.push('**Actual:**')
|
||||
lines.push('```json')
|
||||
lines.push(JSON.stringify(r.actual, null, 2))
|
||||
lines.push('```')
|
||||
if (r.note) {
|
||||
lines.push('')
|
||||
lines.push(`**Note:** ${r.note}`)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
const { writeFile } = await import('node:fs/promises')
|
||||
const reportPath = new URL('./REPORT.md', import.meta.url)
|
||||
await writeFile(reportPath, lines.join('\n'), 'utf8')
|
||||
console.log(`\nwrote report: ${reportPath.pathname}`)
|
||||
console.log(`summary: PASS=${pass} WARN=${warn} FAIL=${fail}`)
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('[t4] fatal:', err)
|
||||
process.exit(1)
|
||||
})
|
||||
Reference in New Issue
Block a user