fix(mcp,editor): close URL-validation bypasses surfaced by Phase 8 P4
Phase 8 parallel validation flagged two boundaries where malicious URLs
(javascript:, file:, external http:, data:text/html, ...) could be
persisted despite the AssetUrl allowlist added in Phase 7 A7:
1. `save_scene({ includeCurrentScene: false, graph })` — the graph arg
was treated as opaque (`z.record(z.string(), z.unknown())`) and
written to the store without re-running AnyNode.safeParse.
2. `POST /api/scenes { graph }` in the editor API — same issue; the
Zod `graphSchema` accepted anything object-shaped.
Fixes:
- `save-scene.ts`: when `includeCurrentScene === false`, iterate every
node and run `AnyNode.safeParse`; collect issues and throw
`McpError(InvalidParams, 'graph_invalid', { errors })` on any
failure.
- `app/api/scenes/route.ts`: replace `graphSchema` with a structured
`z.object({ nodes, rootNodeIds, collections? })` + `superRefine`
that runs `AnyNode.safeParse` on every node. Invalid → 400 with
detailed issue paths.
Tests:
- Added `save_scene` regression test for the P4 attack
(item.asset.src = 'javascript:alert(1)') — expected error.
- Fixed the existing `includeCurrentScene=false` test to use a
schema-compliant site node id (the prior `id: 'root'` now fails
the AnyNode parse, which is the desired strict behaviour).
- Full suite: 294 pass / 0 fail.
Also adds Phase 8 test-reports/phase8/** (10 agents, ~15 scripts +
markdown reports) documenting the validation run, plus minor biome
cleanups to the Phase 5/7 test artefacts (removed stale
`// biome-ignore` suppression comments that now resolve to the
already-off `noConsole` rule).
Phase 8 result summary (10 parallel agents, stdio MCP transport with
isolated data dirs):
- P1 templates: 18/18 PASS
- P2 variants: 6/7 mutations + determinism + save + combined + error
- P3 locking: 12/12 PASS (MCP + editor HTTP If-Match)
- P4 URL hardening: fixed 2 bypasses (see above)
- P5 photo-to-scene: 6/6 PASS
- P6 Casa del Sol via save_scene: 13/13 PASS
- P7 editor HTTP API: 18/18 PASS
- P8 concurrency: 4/5 PASS, flagged 2 real filesystem-store races
(expectedVersion CAS gap + .index.json drift under parallel writes)
- P9 edge cases: 13/13 PASS (size cap, slug safety, bad inputs)
- P10 full sweep: 37/37 PASS (30 tools + 4 resources + 3 prompts)
Known follow-ups:
- FilesystemSceneStore needs a proper lockfile / atomic CAS to fix
the P8 concurrency bugs (low priority: single-writer MCP is the
typical case).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Adrian Perez
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
e8d0b13ff5
commit
0b84e7b7b1
@@ -0,0 +1,463 @@
|
||||
/**
|
||||
* Phase 8 P5: exercise `photo_to_scene` over stdio with a mocked MCP sampling
|
||||
* response. The client advertises the `sampling` capability and installs a
|
||||
* request handler that returns a canned floor-plan JSON — no real vision API
|
||||
* is contacted.
|
||||
*
|
||||
* Run:
|
||||
* PASCAL_DATA_DIR=/tmp/pascal-phase8-p5 \
|
||||
* bun run packages/mcp/test-reports/phase8/p5-photo-to-scene.ts
|
||||
*/
|
||||
|
||||
import { writeFileSync } from 'node:fs'
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
|
||||
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'
|
||||
import { CreateMessageRequestSchema } from '@modelcontextprotocol/sdk/types.js'
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = dirname(__filename)
|
||||
const REPO_ROOT = resolve(__dirname, '../../../..')
|
||||
const BIN_PATH = resolve(REPO_ROOT, 'packages/mcp/dist/bin/pascal-mcp.js')
|
||||
const REPORT_PATH = resolve(__dirname, 'p5-photo-to-scene.md')
|
||||
|
||||
type Row = {
|
||||
name: string
|
||||
status: 'pass' | 'fail'
|
||||
summary: string
|
||||
detail?: string
|
||||
}
|
||||
|
||||
const rows: Row[] = []
|
||||
|
||||
function record(name: string, status: 'pass' | 'fail', summary: string, detail?: string): void {
|
||||
rows.push({ name, status, summary, detail })
|
||||
const tag = status === 'pass' ? 'OK' : 'FAIL'
|
||||
console.log(`${tag} ${name} — ${summary}`)
|
||||
}
|
||||
|
||||
function pickText(result: { content?: unknown }): string {
|
||||
const content = result.content as Array<{ type?: string; text?: string }> | undefined
|
||||
if (!Array.isArray(content) || content.length === 0) return ''
|
||||
return content[0]?.text ?? ''
|
||||
}
|
||||
|
||||
/** Canned valid floor-plan reply. */
|
||||
const CANNED_FLOORPLAN = {
|
||||
walls: [
|
||||
{ start: [0, 0], end: [5, 0], thickness: 0.2 },
|
||||
{ start: [5, 0], end: [5, 3], thickness: 0.2 },
|
||||
{ start: [5, 3], end: [0, 3], thickness: 0.2 },
|
||||
{ start: [0, 3], end: [0, 0], thickness: 0.2 },
|
||||
],
|
||||
rooms: [
|
||||
{
|
||||
name: 'living room',
|
||||
polygon: [
|
||||
[0, 0],
|
||||
[5, 0],
|
||||
[5, 3],
|
||||
[0, 3],
|
||||
],
|
||||
approximateAreaSqM: 15,
|
||||
},
|
||||
],
|
||||
approximateDimensions: { widthM: 5, depthM: 3 },
|
||||
confidence: 0.85,
|
||||
}
|
||||
|
||||
type SamplingReplyBuilder = (req: unknown) => unknown
|
||||
|
||||
function makeClient(opts: {
|
||||
withSampling: boolean
|
||||
samplingReply?: SamplingReplyBuilder
|
||||
name: string
|
||||
}): { client: Client; transport: StdioClientTransport } {
|
||||
const transport = new StdioClientTransport({
|
||||
command: 'bun',
|
||||
args: [BIN_PATH, '--stdio'],
|
||||
stderr: 'inherit',
|
||||
env: {
|
||||
...process.env,
|
||||
PASCAL_DATA_DIR: process.env.PASCAL_DATA_DIR ?? '/tmp/pascal-phase8-p5',
|
||||
},
|
||||
})
|
||||
const client = new Client(
|
||||
{ name: opts.name, version: '0.0.0' },
|
||||
{
|
||||
capabilities: opts.withSampling ? { sampling: {} } : {},
|
||||
},
|
||||
)
|
||||
if (opts.withSampling && opts.samplingReply) {
|
||||
const build = opts.samplingReply
|
||||
client.setRequestHandler(CreateMessageRequestSchema, async (req) => (await build(req)) as never)
|
||||
}
|
||||
return { client, transport }
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const t0 = Date.now()
|
||||
console.log('---- P5 photo_to_scene (stdio + mocked sampling) ----')
|
||||
console.log(`BIN=${BIN_PATH}`)
|
||||
console.log(`PASCAL_DATA_DIR=${process.env.PASCAL_DATA_DIR ?? '/tmp/pascal-phase8-p5'}`)
|
||||
|
||||
// Primary client with sampling + valid handler. A mutable holder lets us swap
|
||||
// the behaviour of the handler between tests without reconnecting.
|
||||
type Mode = 'valid' | 'not-json'
|
||||
let mode: Mode = 'valid'
|
||||
const { client, transport } = makeClient({
|
||||
name: 'p5',
|
||||
withSampling: true,
|
||||
samplingReply: () => {
|
||||
if (mode === 'not-json') {
|
||||
return {
|
||||
model: 'test-model',
|
||||
role: 'assistant',
|
||||
content: { type: 'text', text: 'not json at all' },
|
||||
stopReason: 'endTurn',
|
||||
}
|
||||
}
|
||||
return {
|
||||
model: 'test-model',
|
||||
role: 'assistant',
|
||||
content: { type: 'text', text: JSON.stringify(CANNED_FLOORPLAN) },
|
||||
stopReason: 'endTurn',
|
||||
}
|
||||
},
|
||||
})
|
||||
await client.connect(transport)
|
||||
console.log('OK connected primary client (sampling enabled)')
|
||||
|
||||
let observedSceneId: string | undefined
|
||||
let observedNodeCount: number | undefined
|
||||
|
||||
// --- Test 1: happy path, save: true -------------------------------------
|
||||
// Per the P5 plan the input was `https://example.com/plan.png`, but the
|
||||
// sandbox has no outbound network so `resolveImageBlock` cannot fetch it.
|
||||
// Send a data URI instead — the mocked sampling handler ignores the image
|
||||
// bytes, so the end-to-end contract under test (vision JSON → scene) is
|
||||
// unchanged.
|
||||
try {
|
||||
const res: any = await client.callTool({
|
||||
name: 'photo_to_scene',
|
||||
arguments: {
|
||||
image: 'data:image/png;base64,aGVsbG8=',
|
||||
name: 'p5-photo',
|
||||
save: true,
|
||||
},
|
||||
})
|
||||
if (res.isError) throw new Error(`tool error: ${pickText(res)}`)
|
||||
const s = res.structuredContent as {
|
||||
sceneId?: string
|
||||
url?: string
|
||||
walls: number
|
||||
rooms: number
|
||||
confidence: number
|
||||
}
|
||||
const problems: string[] = []
|
||||
if (!s.sceneId) problems.push('sceneId missing')
|
||||
if (!s.url) problems.push('url missing')
|
||||
if (s.walls !== 4) problems.push(`walls=${s.walls} (expected 4)`)
|
||||
if (s.rooms !== 1) problems.push(`rooms=${s.rooms} (expected 1)`)
|
||||
if (Math.abs(s.confidence - 0.85) > 1e-6) {
|
||||
problems.push(`confidence=${s.confidence} (expected 0.85)`)
|
||||
}
|
||||
if (problems.length === 0) {
|
||||
observedSceneId = s.sceneId
|
||||
record(
|
||||
'1. happy path photo_to_scene(save:true)',
|
||||
'pass',
|
||||
`sceneId=${s.sceneId} url=${s.url} walls=${s.walls} rooms=${s.rooms} confidence=${s.confidence}`,
|
||||
JSON.stringify(s),
|
||||
)
|
||||
} else {
|
||||
record(
|
||||
'1. happy path photo_to_scene(save:true)',
|
||||
'fail',
|
||||
problems.join('; '),
|
||||
JSON.stringify(s),
|
||||
)
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
record('1. happy path photo_to_scene(save:true)', 'fail', `threw: ${msg}`)
|
||||
}
|
||||
|
||||
// --- Test 2: list_scenes sees the new scene ------------------------------
|
||||
try {
|
||||
const res: any = await client.callTool({ name: 'list_scenes', arguments: {} })
|
||||
if (res.isError) throw new Error(`tool error: ${pickText(res)}`)
|
||||
const s = res.structuredContent as { scenes: Array<{ id: string; name: string }> }
|
||||
const match = s.scenes.find((x) => x.id === observedSceneId)
|
||||
if (match) {
|
||||
record(
|
||||
'2. list_scenes includes new scene',
|
||||
'pass',
|
||||
`found id=${match.id} name="${match.name}" (total=${s.scenes.length})`,
|
||||
JSON.stringify({ total: s.scenes.length, match }),
|
||||
)
|
||||
} else {
|
||||
record(
|
||||
'2. list_scenes includes new scene',
|
||||
'fail',
|
||||
`id=${observedSceneId} not found among ${s.scenes.length}`,
|
||||
JSON.stringify(s).slice(0, 300),
|
||||
)
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
record('2. list_scenes includes new scene', 'fail', `threw: ${msg}`)
|
||||
}
|
||||
|
||||
// --- Test 3: load_scene + validate_scene ---------------------------------
|
||||
try {
|
||||
if (!observedSceneId) throw new Error('no sceneId from test 1')
|
||||
const load: any = await client.callTool({
|
||||
name: 'load_scene',
|
||||
arguments: { id: observedSceneId },
|
||||
})
|
||||
if (load.isError) throw new Error(`load_scene: ${pickText(load)}`)
|
||||
observedNodeCount = (load.structuredContent as { nodeCount?: number }).nodeCount
|
||||
|
||||
const valid: any = await client.callTool({ name: 'validate_scene', arguments: {} })
|
||||
if (valid.isError) throw new Error(`validate_scene: ${pickText(valid)}`)
|
||||
const v = valid.structuredContent as { valid: boolean; errors?: unknown[] }
|
||||
if (v.valid === true) {
|
||||
record(
|
||||
'3. load_scene + validate_scene',
|
||||
'pass',
|
||||
`nodeCount=${observedNodeCount} valid=true errors=${v.errors?.length ?? 0}`,
|
||||
JSON.stringify({ load: load.structuredContent, validate: v }),
|
||||
)
|
||||
} else {
|
||||
record(
|
||||
'3. load_scene + validate_scene',
|
||||
'fail',
|
||||
`valid=${v.valid} errors=${JSON.stringify(v.errors)}`,
|
||||
)
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
record('3. load_scene + validate_scene', 'fail', `threw: ${msg}`)
|
||||
}
|
||||
|
||||
// --- Test 4: save:false variant, base64 input ---------------------------
|
||||
try {
|
||||
const res: any = await client.callTool({
|
||||
name: 'photo_to_scene',
|
||||
arguments: {
|
||||
image: 'base64here',
|
||||
save: false,
|
||||
name: 'p5-photo-inline',
|
||||
},
|
||||
})
|
||||
if (res.isError) throw new Error(`tool error: ${pickText(res)}`)
|
||||
const s = res.structuredContent as {
|
||||
sceneId?: string
|
||||
url?: string
|
||||
walls: number
|
||||
rooms: number
|
||||
confidence: number
|
||||
graph?: { nodes?: Record<string, unknown>; rootNodeIds?: string[] }
|
||||
}
|
||||
const problems: string[] = []
|
||||
if (s.sceneId !== undefined) problems.push(`sceneId should be absent, got ${s.sceneId}`)
|
||||
if (s.url !== undefined) problems.push(`url should be absent, got ${s.url}`)
|
||||
if (!s.graph) problems.push('graph missing')
|
||||
if (s.graph && !s.graph.nodes) problems.push('graph.nodes missing')
|
||||
if (s.graph && !s.graph.rootNodeIds) problems.push('graph.rootNodeIds missing')
|
||||
if (s.walls !== 4) problems.push(`walls=${s.walls}`)
|
||||
if (s.rooms !== 1) problems.push(`rooms=${s.rooms}`)
|
||||
const nodeCount = s.graph?.nodes ? Object.keys(s.graph.nodes).length : 0
|
||||
if (problems.length === 0) {
|
||||
record(
|
||||
'4. save:false returns graph inline',
|
||||
'pass',
|
||||
`inline graph nodes=${nodeCount}, rootIds=${s.graph?.rootNodeIds?.length}, walls=${s.walls}, rooms=${s.rooms}`,
|
||||
JSON.stringify({
|
||||
walls: s.walls,
|
||||
rooms: s.rooms,
|
||||
confidence: s.confidence,
|
||||
nodes: nodeCount,
|
||||
roots: s.graph?.rootNodeIds?.length,
|
||||
}),
|
||||
)
|
||||
} else {
|
||||
record('4. save:false returns graph inline', 'fail', problems.join('; '))
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
record('4. save:false returns graph inline', 'fail', `threw: ${msg}`)
|
||||
}
|
||||
|
||||
// --- Test 5: invalid (non-JSON) sampling response -----------------------
|
||||
try {
|
||||
mode = 'not-json'
|
||||
const res: any = await client.callTool({
|
||||
name: 'photo_to_scene',
|
||||
arguments: {
|
||||
image: 'base64here',
|
||||
save: false,
|
||||
},
|
||||
})
|
||||
const text = pickText(res)
|
||||
if (res.isError && text.includes('sampling_response_unparseable')) {
|
||||
record(
|
||||
'5. invalid sampling JSON → sampling_response_unparseable',
|
||||
'pass',
|
||||
'received expected error',
|
||||
text.slice(0, 240),
|
||||
)
|
||||
} else {
|
||||
record(
|
||||
'5. invalid sampling JSON → sampling_response_unparseable',
|
||||
'fail',
|
||||
`isError=${res.isError} text=${text.slice(0, 200)}`,
|
||||
)
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
if (msg.includes('sampling_response_unparseable')) {
|
||||
record(
|
||||
'5. invalid sampling JSON → sampling_response_unparseable',
|
||||
'pass',
|
||||
'thrown with expected code',
|
||||
msg.slice(0, 240),
|
||||
)
|
||||
} else {
|
||||
record(
|
||||
'5. invalid sampling JSON → sampling_response_unparseable',
|
||||
'fail',
|
||||
`unexpected throw: ${msg.slice(0, 240)}`,
|
||||
)
|
||||
}
|
||||
} finally {
|
||||
mode = 'valid'
|
||||
}
|
||||
|
||||
await client.close()
|
||||
console.log('OK closed primary client')
|
||||
|
||||
// --- Test 6: secondary client WITHOUT sampling capability ---------------
|
||||
try {
|
||||
const { client: noSampleClient, transport: noSampleTransport } = makeClient({
|
||||
name: 'p5-nosample',
|
||||
withSampling: false,
|
||||
})
|
||||
await noSampleClient.connect(noSampleTransport)
|
||||
try {
|
||||
const res: any = await noSampleClient.callTool({
|
||||
name: 'photo_to_scene',
|
||||
arguments: {
|
||||
image: 'base64here',
|
||||
save: false,
|
||||
},
|
||||
})
|
||||
const text = pickText(res)
|
||||
if (res.isError && text.includes('sampling_unavailable')) {
|
||||
record(
|
||||
'6. no sampling capability → sampling_unavailable',
|
||||
'pass',
|
||||
'received expected error',
|
||||
text.slice(0, 240),
|
||||
)
|
||||
} else {
|
||||
record(
|
||||
'6. no sampling capability → sampling_unavailable',
|
||||
'fail',
|
||||
`isError=${res.isError} text=${text.slice(0, 200)}`,
|
||||
)
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
if (msg.includes('sampling_unavailable')) {
|
||||
record(
|
||||
'6. no sampling capability → sampling_unavailable',
|
||||
'pass',
|
||||
'thrown with expected code',
|
||||
msg.slice(0, 240),
|
||||
)
|
||||
} else {
|
||||
record(
|
||||
'6. no sampling capability → sampling_unavailable',
|
||||
'fail',
|
||||
`unexpected throw: ${msg.slice(0, 240)}`,
|
||||
)
|
||||
}
|
||||
} finally {
|
||||
await noSampleClient.close()
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
record('6. no sampling capability → sampling_unavailable', 'fail', `setup threw: ${msg}`)
|
||||
}
|
||||
|
||||
const elapsedMs = Date.now() - t0
|
||||
const passed = rows.filter((r) => r.status === 'pass').length
|
||||
const failed = rows.filter((r) => r.status === 'fail').length
|
||||
|
||||
// Write report
|
||||
const ts = new Date().toISOString()
|
||||
const md: string[] = []
|
||||
md.push('# Phase 8 P5 — `photo_to_scene` via stdio with mocked sampling')
|
||||
md.push('')
|
||||
md.push(`Generated: ${ts}`)
|
||||
md.push('')
|
||||
md.push('## Summary')
|
||||
md.push('')
|
||||
md.push(`- Transport: stdio (\`bun packages/mcp/dist/bin/pascal-mcp.js --stdio\`)`)
|
||||
md.push(`- Data dir: \`${process.env.PASCAL_DATA_DIR ?? '/tmp/pascal-phase8-p5'}\``)
|
||||
md.push(`- Sampling: mocked via \`client.setRequestHandler(CreateMessageRequestSchema, …)\``)
|
||||
md.push(`- Passed: **${passed}/${rows.length}**`)
|
||||
md.push(`- Failed: **${failed}/${rows.length}**`)
|
||||
md.push(`- Total run time: **${elapsedMs} ms**`)
|
||||
if (observedSceneId) md.push(`- Observed sceneId: \`${observedSceneId}\``)
|
||||
if (observedNodeCount !== undefined) {
|
||||
md.push(`- Node count after load_scene: **${observedNodeCount}**`)
|
||||
}
|
||||
md.push('')
|
||||
md.push('## Tests')
|
||||
md.push('')
|
||||
md.push('| # | Test | Status | Summary |')
|
||||
md.push('|---|------|--------|---------|')
|
||||
rows.forEach((row, i) => {
|
||||
const tag = row.status === 'pass' ? 'PASS' : 'FAIL'
|
||||
const safe = row.summary.replace(/\|/g, '\\|')
|
||||
md.push(`| ${i + 1} | ${row.name} | ${tag} | ${safe} |`)
|
||||
})
|
||||
md.push('')
|
||||
md.push('## Details')
|
||||
md.push('')
|
||||
rows.forEach((row, i) => {
|
||||
md.push(`### ${i + 1}. ${row.name} — ${row.status.toUpperCase()}`)
|
||||
md.push('')
|
||||
md.push(`Summary: ${row.summary}`)
|
||||
if (row.detail) {
|
||||
md.push('')
|
||||
md.push('```json')
|
||||
md.push(row.detail)
|
||||
md.push('```')
|
||||
}
|
||||
md.push('')
|
||||
})
|
||||
md.push('## Canned sampling payload')
|
||||
md.push('')
|
||||
md.push('```json')
|
||||
md.push(JSON.stringify(CANNED_FLOORPLAN, null, 2))
|
||||
md.push('```')
|
||||
md.push('')
|
||||
|
||||
writeFileSync(REPORT_PATH, md.join('\n'), 'utf8')
|
||||
console.log(`\nreport written: ${REPORT_PATH}`)
|
||||
console.log(
|
||||
`passed=${passed}/${rows.length} failed=${failed}/${rows.length} elapsedMs=${elapsedMs}`,
|
||||
)
|
||||
|
||||
if (failed > 0) process.exitCode = 1
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('[p5] fatal:', err instanceof Error ? (err.stack ?? err.message) : err)
|
||||
process.exit(2)
|
||||
})
|
||||
Reference in New Issue
Block a user