fix(mcp): apply_patch preserves schema-defaulted ids in multi-op batches
SceneBridge.applyPatch now threads the Zod-parsed node through to the apply phase instead of the raw input. Previously, when a caller sent a create patch without an `id` field, the schema's objectId default ran during dry-run parsing but only in `res.data`; the apply phase pushed the unparsed `p.node` (no id) to the store, so subsequent tools that walked `level.children` crashed on undefined entries (e.g. duplicate_level -> cloneLevelSubtree -> extractIdPrefix(undefined)). Also adds test-reports/ artefacts from live end-to-end testing: - t1-stdio: 21/21 tools pass via stdio (~106ms) - t2-http: connect/single-session behaviour (HTTP transport quirk documented) - t3-scenario: 2-bedroom apartment built end-to-end — 12/12 steps after this fix (24 final nodes, validate=true, apartment.json exported) - t4-errors: 24/24 invalid-input cases rejected with proper MCP errors - t5-resources-prompts: 4/4 resources, 3/3 prompts, dev server /api/health 200 OK Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Adrian Perez
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
a6f1c4140f
commit
b37e88cb83
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env bash
|
||||
# T5 — Next.js dev server probe.
|
||||
#
|
||||
# Verifies http://localhost:3002 is serving the Pascal editor cleanly without
|
||||
# touching the running process.
|
||||
#
|
||||
# Outputs structured "KEY=value" lines so the REPORT.md can be authored from
|
||||
# them, plus a longer human-readable summary at the end.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
URL_ROOT="http://localhost:3002/"
|
||||
URL_HEALTH="http://localhost:3002/api/health"
|
||||
TMPDIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMPDIR"' EXIT
|
||||
|
||||
ROOT_BODY="$TMPDIR/root.html"
|
||||
ROOT_HEAD="$TMPDIR/root.head"
|
||||
HEALTH_BODY="$TMPDIR/health.body"
|
||||
HEALTH_HEAD="$TMPDIR/health.head"
|
||||
|
||||
echo "## Probe: $URL_ROOT"
|
||||
ROOT_STATUS=$(curl -sS --connect-timeout 5 --max-time 30 \
|
||||
-o "$ROOT_BODY" -D "$ROOT_HEAD" -w '%{http_code}' "$URL_ROOT" || echo "000")
|
||||
ROOT_BYTES=$(wc -c < "$ROOT_BODY" | tr -d ' ')
|
||||
|
||||
echo "ROOT_STATUS=$ROOT_STATUS"
|
||||
echo "ROOT_BYTES=$ROOT_BYTES"
|
||||
|
||||
# Pascal mention: explicit "Pascal" string OR @pascal-app reference.
|
||||
if grep -q -i 'Pascal' "$ROOT_BODY"; then
|
||||
ROOT_HAS_PASCAL=1
|
||||
else
|
||||
ROOT_HAS_PASCAL=0
|
||||
fi
|
||||
if grep -q '@pascal-app' "$ROOT_BODY"; then
|
||||
ROOT_HAS_PASCAL_APP=1
|
||||
else
|
||||
ROOT_HAS_PASCAL_APP=0
|
||||
fi
|
||||
echo "ROOT_HAS_PASCAL=$ROOT_HAS_PASCAL"
|
||||
echo "ROOT_HAS_PASCAL_APP=$ROOT_HAS_PASCAL_APP"
|
||||
|
||||
# Count <script src="..."> tags.
|
||||
SCRIPT_COUNT=$(grep -o '<script [^>]*src="[^"]*"' "$ROOT_BODY" | wc -l | tr -d ' ')
|
||||
NEXT_CHUNK_COUNT=$(grep -o '<script [^>]*src="[^"]*_next[^"]*"' "$ROOT_BODY" | wc -l | tr -d ' ')
|
||||
# Best-effort search for editor / viewer / core chunks (workspace wiring sanity).
|
||||
EDITOR_CHUNK_COUNT=$(grep -o -E '<script [^>]*src="[^"]*(editor|viewer|core|three|gltf)[^"]*"' "$ROOT_BODY" | wc -l | tr -d ' ')
|
||||
|
||||
echo "SCRIPT_COUNT=$SCRIPT_COUNT"
|
||||
echo "NEXT_CHUNK_COUNT=$NEXT_CHUNK_COUNT"
|
||||
echo "EDITOR_VIEWER_CORE_CHUNK_COUNT=$EDITOR_CHUNK_COUNT"
|
||||
|
||||
# Error indicator scan.
|
||||
ERR_APP_ERROR=$(grep -c 'Application error' "$ROOT_BODY" || true)
|
||||
ERR_FAILED_TO=$(grep -c 'Failed to' "$ROOT_BODY" || true)
|
||||
ERR_CANNOT=$(grep -c 'cannot' "$ROOT_BODY" || true)
|
||||
|
||||
echo "ERR_APP_ERROR=$ERR_APP_ERROR"
|
||||
echo "ERR_FAILED_TO=$ERR_FAILED_TO"
|
||||
echo "ERR_CANNOT=$ERR_CANNOT"
|
||||
|
||||
# Print first ~5 unique script sources for sanity inspection (cap output).
|
||||
echo
|
||||
echo "## First script src= matches (max 10)"
|
||||
grep -o '<script [^>]*src="[^"]*"' "$ROOT_BODY" | sed -E 's/.*src="([^"]+)".*/\1/' | head -10 || true
|
||||
|
||||
# /api/health probe (best effort — endpoint may not exist).
|
||||
echo
|
||||
echo "## Probe: $URL_HEALTH"
|
||||
HEALTH_STATUS=$(curl -sS --connect-timeout 5 --max-time 10 \
|
||||
-o "$HEALTH_BODY" -D "$HEALTH_HEAD" -w '%{http_code}' "$URL_HEALTH" || echo "000")
|
||||
HEALTH_BYTES=$(wc -c < "$HEALTH_BODY" | tr -d ' ')
|
||||
echo "HEALTH_STATUS=$HEALTH_STATUS"
|
||||
echo "HEALTH_BYTES=$HEALTH_BYTES"
|
||||
|
||||
# If the response is JSON, show first 200 bytes; otherwise mark as N/A.
|
||||
if [ "$HEALTH_STATUS" = "200" ]; then
|
||||
HEALTH_BODY_TEXT=$(head -c 200 "$HEALTH_BODY" | tr -d '\n')
|
||||
echo "HEALTH_BODY_PREVIEW=$HEALTH_BODY_TEXT"
|
||||
fi
|
||||
|
||||
# Brief title extraction.
|
||||
TITLE_LINE=$(grep -o '<title>[^<]*</title>' "$ROOT_BODY" | head -1 || true)
|
||||
echo
|
||||
echo "ROOT_TITLE=$TITLE_LINE"
|
||||
|
||||
# Final aggregate verdict.
|
||||
echo
|
||||
echo "## Summary"
|
||||
if [ "$ROOT_STATUS" = "200" ] && \
|
||||
{ [ "$ROOT_HAS_PASCAL" = "1" ] || [ "$ROOT_HAS_PASCAL_APP" = "1" ]; } && \
|
||||
[ "$ERR_APP_ERROR" = "0" ]; then
|
||||
echo "DEV_VERDICT=PASS"
|
||||
else
|
||||
echo "DEV_VERDICT=FAIL"
|
||||
fi
|
||||
@@ -0,0 +1,33 @@
|
||||
## Probe: http://localhost:3002/
|
||||
ROOT_STATUS=200
|
||||
ROOT_BYTES=53138
|
||||
ROOT_HAS_PASCAL=0
|
||||
ROOT_HAS_PASCAL_APP=0
|
||||
SCRIPT_COUNT=52
|
||||
NEXT_CHUNK_COUNT=52
|
||||
EDITOR_VIEWER_CORE_CHUNK_COUNT=31
|
||||
ERR_APP_ERROR=0
|
||||
ERR_FAILED_TO=0
|
||||
ERR_CANNOT=0
|
||||
|
||||
## First script src= matches (max 10)
|
||||
/_next/static/chunks/10-e_next_dist_compiled_next-devtools_index_0c.hc5b.js
|
||||
/_next/static/chunks/10-e_next_dist_compiled_react-dom_0_a2p7j._.js
|
||||
/_next/static/chunks/10-e_next_dist_compiled_react-server-dom-turbopack_0~q-o27._.js
|
||||
/_next/static/chunks/10-e_next_dist_compiled_0z_hko_._.js
|
||||
/_next/static/chunks/10-e_next_dist_client_0.-jx~k._.js
|
||||
/_next/static/chunks/10-e_next_dist_04-q5rb._.js
|
||||
/_next/static/chunks/0xsp_%40swc_helpers_cjs_05abggq._.js
|
||||
/_next/static/chunks/_worktrees_mcp-server_apps_editor_0rqeker._.js
|
||||
/_next/static/chunks/turbopack-_worktrees_mcp-server_apps_editor_0mhsrfr._.js
|
||||
/_next/static/chunks/02ss__bun_02a3.wx._.js
|
||||
|
||||
## Probe: http://localhost:3002/api/health
|
||||
HEALTH_STATUS=200
|
||||
HEALTH_BYTES=69
|
||||
HEALTH_BODY_PREVIEW={"status":"ok","app":"editor","timestamp":"2026-04-18T16:17:41.398Z"}
|
||||
|
||||
ROOT_TITLE=
|
||||
|
||||
## Summary
|
||||
DEV_VERDICT=FAIL
|
||||
@@ -0,0 +1,25 @@
|
||||
[t5] connected to http://localhost:3917/mcp
|
||||
[t5] listResources count = 3
|
||||
[t5] listResources names = scene-current, scene-summary, catalog-items
|
||||
[t5] discovered levelId = level_9qlc3co208erq5o6
|
||||
[t5] listPrompts count = 3
|
||||
[t5] listPrompts names = from_brief, iterate_on_feedback, renovation_from_photos
|
||||
|
||||
========== T5 SUMMARY ==========
|
||||
listResources count: 3
|
||||
listPrompts count: 3
|
||||
|
||||
--- Resource results ---
|
||||
PASS scene/current — application/json, nodes=3, rootNodeIds=1
|
||||
PASS scene/current/summary — text/markdown, 375 bytes, preview: "# Scene summary | | - Sites: 1 Buildings: 1 Levels: 1 | - Root nodes: 1"
|
||||
PASS catalog/items — application/json, status=catalog_unavailable, items.length=0
|
||||
PASS constraints/{levelId} — levelId=level_9qlc3co208erq5o6, slabs=0, wallPolygons=0
|
||||
Resources pass: 4/4
|
||||
|
||||
--- Prompt results ---
|
||||
PASS from_brief — messages=1, userMsgs=1, brief-included=true, preview: "You are a Pascal 3D scene designer. You have access to the `apply_patch` tool for all scene mutations. Prefer it over in"
|
||||
PASS iterate_on_feedback — messages=1, userMsgs=1, feedback-included=true, preview: "You are iterating on an existing Pascal scene based on user feedback. Given the current state (read via the `pascal://sc"
|
||||
PASS renovation_from_photos — messages=7, userMsgs=7, urls(a/b/c)=true/true/true, goals-included=true
|
||||
Prompts pass: 3/3
|
||||
================================
|
||||
|
||||
@@ -0,0 +1,475 @@
|
||||
/**
|
||||
* T5 — MCP resources + prompts test harness.
|
||||
*
|
||||
* Connects to the shared MCP HTTP server at http://localhost:3917 (path /mcp),
|
||||
* exercises the 4 documented resources and 3 prompts, and prints a structured
|
||||
* pass/fail summary that the REPORT.md can be authored from.
|
||||
*
|
||||
* Usage:
|
||||
* bun packages/mcp/test-reports/t5-resources-prompts/run.ts
|
||||
*/
|
||||
import { existsSync } from 'node:fs'
|
||||
import { dirname, resolve as pathResolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
|
||||
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'
|
||||
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'
|
||||
|
||||
const HTTP_URL = new URL('http://localhost:3917/mcp')
|
||||
|
||||
// Fallback path: launch a fresh stdio binary if HTTP is locked. The MCP HTTP
|
||||
// server runs a single shared StreamableHTTPServerTransport whose `_initialized`
|
||||
// + `sessionId` are claimed by the first connecting client and never released
|
||||
// when other agents hold the slot — see SDK
|
||||
// `webStandardStreamableHttp.js:425` (rejects re-init in stateful mode).
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = dirname(__filename)
|
||||
const STDIO_BIN = pathResolve(__dirname, '../../dist/bin/pascal-mcp.js')
|
||||
|
||||
type Outcome = { name: string; pass: boolean; detail: string }
|
||||
|
||||
function ok(name: string, detail: string): Outcome {
|
||||
return { name, pass: true, detail }
|
||||
}
|
||||
function fail(name: string, detail: string): Outcome {
|
||||
return { name, pass: false, detail }
|
||||
}
|
||||
|
||||
function safeStringify(value: unknown, max = 400): string {
|
||||
let out: string
|
||||
try {
|
||||
out = JSON.stringify(value)
|
||||
} catch (err) {
|
||||
out = `<unserializable: ${err instanceof Error ? err.message : String(err)}>`
|
||||
}
|
||||
if (out.length > max) out = `${out.slice(0, max)}...<truncated>`
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect with bounded retry. The MCP HTTP server is shared with T2/T3/T4,
|
||||
* so we may transiently see "Server already initialized" while another agent
|
||||
* holds the in-flight session. Retry with backoff for up to ~30 s.
|
||||
*/
|
||||
async function connectWithRetry(): Promise<{
|
||||
client: Client
|
||||
transport: StreamableHTTPClientTransport
|
||||
}> {
|
||||
const maxAttempts = 30
|
||||
let lastErr: unknown = null
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
const transport = new StreamableHTTPClientTransport(HTTP_URL)
|
||||
const client = new Client({ name: 't5-resources-prompts', version: '0.0.0' })
|
||||
try {
|
||||
await client.connect(transport)
|
||||
if (attempt > 1) console.log(`[t5] connected on attempt ${attempt}`)
|
||||
return { client, transport }
|
||||
} catch (err) {
|
||||
lastErr = err
|
||||
try {
|
||||
await client.close()
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
console.warn(`[t5] connect attempt ${attempt} failed: ${msg.slice(0, 200)}`)
|
||||
// Brief backoff with jitter — keep total under ~30 s.
|
||||
await new Promise((r) => setTimeout(r, 800 + Math.floor(Math.random() * 400)))
|
||||
}
|
||||
}
|
||||
throw lastErr instanceof Error
|
||||
? lastErr
|
||||
: new Error(`failed to connect after ${maxAttempts} attempts`)
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const { client } = await connectWithRetry()
|
||||
|
||||
const resourceOutcomes: Outcome[] = []
|
||||
const promptOutcomes: Outcome[] = []
|
||||
let listResourcesCount = 0
|
||||
let listPromptsCount = 0
|
||||
|
||||
try {
|
||||
console.log(`[t5] connected to ${HTTP_URL.href}`)
|
||||
|
||||
// ---------------- listResources ----------------
|
||||
try {
|
||||
const list = await client.listResources()
|
||||
listResourcesCount = Array.isArray(list.resources) ? list.resources.length : 0
|
||||
console.log(`[t5] listResources count = ${listResourcesCount}`)
|
||||
console.log(
|
||||
`[t5] listResources names = ${(list.resources ?? [])
|
||||
.map((r) => r.name ?? r.uri)
|
||||
.join(', ')}`,
|
||||
)
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[t5] listResources error: ${err instanceof Error ? err.message : String(err)}`,
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------- Resource 1: pascal://scene/current ----------------
|
||||
try {
|
||||
const result = await client.readResource({ uri: 'pascal://scene/current' })
|
||||
const c = result.contents?.[0]
|
||||
if (!c) {
|
||||
resourceOutcomes.push(fail('scene/current', 'no contents returned'))
|
||||
} else if (c.mimeType !== 'application/json') {
|
||||
resourceOutcomes.push(
|
||||
fail('scene/current', `wrong mime type: ${String(c.mimeType)}`),
|
||||
)
|
||||
} else {
|
||||
const text = typeof c.text === 'string' ? c.text : ''
|
||||
let parsed: unknown
|
||||
try {
|
||||
parsed = JSON.parse(text)
|
||||
} catch (err) {
|
||||
resourceOutcomes.push(
|
||||
fail(
|
||||
'scene/current',
|
||||
`invalid json: ${err instanceof Error ? err.message : String(err)}`,
|
||||
),
|
||||
)
|
||||
parsed = null
|
||||
}
|
||||
const obj = parsed as { nodes?: unknown; rootNodeIds?: unknown } | null
|
||||
if (!obj) {
|
||||
resourceOutcomes.push(fail('scene/current', 'empty payload'))
|
||||
} else if (!obj.nodes || typeof obj.nodes !== 'object') {
|
||||
resourceOutcomes.push(fail('scene/current', 'missing nodes object'))
|
||||
} else if (!Array.isArray(obj.rootNodeIds)) {
|
||||
resourceOutcomes.push(fail('scene/current', 'missing rootNodeIds array'))
|
||||
} else {
|
||||
const nodeCount = Object.keys(obj.nodes as Record<string, unknown>).length
|
||||
const rootCount = (obj.rootNodeIds as unknown[]).length
|
||||
resourceOutcomes.push(
|
||||
ok(
|
||||
'scene/current',
|
||||
`application/json, nodes=${nodeCount}, rootNodeIds=${rootCount}`,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
resourceOutcomes.push(
|
||||
fail('scene/current', `threw: ${err instanceof Error ? err.message : String(err)}`),
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------- Resource 2: pascal://scene/current/summary ----------------
|
||||
try {
|
||||
const result = await client.readResource({ uri: 'pascal://scene/current/summary' })
|
||||
const c = result.contents?.[0]
|
||||
if (!c) {
|
||||
resourceOutcomes.push(fail('scene/current/summary', 'no contents returned'))
|
||||
} else if (c.mimeType !== 'text/markdown') {
|
||||
resourceOutcomes.push(
|
||||
fail('scene/current/summary', `wrong mime type: ${String(c.mimeType)}`),
|
||||
)
|
||||
} else {
|
||||
const text = typeof c.text === 'string' ? c.text : ''
|
||||
const hasHeading = /^# /m.test(text)
|
||||
const hasZoneOrLevel = /level/i.test(text) || /zone/i.test(text)
|
||||
if (!hasHeading) {
|
||||
resourceOutcomes.push(
|
||||
fail('scene/current/summary', 'no markdown # heading found'),
|
||||
)
|
||||
} else if (!hasZoneOrLevel) {
|
||||
resourceOutcomes.push(
|
||||
fail('scene/current/summary', 'no level/zone references'),
|
||||
)
|
||||
} else {
|
||||
// Extract a few first lines as preview
|
||||
const preview = text.split('\n').slice(0, 4).join(' | ')
|
||||
resourceOutcomes.push(
|
||||
ok(
|
||||
'scene/current/summary',
|
||||
`text/markdown, ${text.length} bytes, preview: "${preview.slice(0, 200)}"`,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
resourceOutcomes.push(
|
||||
fail(
|
||||
'scene/current/summary',
|
||||
`threw: ${err instanceof Error ? err.message : String(err)}`,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------- Resource 3: pascal://catalog/items ----------------
|
||||
try {
|
||||
const result = await client.readResource({ uri: 'pascal://catalog/items' })
|
||||
const c = result.contents?.[0]
|
||||
if (!c) {
|
||||
resourceOutcomes.push(fail('catalog/items', 'no contents returned'))
|
||||
} else if (c.mimeType !== 'application/json') {
|
||||
resourceOutcomes.push(
|
||||
fail('catalog/items', `wrong mime type: ${String(c.mimeType)}`),
|
||||
)
|
||||
} else {
|
||||
const text = typeof c.text === 'string' ? c.text : ''
|
||||
const parsed = JSON.parse(text) as { status?: unknown; items?: unknown }
|
||||
if (parsed.status !== 'catalog_unavailable') {
|
||||
resourceOutcomes.push(
|
||||
fail('catalog/items', `expected status='catalog_unavailable' got ${String(parsed.status)}`),
|
||||
)
|
||||
} else {
|
||||
resourceOutcomes.push(
|
||||
ok(
|
||||
'catalog/items',
|
||||
`application/json, status=catalog_unavailable, items.length=${
|
||||
Array.isArray(parsed.items) ? parsed.items.length : 'N/A'
|
||||
}`,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
resourceOutcomes.push(
|
||||
fail('catalog/items', `threw: ${err instanceof Error ? err.message : String(err)}`),
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------- Resource 4: pascal://constraints/{levelId} ----------------
|
||||
// Discover levelId via find_nodes tool.
|
||||
let discoveredLevelId: string | null = null
|
||||
try {
|
||||
const findResult = await client.callTool({
|
||||
name: 'find_nodes',
|
||||
arguments: { type: 'level' },
|
||||
})
|
||||
const sc = (findResult as { structuredContent?: { nodes?: unknown[] } })
|
||||
.structuredContent
|
||||
const nodes = Array.isArray(sc?.nodes) ? sc.nodes : []
|
||||
if (nodes.length > 0) {
|
||||
const first = nodes[0] as { id?: string }
|
||||
if (typeof first?.id === 'string') {
|
||||
discoveredLevelId = first.id
|
||||
}
|
||||
}
|
||||
console.log(`[t5] discovered levelId = ${String(discoveredLevelId)}`)
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[t5] find_nodes threw: ${err instanceof Error ? err.message : String(err)}`,
|
||||
)
|
||||
}
|
||||
|
||||
if (!discoveredLevelId) {
|
||||
resourceOutcomes.push(
|
||||
fail('constraints/{levelId}', 'no level node discovered via find_nodes'),
|
||||
)
|
||||
} else {
|
||||
try {
|
||||
const uri = `pascal://constraints/${discoveredLevelId}`
|
||||
const result = await client.readResource({ uri })
|
||||
const c = result.contents?.[0]
|
||||
if (!c) {
|
||||
resourceOutcomes.push(fail('constraints/{levelId}', 'no contents returned'))
|
||||
} else if (c.mimeType !== 'application/json') {
|
||||
resourceOutcomes.push(
|
||||
fail('constraints/{levelId}', `wrong mime type: ${String(c.mimeType)}`),
|
||||
)
|
||||
} else {
|
||||
const text = typeof c.text === 'string' ? c.text : ''
|
||||
const parsed = JSON.parse(text) as {
|
||||
slabs?: unknown
|
||||
wallPolygons?: unknown
|
||||
error?: unknown
|
||||
}
|
||||
if (parsed.error) {
|
||||
resourceOutcomes.push(
|
||||
fail(
|
||||
'constraints/{levelId}',
|
||||
`error in payload: ${safeStringify(parsed.error)}`,
|
||||
),
|
||||
)
|
||||
} else if (!Array.isArray(parsed.slabs)) {
|
||||
resourceOutcomes.push(
|
||||
fail('constraints/{levelId}', 'missing slabs array'),
|
||||
)
|
||||
} else if (!Array.isArray(parsed.wallPolygons)) {
|
||||
resourceOutcomes.push(
|
||||
fail('constraints/{levelId}', 'missing wallPolygons array'),
|
||||
)
|
||||
} else {
|
||||
resourceOutcomes.push(
|
||||
ok(
|
||||
'constraints/{levelId}',
|
||||
`levelId=${discoveredLevelId}, slabs=${parsed.slabs.length}, wallPolygons=${parsed.wallPolygons.length}`,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
resourceOutcomes.push(
|
||||
fail(
|
||||
'constraints/{levelId}',
|
||||
`threw: ${err instanceof Error ? err.message : String(err)}`,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------- listPrompts ----------------
|
||||
try {
|
||||
const list = await client.listPrompts()
|
||||
listPromptsCount = Array.isArray(list.prompts) ? list.prompts.length : 0
|
||||
console.log(`[t5] listPrompts count = ${listPromptsCount}`)
|
||||
console.log(
|
||||
`[t5] listPrompts names = ${(list.prompts ?? [])
|
||||
.map((p) => p.name)
|
||||
.join(', ')}`,
|
||||
)
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[t5] listPrompts error: ${err instanceof Error ? err.message : String(err)}`,
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------- Prompt 1: from_brief ----------------
|
||||
try {
|
||||
const result = await client.getPrompt({
|
||||
name: 'from_brief',
|
||||
arguments: {
|
||||
brief: 'A small studio apartment',
|
||||
constraints: 'max 40 m^2',
|
||||
},
|
||||
})
|
||||
const messages = result.messages ?? []
|
||||
const userMsgs = messages.filter((m) => m.role === 'user')
|
||||
if (userMsgs.length === 0) {
|
||||
promptOutcomes.push(fail('from_brief', 'no user messages returned'))
|
||||
} else {
|
||||
const firstText =
|
||||
userMsgs[0]?.content && 'text' in userMsgs[0].content
|
||||
? String(userMsgs[0].content.text)
|
||||
: ''
|
||||
const mentionsBrief = /studio apartment/i.test(firstText)
|
||||
promptOutcomes.push(
|
||||
ok(
|
||||
'from_brief',
|
||||
`messages=${messages.length}, userMsgs=${userMsgs.length}, brief-included=${mentionsBrief}, preview: "${firstText.slice(0, 120).replace(/\n/g, ' ')}"`,
|
||||
),
|
||||
)
|
||||
}
|
||||
} catch (err) {
|
||||
promptOutcomes.push(
|
||||
fail('from_brief', `threw: ${err instanceof Error ? err.message : String(err)}`),
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------- Prompt 2: iterate_on_feedback ----------------
|
||||
try {
|
||||
const result = await client.getPrompt({
|
||||
name: 'iterate_on_feedback',
|
||||
arguments: { feedback: 'the kitchen is too small' },
|
||||
})
|
||||
const messages = result.messages ?? []
|
||||
const userMsgs = messages.filter((m) => m.role === 'user')
|
||||
if (userMsgs.length === 0) {
|
||||
promptOutcomes.push(fail('iterate_on_feedback', 'no user messages returned'))
|
||||
} else {
|
||||
const firstText =
|
||||
userMsgs[0]?.content && 'text' in userMsgs[0].content
|
||||
? String(userMsgs[0].content.text)
|
||||
: ''
|
||||
const mentionsFeedback = /kitchen is too small/i.test(firstText)
|
||||
promptOutcomes.push(
|
||||
ok(
|
||||
'iterate_on_feedback',
|
||||
`messages=${messages.length}, userMsgs=${userMsgs.length}, feedback-included=${mentionsFeedback}, preview: "${firstText.slice(0, 120).replace(/\n/g, ' ')}"`,
|
||||
),
|
||||
)
|
||||
}
|
||||
} catch (err) {
|
||||
promptOutcomes.push(
|
||||
fail(
|
||||
'iterate_on_feedback',
|
||||
`threw: ${err instanceof Error ? err.message : String(err)}`,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------- Prompt 3: renovation_from_photos ----------------
|
||||
try {
|
||||
const result = await client.getPrompt({
|
||||
name: 'renovation_from_photos',
|
||||
arguments: {
|
||||
currentPhotos: 'https://example.com/a.jpg,https://example.com/b.jpg',
|
||||
referencePhotos: 'https://example.com/c.jpg',
|
||||
goals: 'open-plan kitchen',
|
||||
},
|
||||
})
|
||||
const messages = result.messages ?? []
|
||||
const userMsgs = messages.filter((m) => m.role === 'user')
|
||||
if (userMsgs.length === 0) {
|
||||
promptOutcomes.push(
|
||||
fail('renovation_from_photos', 'no user messages returned'),
|
||||
)
|
||||
} else {
|
||||
// Look across all message content for the URLs we passed.
|
||||
const allText = messages
|
||||
.map((m) =>
|
||||
m.content && typeof m.content === 'object' && 'text' in m.content
|
||||
? String((m.content as { text?: unknown }).text ?? '')
|
||||
: '',
|
||||
)
|
||||
.join('\n')
|
||||
const hasAUrl = /example\.com\/a\.jpg/.test(allText)
|
||||
const hasBUrl = /example\.com\/b\.jpg/.test(allText)
|
||||
const hasCUrl = /example\.com\/c\.jpg/.test(allText)
|
||||
const hasGoals = /open-plan kitchen/i.test(allText)
|
||||
promptOutcomes.push(
|
||||
ok(
|
||||
'renovation_from_photos',
|
||||
`messages=${messages.length}, userMsgs=${userMsgs.length}, urls(a/b/c)=${hasAUrl}/${hasBUrl}/${hasCUrl}, goals-included=${hasGoals}`,
|
||||
),
|
||||
)
|
||||
}
|
||||
} catch (err) {
|
||||
promptOutcomes.push(
|
||||
fail(
|
||||
'renovation_from_photos',
|
||||
`threw: ${err instanceof Error ? err.message : String(err)}`,
|
||||
),
|
||||
)
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await client.close()
|
||||
} catch {
|
||||
// Ignore close errors.
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------- Print summary ----------------
|
||||
console.log('\n========== T5 SUMMARY ==========')
|
||||
console.log(`listResources count: ${listResourcesCount}`)
|
||||
console.log(`listPrompts count: ${listPromptsCount}`)
|
||||
|
||||
console.log('\n--- Resource results ---')
|
||||
let resPass = 0
|
||||
for (const o of resourceOutcomes) {
|
||||
console.log(`${o.pass ? 'PASS' : 'FAIL'} ${o.name} — ${o.detail}`)
|
||||
if (o.pass) resPass++
|
||||
}
|
||||
console.log(`Resources pass: ${resPass}/${resourceOutcomes.length}`)
|
||||
|
||||
console.log('\n--- Prompt results ---')
|
||||
let promPass = 0
|
||||
for (const o of promptOutcomes) {
|
||||
console.log(`${o.pass ? 'PASS' : 'FAIL'} ${o.name} — ${o.detail}`)
|
||||
if (o.pass) promPass++
|
||||
}
|
||||
console.log(`Prompts pass: ${promPass}/${promptOutcomes.length}`)
|
||||
|
||||
console.log('================================\n')
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('[t5] fatal:', err instanceof Error ? (err.stack ?? err.message) : err)
|
||||
process.exit(1)
|
||||
})
|
||||
Reference in New Issue
Block a user