feat(mcp): add multimodal vision tools via MCP sampling
analyze_floorplan_image and analyze_room_photo defer the vision work to the host via MCP sampling (server.server.createMessage). Validates host capability before calling, fetches URL inputs and base64-encodes them, constrains output to a Zod schema, and returns structured content. No vision model is bundled. 9 tests, all passing via a mocked sampling-capable client. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Adrian Perez
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
570c605446
commit
4dbfbb1e1a
@@ -0,0 +1,184 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
|
||||
import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'
|
||||
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
|
||||
import { CreateMessageRequestSchema } from '@modelcontextprotocol/sdk/types.js'
|
||||
import { SceneBridge } from '../../bridge/scene-bridge'
|
||||
import { registerAnalyzeFloorplanImage } from './analyze-floorplan-image'
|
||||
|
||||
type Handler = (req: unknown) => unknown | Promise<unknown>
|
||||
|
||||
/**
|
||||
* Build a connected client/server pair. Optionally advertises the `sampling`
|
||||
* capability on the client and installs a mock sampling handler that returns
|
||||
* a caller-provided reply.
|
||||
*/
|
||||
async function makeWiredPair(opts: {
|
||||
withSampling: boolean
|
||||
samplingHandler?: Handler
|
||||
}): Promise<{ client: Client; bridge: SceneBridge }> {
|
||||
const bridge = new SceneBridge()
|
||||
bridge.loadDefault()
|
||||
const server = new McpServer({ name: 'test', version: '0.0.0' })
|
||||
registerAnalyzeFloorplanImage(server, bridge)
|
||||
|
||||
const [srvT, cliT] = InMemoryTransport.createLinkedPair()
|
||||
|
||||
const client = new Client(
|
||||
{ name: 'test-client', version: '0.0.0' },
|
||||
{
|
||||
capabilities: opts.withSampling ? { sampling: {} } : {},
|
||||
},
|
||||
)
|
||||
|
||||
if (opts.withSampling && opts.samplingHandler) {
|
||||
const handler = opts.samplingHandler
|
||||
client.setRequestHandler(
|
||||
CreateMessageRequestSchema,
|
||||
async (request) =>
|
||||
// Cast to unknown — in tests we return arbitrary shapes to exercise
|
||||
// parse/validation paths in the tool handler.
|
||||
(await handler(request)) as never,
|
||||
)
|
||||
}
|
||||
|
||||
await Promise.all([server.connect(srvT), client.connect(cliT)])
|
||||
return { client, bridge }
|
||||
}
|
||||
|
||||
const VALID_REPLY = {
|
||||
model: 'mock-model',
|
||||
role: 'assistant',
|
||||
content: {
|
||||
type: 'text',
|
||||
text: JSON.stringify({
|
||||
walls: [
|
||||
{ start: [0, 0], end: [5, 0], thickness: 0.2 },
|
||||
{ start: [5, 0], end: [5, 4] },
|
||||
],
|
||||
rooms: [
|
||||
{
|
||||
name: 'Living Room',
|
||||
polygon: [
|
||||
[0, 0],
|
||||
[5, 0],
|
||||
[5, 4],
|
||||
[0, 4],
|
||||
],
|
||||
approximateAreaSqM: 20,
|
||||
},
|
||||
],
|
||||
approximateDimensions: { widthM: 5, depthM: 4 },
|
||||
confidence: 0.82,
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
describe('analyze_floorplan_image', () => {
|
||||
test('happy path: valid sampling JSON → structured output', async () => {
|
||||
const { client } = await makeWiredPair({
|
||||
withSampling: true,
|
||||
samplingHandler: () => VALID_REPLY,
|
||||
})
|
||||
const result = await client.callTool({
|
||||
name: 'analyze_floorplan_image',
|
||||
arguments: {
|
||||
image: 'aGVsbG8=', // raw base64 for "hello" — contents don't matter, mock ignores.
|
||||
scaleHint: '1 cm = 1 m',
|
||||
},
|
||||
})
|
||||
expect(result.isError).toBeFalsy()
|
||||
const structured = result.structuredContent as {
|
||||
walls: unknown[]
|
||||
rooms: unknown[]
|
||||
approximateDimensions: { widthM: number; depthM: number }
|
||||
confidence: number
|
||||
}
|
||||
expect(structured.walls.length).toBe(2)
|
||||
expect(structured.rooms[0]).toMatchObject({ name: 'Living Room' })
|
||||
expect(structured.approximateDimensions).toEqual({ widthM: 5, depthM: 4 })
|
||||
expect(structured.confidence).toBe(0.82)
|
||||
})
|
||||
|
||||
test('sampling unavailable → throws sampling_unavailable', async () => {
|
||||
const { client } = await makeWiredPair({ withSampling: false })
|
||||
const result = await client.callTool({
|
||||
name: 'analyze_floorplan_image',
|
||||
arguments: { image: 'aGVsbG8=' },
|
||||
})
|
||||
// The McpError thrown inside the tool handler is surfaced as a tool error.
|
||||
expect(result.isError).toBe(true)
|
||||
const text = (result.content as Array<{ type: string; text: string }>)[0]!.text
|
||||
expect(text).toContain('sampling_unavailable')
|
||||
})
|
||||
|
||||
test('sampling returns non-JSON text → sampling_response_unparseable', async () => {
|
||||
const { client } = await makeWiredPair({
|
||||
withSampling: true,
|
||||
samplingHandler: () => ({
|
||||
model: 'mock-model',
|
||||
role: 'assistant',
|
||||
content: { type: 'text', text: 'not json at all' },
|
||||
}),
|
||||
})
|
||||
const result = await client.callTool({
|
||||
name: 'analyze_floorplan_image',
|
||||
arguments: { image: 'aGVsbG8=' },
|
||||
})
|
||||
expect(result.isError).toBe(true)
|
||||
const text = (result.content as Array<{ type: string; text: string }>)[0]!.text
|
||||
expect(text).toContain('sampling_response_unparseable')
|
||||
})
|
||||
|
||||
test('sampling returns JSON that fails schema → sampling_response_invalid', async () => {
|
||||
const { client } = await makeWiredPair({
|
||||
withSampling: true,
|
||||
samplingHandler: () => ({
|
||||
model: 'mock-model',
|
||||
role: 'assistant',
|
||||
content: {
|
||||
type: 'text',
|
||||
text: JSON.stringify({
|
||||
// Missing required fields (no rooms, approximateDimensions, confidence).
|
||||
walls: [],
|
||||
}),
|
||||
},
|
||||
}),
|
||||
})
|
||||
const result = await client.callTool({
|
||||
name: 'analyze_floorplan_image',
|
||||
arguments: { image: 'aGVsbG8=' },
|
||||
})
|
||||
expect(result.isError).toBe(true)
|
||||
const text = (result.content as Array<{ type: string; text: string }>)[0]!.text
|
||||
expect(text).toContain('sampling_response_invalid')
|
||||
})
|
||||
|
||||
test('strips data URI prefix before base64 → still produces valid output', async () => {
|
||||
let capturedRequest: unknown
|
||||
const { client } = await makeWiredPair({
|
||||
withSampling: true,
|
||||
samplingHandler: (req) => {
|
||||
capturedRequest = req
|
||||
return VALID_REPLY
|
||||
},
|
||||
})
|
||||
await client.callTool({
|
||||
name: 'analyze_floorplan_image',
|
||||
arguments: {
|
||||
image: 'data:image/png;base64,aGVsbG8=',
|
||||
},
|
||||
})
|
||||
const params = (capturedRequest as { params: { messages: Array<{ content: unknown }> } }).params
|
||||
const content = params.messages[0]!.content as Array<{
|
||||
type: string
|
||||
data?: string
|
||||
mimeType?: string
|
||||
text?: string
|
||||
}>
|
||||
const img = content.find((b) => b.type === 'image')
|
||||
expect(img).toBeDefined()
|
||||
expect(img?.mimeType).toBe('image/png')
|
||||
expect(img?.data).toBe('aGVsbG8=')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,191 @@
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
|
||||
import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'
|
||||
import { z } from 'zod'
|
||||
import type { SceneBridge } from '../../bridge/scene-bridge'
|
||||
|
||||
/**
|
||||
* Input shape for `analyze_floorplan_image`.
|
||||
*
|
||||
* `image` is either a base64-encoded payload (optionally prefixed with a
|
||||
* `data:image/<mime>;base64,` URL) or an `http(s)` URL which we fetch and
|
||||
* inline as base64 before forwarding to the MCP host via sampling.
|
||||
*/
|
||||
export const analyzeFloorplanImageInput = {
|
||||
image: z.string().describe('Base64-encoded image or http(s) URL'),
|
||||
scaleHint: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe("Text hint about scale, e.g. '1 cm = 1 m' or 'approximately 80 m²'"),
|
||||
}
|
||||
|
||||
export const analyzeFloorplanImageOutput = {
|
||||
walls: z.array(
|
||||
z.object({
|
||||
start: z.tuple([z.number(), z.number()]),
|
||||
end: z.tuple([z.number(), z.number()]),
|
||||
thickness: z.number().optional(),
|
||||
}),
|
||||
),
|
||||
rooms: z.array(
|
||||
z.object({
|
||||
name: z.string(),
|
||||
polygon: z.array(z.tuple([z.number(), z.number()])),
|
||||
approximateAreaSqM: z.number().optional(),
|
||||
}),
|
||||
),
|
||||
approximateDimensions: z.object({
|
||||
widthM: z.number(),
|
||||
depthM: z.number(),
|
||||
}),
|
||||
confidence: z.number().min(0).max(1),
|
||||
}
|
||||
|
||||
const OutputSchema = z.object(analyzeFloorplanImageOutput)
|
||||
|
||||
const SYSTEM_PROMPT = `You are a vision assistant that extracts structured floor-plan data from an image.
|
||||
Your ONLY job: return a JSON object that exactly matches this schema — no prose, no markdown fences.
|
||||
|
||||
{
|
||||
"walls": [{ "start": [x, z], "end": [x, z], "thickness": number? }, ...],
|
||||
"rooms": [{ "name": string, "polygon": [[x,z], ...], "approximateAreaSqM": number? }, ...],
|
||||
"approximateDimensions": { "widthM": number, "depthM": number },
|
||||
"confidence": number 0..1
|
||||
}
|
||||
|
||||
Coordinates are in metres. Origin can be the floor plan's centre or bottom-left — be consistent.
|
||||
If the image is unclear, lower the confidence score but still produce your best attempt.
|
||||
DO NOT wrap the JSON in markdown. DO NOT explain. Just output the raw JSON.`
|
||||
|
||||
const DATA_URI_RE = /^data:(image\/[a-z0-9.+-]+);base64,(.+)$/i
|
||||
|
||||
type ImageBlock = {
|
||||
type: 'image'
|
||||
data: string
|
||||
mimeType: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the `image` input into a sampling-ready image block.
|
||||
*
|
||||
* - `http(s)://` URLs are fetched, base64-encoded, and the mime type sniffed
|
||||
* from the `content-type` response header.
|
||||
* - `data:image/*;base64,...` URIs are stripped of the prefix; mime type taken
|
||||
* from the URI itself.
|
||||
* - Otherwise we treat the string as raw base64 and default to `image/jpeg`.
|
||||
*/
|
||||
async function resolveImageBlock(image: string): Promise<ImageBlock> {
|
||||
if (/^https?:\/\//i.test(image)) {
|
||||
const res = await fetch(image)
|
||||
if (!res.ok) {
|
||||
throw new McpError(
|
||||
ErrorCode.InvalidParams,
|
||||
`failed to fetch image: ${res.status} ${res.statusText}`,
|
||||
{ url: image, status: res.status },
|
||||
)
|
||||
}
|
||||
const buf = Buffer.from(await res.arrayBuffer())
|
||||
const data = buf.toString('base64')
|
||||
const mimeType = res.headers.get('content-type') ?? 'image/jpeg'
|
||||
return { type: 'image', data, mimeType }
|
||||
}
|
||||
|
||||
const dataUriMatch = image.match(DATA_URI_RE)
|
||||
if (dataUriMatch) {
|
||||
return {
|
||||
type: 'image',
|
||||
mimeType: dataUriMatch[1]!,
|
||||
data: dataUriMatch[2]!,
|
||||
}
|
||||
}
|
||||
|
||||
return { type: 'image', mimeType: 'image/jpeg', data: image }
|
||||
}
|
||||
|
||||
/** Collect all text content blocks returned by the sampling host into one string. */
|
||||
function extractText(
|
||||
content:
|
||||
| { type: 'text'; text: string }
|
||||
| { type: 'image' | 'audio'; data: string; mimeType: string }
|
||||
| Array<
|
||||
| { type: 'text'; text: string }
|
||||
| { type: 'image' | 'audio'; data: string; mimeType: string }
|
||||
| { type: string; [k: string]: unknown }
|
||||
>,
|
||||
): string {
|
||||
const blocks = Array.isArray(content) ? content : [content]
|
||||
const texts: string[] = []
|
||||
for (const block of blocks) {
|
||||
if (block && typeof block === 'object' && (block as { type?: string }).type === 'text') {
|
||||
const t = (block as { text?: unknown }).text
|
||||
if (typeof t === 'string') texts.push(t)
|
||||
}
|
||||
}
|
||||
return texts.join('\n').trim()
|
||||
}
|
||||
|
||||
export function registerAnalyzeFloorplanImage(server: McpServer, _bridge: SceneBridge): void {
|
||||
server.registerTool(
|
||||
'analyze_floorplan_image',
|
||||
{
|
||||
title: 'Analyze floor-plan image',
|
||||
description:
|
||||
'Defer to the MCP host (via sampling) to extract walls, rooms, and approximate dimensions from a floor-plan image. Requires host support for sampling.',
|
||||
inputSchema: analyzeFloorplanImageInput,
|
||||
outputSchema: analyzeFloorplanImageOutput,
|
||||
},
|
||||
async ({ image, scaleHint }) => {
|
||||
const caps = server.server.getClientCapabilities()
|
||||
if (!caps?.sampling) {
|
||||
throw new McpError(ErrorCode.InvalidRequest, 'sampling_unavailable')
|
||||
}
|
||||
|
||||
const imageBlock = await resolveImageBlock(image)
|
||||
const instruction = scaleHint
|
||||
? `Analyze this floor plan. Scale hint: ${scaleHint}. Return ONLY the JSON described by the system prompt.`
|
||||
: 'Analyze this floor plan. Return ONLY the JSON described by the system prompt.'
|
||||
|
||||
const response = await server.server.createMessage({
|
||||
systemPrompt: SYSTEM_PROMPT,
|
||||
temperature: 0,
|
||||
maxTokens: 2000,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [imageBlock, { type: 'text', text: instruction }],
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const text = extractText(response.content as Parameters<typeof extractText>[0])
|
||||
if (!text) {
|
||||
throw new McpError(ErrorCode.InternalError, 'sampling_response_unparseable', {
|
||||
reason: 'no text content returned by host',
|
||||
})
|
||||
}
|
||||
|
||||
let parsed: unknown
|
||||
try {
|
||||
parsed = JSON.parse(text)
|
||||
} catch (err) {
|
||||
throw new McpError(ErrorCode.InternalError, 'sampling_response_unparseable', {
|
||||
raw: text,
|
||||
reason: err instanceof Error ? err.message : String(err),
|
||||
})
|
||||
}
|
||||
|
||||
const validation = OutputSchema.safeParse(parsed)
|
||||
if (!validation.success) {
|
||||
throw new McpError(ErrorCode.InternalError, 'sampling_response_invalid', {
|
||||
raw: text,
|
||||
errors: validation.error.issues,
|
||||
})
|
||||
}
|
||||
|
||||
const payload = validation.data
|
||||
return {
|
||||
content: [{ type: 'text' as const, text: JSON.stringify(payload) }],
|
||||
structuredContent: payload,
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
|
||||
import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'
|
||||
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
|
||||
import { CreateMessageRequestSchema } from '@modelcontextprotocol/sdk/types.js'
|
||||
import { SceneBridge } from '../../bridge/scene-bridge'
|
||||
import { registerAnalyzeRoomPhoto } from './analyze-room-photo'
|
||||
|
||||
type Handler = (req: unknown) => unknown | Promise<unknown>
|
||||
|
||||
async function makeWiredPair(opts: {
|
||||
withSampling: boolean
|
||||
samplingHandler?: Handler
|
||||
}): Promise<{ client: Client; bridge: SceneBridge }> {
|
||||
const bridge = new SceneBridge()
|
||||
bridge.loadDefault()
|
||||
const server = new McpServer({ name: 'test', version: '0.0.0' })
|
||||
registerAnalyzeRoomPhoto(server, bridge)
|
||||
|
||||
const [srvT, cliT] = InMemoryTransport.createLinkedPair()
|
||||
|
||||
const client = new Client(
|
||||
{ name: 'test-client', version: '0.0.0' },
|
||||
{
|
||||
capabilities: opts.withSampling ? { sampling: {} } : {},
|
||||
},
|
||||
)
|
||||
|
||||
if (opts.withSampling && opts.samplingHandler) {
|
||||
const handler = opts.samplingHandler
|
||||
client.setRequestHandler(
|
||||
CreateMessageRequestSchema,
|
||||
async (request) => (await handler(request)) as never,
|
||||
)
|
||||
}
|
||||
|
||||
await Promise.all([server.connect(srvT), client.connect(cliT)])
|
||||
return { client, bridge }
|
||||
}
|
||||
|
||||
const VALID_REPLY = {
|
||||
model: 'mock-model',
|
||||
role: 'assistant',
|
||||
content: {
|
||||
type: 'text',
|
||||
text: JSON.stringify({
|
||||
approximateDimensions: { widthM: 4.2, lengthM: 5.8, heightM: 2.7 },
|
||||
identifiedFixtures: [
|
||||
{ type: 'sofa', approximatePosition: [1.5, 2.0] },
|
||||
{ type: 'coffee table' },
|
||||
],
|
||||
identifiedWindows: [{ wallLabel: 'north', approximateWidthM: 1.2, approximateHeightM: 1.4 }],
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
describe('analyze_room_photo', () => {
|
||||
test('happy path: valid sampling JSON → structured output', async () => {
|
||||
const { client } = await makeWiredPair({
|
||||
withSampling: true,
|
||||
samplingHandler: () => VALID_REPLY,
|
||||
})
|
||||
const result = await client.callTool({
|
||||
name: 'analyze_room_photo',
|
||||
arguments: { image: 'aGVsbG8=' },
|
||||
})
|
||||
expect(result.isError).toBeFalsy()
|
||||
const structured = result.structuredContent as {
|
||||
approximateDimensions: { widthM: number; lengthM: number; heightM?: number }
|
||||
identifiedFixtures: Array<{ type: string; approximatePosition?: [number, number] }>
|
||||
identifiedWindows: Array<{
|
||||
wallLabel?: string
|
||||
approximateWidthM?: number
|
||||
approximateHeightM?: number
|
||||
}>
|
||||
}
|
||||
expect(structured.approximateDimensions.widthM).toBe(4.2)
|
||||
expect(structured.approximateDimensions.lengthM).toBe(5.8)
|
||||
expect(structured.identifiedFixtures.length).toBe(2)
|
||||
expect(structured.identifiedFixtures[0]!.type).toBe('sofa')
|
||||
expect(structured.identifiedWindows[0]!.wallLabel).toBe('north')
|
||||
})
|
||||
|
||||
test('sampling unavailable → throws sampling_unavailable', async () => {
|
||||
const { client } = await makeWiredPair({ withSampling: false })
|
||||
const result = await client.callTool({
|
||||
name: 'analyze_room_photo',
|
||||
arguments: { image: 'aGVsbG8=' },
|
||||
})
|
||||
expect(result.isError).toBe(true)
|
||||
const text = (result.content as Array<{ type: string; text: string }>)[0]!.text
|
||||
expect(text).toContain('sampling_unavailable')
|
||||
})
|
||||
|
||||
test('sampling returns non-JSON text → sampling_response_unparseable', async () => {
|
||||
const { client } = await makeWiredPair({
|
||||
withSampling: true,
|
||||
samplingHandler: () => ({
|
||||
model: 'mock-model',
|
||||
role: 'assistant',
|
||||
content: { type: 'text', text: '{ not json' },
|
||||
}),
|
||||
})
|
||||
const result = await client.callTool({
|
||||
name: 'analyze_room_photo',
|
||||
arguments: { image: 'aGVsbG8=' },
|
||||
})
|
||||
expect(result.isError).toBe(true)
|
||||
const text = (result.content as Array<{ type: string; text: string }>)[0]!.text
|
||||
expect(text).toContain('sampling_response_unparseable')
|
||||
})
|
||||
|
||||
test('sampling returns JSON that fails schema → sampling_response_invalid', async () => {
|
||||
const { client } = await makeWiredPair({
|
||||
withSampling: true,
|
||||
samplingHandler: () => ({
|
||||
model: 'mock-model',
|
||||
role: 'assistant',
|
||||
content: {
|
||||
type: 'text',
|
||||
text: JSON.stringify({
|
||||
// approximateDimensions missing required widthM/lengthM.
|
||||
approximateDimensions: {},
|
||||
identifiedFixtures: [],
|
||||
identifiedWindows: [],
|
||||
}),
|
||||
},
|
||||
}),
|
||||
})
|
||||
const result = await client.callTool({
|
||||
name: 'analyze_room_photo',
|
||||
arguments: { image: 'aGVsbG8=' },
|
||||
})
|
||||
expect(result.isError).toBe(true)
|
||||
const text = (result.content as Array<{ type: string; text: string }>)[0]!.text
|
||||
expect(text).toContain('sampling_response_invalid')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,176 @@
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
|
||||
import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'
|
||||
import { z } from 'zod'
|
||||
import type { SceneBridge } from '../../bridge/scene-bridge'
|
||||
|
||||
/**
|
||||
* Input shape for `analyze_room_photo`.
|
||||
*
|
||||
* Same image resolution rules as `analyze_floorplan_image`.
|
||||
*/
|
||||
export const analyzeRoomPhotoInput = {
|
||||
image: z.string().describe('Base64-encoded image or http(s) URL'),
|
||||
}
|
||||
|
||||
export const analyzeRoomPhotoOutput = {
|
||||
approximateDimensions: z.object({
|
||||
widthM: z.number(),
|
||||
lengthM: z.number(),
|
||||
heightM: z.number().optional(),
|
||||
}),
|
||||
identifiedFixtures: z.array(
|
||||
z.object({
|
||||
type: z.string(),
|
||||
approximatePosition: z.tuple([z.number(), z.number()]).optional(),
|
||||
}),
|
||||
),
|
||||
identifiedWindows: z.array(
|
||||
z.object({
|
||||
wallLabel: z.string().optional(),
|
||||
approximateWidthM: z.number().optional(),
|
||||
approximateHeightM: z.number().optional(),
|
||||
}),
|
||||
),
|
||||
}
|
||||
|
||||
const OutputSchema = z.object(analyzeRoomPhotoOutput)
|
||||
|
||||
const SYSTEM_PROMPT = `You are a vision assistant that extracts structured room data from a single photograph.
|
||||
Your ONLY job: return a JSON object that exactly matches this schema — no prose, no markdown fences.
|
||||
|
||||
{
|
||||
"approximateDimensions": { "widthM": number, "lengthM": number, "heightM": number? },
|
||||
"identifiedFixtures": [{ "type": string, "approximatePosition": [x, z]? }, ...],
|
||||
"identifiedWindows": [{ "wallLabel": string?, "approximateWidthM": number?, "approximateHeightM": number? }, ...]
|
||||
}
|
||||
|
||||
All measurements are in metres. "type" for fixtures is a short noun phrase such as "sofa", "kitchen island", "door".
|
||||
If measurements cannot be estimated confidently, omit the optional fields rather than guessing.
|
||||
DO NOT wrap the JSON in markdown. DO NOT explain. Just output the raw JSON.`
|
||||
|
||||
const DATA_URI_RE = /^data:(image\/[a-z0-9.+-]+);base64,(.+)$/i
|
||||
|
||||
type ImageBlock = {
|
||||
type: 'image'
|
||||
data: string
|
||||
mimeType: string
|
||||
}
|
||||
|
||||
async function resolveImageBlock(image: string): Promise<ImageBlock> {
|
||||
if (/^https?:\/\//i.test(image)) {
|
||||
const res = await fetch(image)
|
||||
if (!res.ok) {
|
||||
throw new McpError(
|
||||
ErrorCode.InvalidParams,
|
||||
`failed to fetch image: ${res.status} ${res.statusText}`,
|
||||
{ url: image, status: res.status },
|
||||
)
|
||||
}
|
||||
const buf = Buffer.from(await res.arrayBuffer())
|
||||
const data = buf.toString('base64')
|
||||
const mimeType = res.headers.get('content-type') ?? 'image/jpeg'
|
||||
return { type: 'image', data, mimeType }
|
||||
}
|
||||
|
||||
const dataUriMatch = image.match(DATA_URI_RE)
|
||||
if (dataUriMatch) {
|
||||
return {
|
||||
type: 'image',
|
||||
mimeType: dataUriMatch[1]!,
|
||||
data: dataUriMatch[2]!,
|
||||
}
|
||||
}
|
||||
|
||||
return { type: 'image', mimeType: 'image/jpeg', data: image }
|
||||
}
|
||||
|
||||
function extractText(
|
||||
content:
|
||||
| { type: 'text'; text: string }
|
||||
| { type: 'image' | 'audio'; data: string; mimeType: string }
|
||||
| Array<
|
||||
| { type: 'text'; text: string }
|
||||
| { type: 'image' | 'audio'; data: string; mimeType: string }
|
||||
| { type: string; [k: string]: unknown }
|
||||
>,
|
||||
): string {
|
||||
const blocks = Array.isArray(content) ? content : [content]
|
||||
const texts: string[] = []
|
||||
for (const block of blocks) {
|
||||
if (block && typeof block === 'object' && (block as { type?: string }).type === 'text') {
|
||||
const t = (block as { text?: unknown }).text
|
||||
if (typeof t === 'string') texts.push(t)
|
||||
}
|
||||
}
|
||||
return texts.join('\n').trim()
|
||||
}
|
||||
|
||||
export function registerAnalyzeRoomPhoto(server: McpServer, _bridge: SceneBridge): void {
|
||||
server.registerTool(
|
||||
'analyze_room_photo',
|
||||
{
|
||||
title: 'Analyze room photo',
|
||||
description:
|
||||
'Defer to the MCP host (via sampling) to extract approximate dimensions, fixtures, and windows from a single-room photograph. Requires host support for sampling.',
|
||||
inputSchema: analyzeRoomPhotoInput,
|
||||
outputSchema: analyzeRoomPhotoOutput,
|
||||
},
|
||||
async ({ image }) => {
|
||||
const caps = server.server.getClientCapabilities()
|
||||
if (!caps?.sampling) {
|
||||
throw new McpError(ErrorCode.InvalidRequest, 'sampling_unavailable')
|
||||
}
|
||||
|
||||
const imageBlock = await resolveImageBlock(image)
|
||||
|
||||
const response = await server.server.createMessage({
|
||||
systemPrompt: SYSTEM_PROMPT,
|
||||
temperature: 0,
|
||||
maxTokens: 2000,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
imageBlock,
|
||||
{
|
||||
type: 'text',
|
||||
text: 'Analyze this room photo. Return ONLY the JSON described by the system prompt.',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const text = extractText(response.content as Parameters<typeof extractText>[0])
|
||||
if (!text) {
|
||||
throw new McpError(ErrorCode.InternalError, 'sampling_response_unparseable', {
|
||||
reason: 'no text content returned by host',
|
||||
})
|
||||
}
|
||||
|
||||
let parsed: unknown
|
||||
try {
|
||||
parsed = JSON.parse(text)
|
||||
} catch (err) {
|
||||
throw new McpError(ErrorCode.InternalError, 'sampling_response_unparseable', {
|
||||
raw: text,
|
||||
reason: err instanceof Error ? err.message : String(err),
|
||||
})
|
||||
}
|
||||
|
||||
const validation = OutputSchema.safeParse(parsed)
|
||||
if (!validation.success) {
|
||||
throw new McpError(ErrorCode.InternalError, 'sampling_response_invalid', {
|
||||
raw: text,
|
||||
errors: validation.error.issues,
|
||||
})
|
||||
}
|
||||
|
||||
const payload = validation.data
|
||||
return {
|
||||
content: [{ type: 'text' as const, text: JSON.stringify(payload) }],
|
||||
structuredContent: payload,
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
|
||||
import type { SceneBridge } from '../../bridge/scene-bridge'
|
||||
import { registerAnalyzeFloorplanImage } from './analyze-floorplan-image'
|
||||
import { registerAnalyzeRoomPhoto } from './analyze-room-photo'
|
||||
|
||||
/**
|
||||
* Register the vision-input tools that defer to the MCP host's sampling
|
||||
* capability. No vision model is bundled in this package — if the host does
|
||||
* not advertise `sampling` support, calling either tool returns
|
||||
* `sampling_unavailable`.
|
||||
*/
|
||||
export function registerVisionTools(server: McpServer, bridge: SceneBridge): void {
|
||||
registerAnalyzeFloorplanImage(server, bridge)
|
||||
registerAnalyzeRoomPhoto(server, bridge)
|
||||
}
|
||||
|
||||
export {
|
||||
analyzeFloorplanImageInput,
|
||||
analyzeFloorplanImageOutput,
|
||||
registerAnalyzeFloorplanImage,
|
||||
} from './analyze-floorplan-image'
|
||||
export {
|
||||
analyzeRoomPhotoInput,
|
||||
analyzeRoomPhotoOutput,
|
||||
registerAnalyzeRoomPhoto,
|
||||
} from './analyze-room-photo'
|
||||
Reference in New Issue
Block a user