scans and ref image

packages/core/package.json

@@ -24,6 +24,7 @@
   },
   "dependencies": {
     "dedent": "^1.7.1",
+    "idb-keyval": "^6.2.2",
     "mitt": "^3.0.1",
     "nanoid": "^5.1.6",
     "zod": "^4.3.5",

packages/core/src/hooks/scene-registry/scene-registry.ts

@@ -15,6 +15,8 @@ export const sceneRegistry = {
     item: new Set<string>(),
     slab: new Set<string>(),
     zone: new Set<string>(),
+    scan: new Set<string>(),
+    guide: new Set<string>(),
   },
 };
 

packages/core/src/index.ts

@@ -34,3 +34,5 @@ export { SlabSystem } from './systems/slab/slab-system'
 export { WallSystem } from './systems/wall/wall-system'
 
 export { isObject } from './utils/types'
+// Asset storage
+export { saveAsset, loadAssetUrl } from './lib/asset-storage'

packages/core/src/lib/asset-storage.ts

@@ -0,0 +1,55 @@
+import { get, set } from 'idb-keyval'
+
+export const ASSET_PREFIX = 'asset_data:'
+
+// Cache for active object URLs to prevent leaks and flickering
+const urlCache = new Map<string, string>()
+
+/**
+ * Save a file to IndexedDB and return a custom protocol URL
+ */
+export async function saveAsset(file: File): Promise<string> {
+  const id = crypto.randomUUID()
+  await set(`${ASSET_PREFIX}${id}`, file)
+  return `asset://${id}`
+}
+
+/**
+ * Load a file from IndexedDB and return an object URL
+ * If the URL is not a custom protocol URL, return it as is
+ */
+export async function loadAssetUrl(url: string): Promise<string | null> {
+  if (!url) return null
+
+  // If it's already a blob or http URL, return as is
+  if (url.startsWith('blob:') || url.startsWith('http')) {
+    return url
+  }
+
+  // Handle our custom asset protocol
+  if (url.startsWith('asset://')) {
+    const id = url.replace('asset://', '')
+
+    // Check cache first
+    if (urlCache.has(id)) {
+      return urlCache.get(id)!
+    }
+
+    try {
+      const file = await get<File | Blob>(`${ASSET_PREFIX}${id}`)
+      if (!file) {
+        console.warn(`Asset not found: ${id}`)
+        return null
+      }
+      const objectUrl = URL.createObjectURL(file)
+      urlCache.set(id, objectUrl)
+      return objectUrl
+    } catch (error) {
+      console.error('Failed to load asset:', error)
+      return null
+    }
+  }
+
+  // Legacy data URLs are returned as is
+  return url
+}

packages/core/src/schema/index.ts

@@ -13,6 +13,8 @@ export { BuildingNode } from './nodes/building'
 export { CeilingNode } from './nodes/ceiling'
 
 export { ZoneNode } from './nodes/zone'
+export { ScanNode } from './nodes/scan'
+export { GuideNode } from './nodes/guide'
 export type { AnyNodeId, AnyNodeType } from './types'
 // Union types
 export { AnyNode } from './types'

packages/core/src/schema/nodes/guide.ts

@@ -0,0 +1,14 @@
+import { z } from 'zod'
+import { BaseNode, nodeType, objectId } from '../base'
+
+export const GuideNode = BaseNode.extend({
+  id: objectId('guide'),
+  type: nodeType('guide'),
+  url: z.string(),
+  position: z.tuple([z.number(), z.number(), z.number()]).default([0, 0, 0]),
+  rotation: z.tuple([z.number(), z.number(), z.number()]).default([0, 0, 0]),
+  scale: z.number().default(1),
+  opacity: z.number().min(0).max(100).default(50),
+})
+
+export type GuideNode = z.infer<typeof GuideNode>

packages/core/src/schema/nodes/level.ts

@@ -2,6 +2,8 @@ import dedent from 'dedent'
 import { z } from 'zod'
 import { BaseNode, nodeType, objectId } from '../base'
 import { CeilingNode } from './ceiling'
+import { GuideNode } from './guide'
+import { ScanNode } from './scan'
 import { SlabNode } from './slab'
 import { WallNode } from './wall'
 import { ZoneNode } from './zone'
@@ -9,7 +11,7 @@ import { ZoneNode } from './zone'
 export const LevelNode = BaseNode.extend({
   id: objectId('level'),
   type: nodeType('level'),
-  children: z.array(z.union([WallNode.shape.id, ZoneNode.shape.id, SlabNode.shape.id, CeilingNode.shape.id])).default([]),
+  children: z.array(z.union([WallNode.shape.id, ZoneNode.shape.id, SlabNode.shape.id, CeilingNode.shape.id, ScanNode.shape.id, GuideNode.shape.id])).default([]),
   // Specific props
   level: z.number().default(0),
 }).describe(

packages/core/src/schema/nodes/scan.ts

@@ -0,0 +1,14 @@
+import { z } from 'zod'
+import { BaseNode, nodeType, objectId } from '../base'
+
+export const ScanNode = BaseNode.extend({
+  id: objectId('scan'),
+  type: nodeType('scan'),
+  url: z.string(),
+  position: z.tuple([z.number(), z.number(), z.number()]).default([0, 0, 0]),
+  rotation: z.tuple([z.number(), z.number(), z.number()]).default([0, 0, 0]),
+  scale: z.number().default(1),
+  opacity: z.number().min(0).max(100).default(100),
+})
+
+export type ScanNode = z.infer<typeof ScanNode>

packages/core/src/schema/types.ts

@@ -1,8 +1,10 @@
 import z from 'zod'
 import { BuildingNode } from './nodes/building'
 import { CeilingNode } from './nodes/ceiling'
+import { GuideNode } from './nodes/guide'
 import { ItemNode } from './nodes/item'
 import { LevelNode } from './nodes/level'
+import { ScanNode } from './nodes/scan'
 import { SiteNode } from './nodes/site'
 import { SlabNode } from './nodes/slab'
 import { WallNode } from './nodes/wall'
@@ -17,6 +19,8 @@ export const AnyNode = z.discriminatedUnion('type', [
   ZoneNode,
   SlabNode,
   CeilingNode,
+  ScanNode,
+  GuideNode,
 ])
 
 export type AnyNode = z.infer<typeof AnyNode>

packages/viewer/src/components/renderers/guide/guide-renderer.tsx

@@ -0,0 +1,61 @@
+import { type GuideNode, useRegistry } from '@pascal-app/core'
+import { Suspense, useMemo, useRef } from 'react'
+import { DoubleSide, type Group, type Texture, TextureLoader } from 'three'
+import { float, texture } from 'three/tsl'
+import { MeshBasicNodeMaterial } from 'three/webgpu'
+import { useLoader } from '@react-three/fiber'
+import { useAssetUrl } from '../../../hooks/use-asset-url'
+
+export const GuideRenderer = ({ node }: { node: GuideNode }) => {
+  const ref = useRef<Group>(null!)
+  useRegistry(node.id, 'guide', ref)
+
+  const resolvedUrl = useAssetUrl(node.url)
+
+  return (
+    <group
+      ref={ref}
+      position={node.position}
+      rotation={[0, node.rotation[1], 0]}
+    >
+      {resolvedUrl && (
+        <Suspense>
+          <GuidePlane url={resolvedUrl} scale={node.scale} opacity={node.opacity} />
+        </Suspense>
+      )}
+    </group>
+  )
+}
+
+const GuidePlane = ({ url, scale, opacity }: { url: string; scale: number; opacity: number }) => {
+  const tex = useLoader(TextureLoader, url) as Texture
+
+  const { width, height, material } = useMemo(() => {
+    const img = tex.image as HTMLImageElement | ImageBitmap
+    const w = img.width || 1
+    const h = img.height || 1
+    const aspect = w / h
+
+    // Default: 10 meters wide, height from aspect ratio
+    const planeWidth = 10 * scale
+    const planeHeight = (10 / aspect) * scale
+
+    const normalizedOpacity = opacity / 100
+
+    const mat = new MeshBasicNodeMaterial({
+      transparent: true,
+      colorNode: texture(tex),
+      opacityNode: float(normalizedOpacity),
+      side: DoubleSide,
+      depthWrite: false,
+    })
+
+    return { width: planeWidth, height: planeHeight, material: mat }
+  }, [tex, scale, opacity])
+
+  return (
+    <mesh rotation={[-Math.PI / 2, 0, 0]} material={material}>
+      <planeGeometry args={[width, height]} />
+    </mesh>
+  )
+}

packages/viewer/src/components/renderers/node-renderer.tsx

@@ -3,8 +3,10 @@
 import { type AnyNode, useScene } from '@pascal-app/core'
 import { BuildingRenderer } from './building/building-renderer'
 import { CeilingRenderer } from './ceiling/ceiling-renderer'
+import { GuideRenderer } from './guide/guide-renderer'
 import { ItemRenderer } from './item/item-renderer'
 import { LevelRenderer } from './level/level-renderer'
+import { ScanRenderer } from './scan/scan-renderer'
 import { SlabRenderer } from './slab/slab-renderer'
 import { WallRenderer } from './wall/wall-renderer'
 import { ZoneRenderer } from './zone/zone-renderer'
@@ -23,6 +25,8 @@ export const NodeRenderer = ({ nodeId }: { nodeId: AnyNode['id'] }) => {
       {node.type === 'slab' && <SlabRenderer node={node} />}
       {node.type === 'wall' && <WallRenderer node={node} />}
       {node.type === 'zone' && <ZoneRenderer node={node} />}
+      {node.type === 'scan' && <ScanRenderer node={node} />}
+      {node.type === 'guide' && <GuideRenderer node={node} />}
     </>
   )
 }

packages/viewer/src/components/renderers/scan/scan-renderer.tsx

@@ -0,0 +1,64 @@
+import { type ScanNode, useRegistry } from '@pascal-app/core'
+import { Clone } from '@react-three/drei/core/Clone'
+import { Suspense, useMemo, useRef } from 'react'
+import type { Group, Material, Mesh } from 'three'
+import { useAssetUrl } from '../../../hooks/use-asset-url'
+import { useGLTFKTX2 } from '../../../hooks/use-gltf-ktx2'
+
+export const ScanRenderer = ({ node }: { node: ScanNode }) => {
+  const ref = useRef<Group>(null!)
+  useRegistry(node.id, 'scan', ref)
+
+  const resolvedUrl = useAssetUrl(node.url)
+
+  return (
+    <group
+      ref={ref}
+      position={node.position}
+      rotation={node.rotation}
+      scale={[node.scale, node.scale, node.scale]}
+    >
+      {resolvedUrl && (
+        <Suspense>
+          <ScanModel url={resolvedUrl} opacity={node.opacity} />
+        </Suspense>
+      )}
+    </group>
+  )
+}
+
+const ScanModel = ({ url, opacity }: { url: string; opacity: number }) => {
+  const { scene } = useGLTFKTX2(url)
+
+  useMemo(() => {
+    const normalizedOpacity = opacity / 100
+    const isTransparent = normalizedOpacity < 1
+
+    const updateMaterial = (material: Material) => {
+      if (isTransparent) {
+        material.transparent = true
+        material.opacity = normalizedOpacity
+      } else {
+        material.transparent = false
+        material.opacity = 1
+      }
+      material.needsUpdate = true
+    }
+
+    scene.traverse((child) => {
+      if ((child as Mesh).isMesh) {
+        const mesh = child as Mesh
+
+        if (Array.isArray(mesh.material)) {
+          mesh.material.forEach((material) => {
+            updateMaterial(material)
+          })
+        } else {
+          updateMaterial(mesh.material)
+        }
+      }
+    })
+  }, [scene, opacity])
+
+  return <Clone object={scene} />
+}

packages/viewer/src/hooks/use-asset-url.ts

@@ -0,0 +1,23 @@
+import { loadAssetUrl } from '@pascal-app/core'
+import { useEffect, useState } from 'react'
+
+/**
+ * Resolves an asset:// URL to a blob URL for use with Three.js loaders.
+ * Returns null while loading or if resolution fails.
+ */
+export function useAssetUrl(url: string): string | null {
+  const [resolved, setResolved] = useState<string | null>(null)
+
+  useEffect(() => {
+    let cancelled = false
+    setResolved(null)
+    loadAssetUrl(url).then((result) => {
+      if (!cancelled) setResolved(result)
+    })
+    return () => {
+      cancelled = true
+    }
+  }, [url])
+
+  return resolved
+}

packages/viewer/src/hooks/use-gltf-ktx2.tsx

@@ -0,0 +1,19 @@
+import { useGLTF } from "@react-three/drei"
+import { useThree } from "@react-three/fiber"
+import { KTX2Loader } from "three/examples/jsm/Addons.js"
+import { MeshoptDecoder } from "three/examples/jsm/libs/meshopt_decoder.module.js"
+
+const ktx2LoaderInstance = new KTX2Loader()
+ktx2LoaderInstance.setTranscoderPath('https://cdn.jsdelivr.net/gh/pmndrs/drei-assets@master/basis/')
+
+const useGLTFKTX2 = (path: string) => {
+  const gl = useThree((state) => state.gl)
+
+  return useGLTF(path, true, true, (loader) => {
+    ktx2LoaderInstance.detectSupport(gl)
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    loader.setKTX2Loader(ktx2LoaderInstance as any)
+    loader.setMeshoptDecoder(MeshoptDecoder)
+  })
+}
+export { useGLTFKTX2 }