From a7aa385d98ecdc790bf64c9dc088f3918268d612 Mon Sep 17 00:00:00 2001 From: Michael Mainguy Date: Sat, 20 Dec 2025 16:14:44 -0600 Subject: [PATCH] Add model selection and ground-projected directional placement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model management: - Add list_models, get_current_model, set_model tools - Support Claude Sonnet 4, Opus 4, and Haiku 3.5 - Model selection persists for session duration Directional placement improvements: - Compute ground-projected forward/right vectors from camera - Accounts for camera being parented to moving platform - "Forward" means forward on ground plane, ignoring vertical look angle - Pre-calculate example positions for left/right/forward/back - Update system prompt to use get_camera_position for relative directions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- src/diagram/diagramManager.ts | 28 +++++- src/react/services/diagramAI.ts | 145 ++++++++++++++++++++++++++++- src/react/services/entityBridge.ts | 46 +++++++-- src/react/types/chatTypes.ts | 9 +- 4 files changed, 213 insertions(+), 15 deletions(-) diff --git a/src/diagram/diagramManager.ts b/src/diagram/diagramManager.ts index da097cb..337fc5e 100644 --- a/src/diagram/diagramManager.ts +++ b/src/diagram/diagramManager.ts @@ -1,4 +1,4 @@ -import {AbstractActionManager, AbstractMesh, ActionManager, Observable, Scene, WebXRDefaultExperience} from "@babylonjs/core"; +import {AbstractActionManager, AbstractMesh, ActionManager, Observable, Scene, Vector3, WebXRDefaultExperience} from "@babylonjs/core"; import {DiagramEntity, DiagramEntityType, DiagramEvent, DiagramEventType} from "./types/diagramEntity"; import log from "loglevel"; @@ -220,6 +220,7 @@ export class DiagramManager { }); // Get current camera position and orientation + // Camera may be parented to a platform, so we use world-space coordinates document.addEventListener('chatGetCamera', () => { this._logger.debug('chatGetCamera'); const camera = this._scene.activeCamera; @@ -228,15 +229,36 @@ export class DiagramManager { return; } + // World-space position (accounts for parent transforms) const position = camera.globalPosition; + + // World-space forward direction (where camera is looking) const forward = camera.getForwardRay(1).direction; - const up = camera.upVector || {x: 0, y: 1, z: 0}; + + // World up vector + const worldUp = new Vector3(0, 1, 0); + + // Compute ground-projected forward (for intuitive forward/back movement) + // This ignores pitch so looking up/down doesn't affect horizontal movement + const groundForward = new Vector3(forward.x, 0, forward.z); + const groundForwardLength = groundForward.length(); + if (groundForwardLength > 0.001) { + groundForward.scaleInPlace(1 / groundForwardLength); + } else { + // Looking straight up/down - use a fallback forward + groundForward.set(0, 0, -1); + } + + // Compute right vector (perpendicular to groundForward in XZ plane) + // Right = Cross(groundForward, worldUp) gives left, so we negate or swap + const groundRight = Vector3.Cross(worldUp, groundForward).normalize(); const responseEvent = new CustomEvent('chatGetCameraResponse', { detail: { position: {x: position.x, y: position.y, z: position.z}, forward: {x: forward.x, y: forward.y, z: forward.z}, - up: {x: up.x, y: up.y, z: up.z} + groundForward: {x: groundForward.x, y: groundForward.y, z: groundForward.z}, + groundRight: {x: groundRight.x, y: groundRight.y, z: groundRight.z} }, bubbles: true }); diff --git a/src/react/services/diagramAI.ts b/src/react/services/diagramAI.ts index d0587ef..6359f09 100644 --- a/src/react/services/diagramAI.ts +++ b/src/react/services/diagramAI.ts @@ -9,6 +9,61 @@ logger.setLevel('debug'); // Session management let currentSessionId: string | null = null; +// Model management +export interface ModelInfo { + id: string; + name: string; + description: string; +} + +const AVAILABLE_MODELS: ModelInfo[] = [ + { + id: 'claude-sonnet-4-20250514', + name: 'Claude Sonnet 4', + description: 'Balanced performance and speed (default)' + }, + { + id: 'claude-opus-4-20250514', + name: 'Claude Opus 4', + description: 'Most capable, best for complex tasks' + }, + { + id: 'claude-haiku-3-5-20241022', + name: 'Claude Haiku 3.5', + description: 'Fastest responses, good for simple tasks' + } +]; + +let currentModelId: string = 'claude-sonnet-4-20250514'; + +/** + * Get available models + */ +export function getAvailableModels(): ModelInfo[] { + return [...AVAILABLE_MODELS]; +} + +/** + * Get current model + */ +export function getCurrentModel(): ModelInfo { + return AVAILABLE_MODELS.find(m => m.id === currentModelId) || AVAILABLE_MODELS[0]; +} + +/** + * Set current model + */ +export function setCurrentModel(modelId: string): boolean { + const model = AVAILABLE_MODELS.find(m => m.id === modelId); + if (model) { + currentModelId = modelId; + logger.info('Model changed to:', model.name); + return true; + } + logger.warn('Invalid model ID:', modelId); + return false; +} + /** * Create a new session or resume existing one for a diagram */ @@ -90,15 +145,24 @@ const SYSTEM_PROMPT = `You are a 3D diagram assistant helping users create and m Available entity shapes: box, sphere, cylinder, cone, plane, person Available colors: red, blue, green, yellow, orange, purple, cyan, pink, white, black, brown, gray (or any hex color like #ff5500) -Position coordinates: -- x: left/right (negative = left, positive = right) +IMPORTANT - Handling directional commands (left, right, forward, back, in front of me, etc.): +When the user uses relative directions, you MUST first call get_camera_position to get their current view. +The response provides: +- Their world position +- Ground-projected forward/right vectors (accounts for camera being on a moving platform) +- Pre-calculated positions for forward/back/left/right at 2m distance + +Use the ground-plane directions for intuitive placement - these ignore vertical look angle so "forward" means "forward on the ground" not "where I'm looking vertically". + +Position coordinates (when not using relative directions): +- x: world left/right - y: up/down (1.5 is eye level, 0 is floor) -- z: forward/backward (positive = toward user, negative = away) +- z: world forward/backward When creating diagrams, think about good spatial layout: - Spread entities apart to avoid overlap (at least 0.5 units) - Use y=1.5 for entities at eye level -- Use z=2 to z=4 for comfortable viewing distance +- For relative placement, use get_camera_position first Always use the provided tools to create, modify, or interact with entities. Be concise in your responses.`; @@ -230,6 +294,36 @@ const TOOLS = [ type: "object", properties: {} } + }, + { + name: "list_models", + description: "List all available AI models that can be used for this conversation.", + input_schema: { + type: "object", + properties: {} + } + }, + { + name: "get_current_model", + description: "Get information about the currently active AI model.", + input_schema: { + type: "object", + properties: {} + } + }, + { + name: "set_model", + description: "Change the AI model used for this conversation. Use list_models first to see available options.", + input_schema: { + type: "object", + properties: { + model_id: { + type: "string", + description: "The model ID to switch to (e.g., 'claude-sonnet-4-20250514', 'claude-opus-4-20250514', 'claude-haiku-3-5-20241022')" + } + }, + required: ["model_id"] + } } ]; @@ -285,6 +379,47 @@ async function executeToolCall(toolCall: DiagramToolCall): Promise { case 'get_camera_position': result = await getCameraPosition(); break; + case 'list_models': { + const models = getAvailableModels(); + const current = getCurrentModel(); + const modelList = models.map(m => + `• ${m.name} (${m.id})${m.id === current.id ? ' [CURRENT]' : ''}\n ${m.description}` + ).join('\n\n'); + result = { + toolName: 'list_models', + success: true, + message: `Available models:\n\n${modelList}` + }; + break; + } + case 'get_current_model': { + const model = getCurrentModel(); + result = { + toolName: 'get_current_model', + success: true, + message: `Current model: ${model.name} (${model.id})\n${model.description}` + }; + break; + } + case 'set_model': { + const success = setCurrentModel(toolCall.input.model_id); + if (success) { + const model = getCurrentModel(); + result = { + toolName: 'set_model', + success: true, + message: `Model changed to: ${model.name}\n${model.description}\n\nNote: The new model will be used starting from the next message.` + }; + } else { + const models = getAvailableModels(); + result = { + toolName: 'set_model', + success: false, + message: `Invalid model ID: "${toolCall.input.model_id}"\n\nAvailable models: ${models.map(m => m.id).join(', ')}` + }; + } + break; + } default: result = { toolName: 'unknown', @@ -332,7 +467,7 @@ export async function sendMessage( logger.debug(`[sendMessage] Loop iteration ${loopCount}`); const requestBody = { - model: 'claude-sonnet-4-20250514', + model: currentModelId, max_tokens: 1024, system: SYSTEM_PROMPT, tools: TOOLS, diff --git a/src/react/services/entityBridge.ts b/src/react/services/entityBridge.ts index abac291..9e3856d 100644 --- a/src/react/services/entityBridge.ts +++ b/src/react/services/entityBridge.ts @@ -313,21 +313,55 @@ export async function clearDiagram(params: ClearDiagramParams): Promise { logger.debug('[getCameraPosition] Getting camera position...'); return new Promise((resolve) => { const responseHandler = (e: CustomEvent) => { document.removeEventListener('chatGetCameraResponse', responseHandler as EventListener); - const {position, forward, up} = e.detail; + const {position, forward, groundForward, groundRight} = e.detail; logger.debug('[getCameraPosition] Got response:', e.detail); - const message = `Camera position: (${position.x.toFixed(2)}, ${position.y.toFixed(2)}, ${position.z.toFixed(2)}) -Forward direction: (${forward.x.toFixed(2)}, ${forward.y.toFixed(2)}, ${forward.z.toFixed(2)}) -Up direction: (${up.x.toFixed(2)}, ${up.y.toFixed(2)}, ${up.z.toFixed(2)}) + // Compute example positions for each direction + const distance = 2; + const frontPos = { + x: position.x + groundForward.x * distance, + y: position.y, + z: position.z + groundForward.z * distance + }; + const rightPos = { + x: position.x + groundRight.x * distance, + y: position.y, + z: position.z + groundRight.z * distance + }; + const leftPos = { + x: position.x - groundRight.x * distance, + y: position.y, + z: position.z - groundRight.z * distance + }; + const backPos = { + x: position.x - groundForward.x * distance, + y: position.y, + z: position.z - groundForward.z * distance + }; -To place an entity in front of the camera, add the forward direction scaled by desired distance to the camera position. -Example: For an entity 2 units in front: position = (${(position.x + forward.x * 2).toFixed(2)}, ${(position.y + forward.y * 2).toFixed(2)}, ${(position.z + forward.z * 2).toFixed(2)})`; + const message = `User's current view (world coordinates): + +Position: (${position.x.toFixed(2)}, ${position.y.toFixed(2)}, ${position.z.toFixed(2)}) +Looking direction: (${forward.x.toFixed(2)}, ${forward.y.toFixed(2)}, ${forward.z.toFixed(2)}) + +Ground-plane directions (use these for left/right/forward/back): +• Forward: (${groundForward.x.toFixed(2)}, 0, ${groundForward.z.toFixed(2)}) +• Right: (${groundRight.x.toFixed(2)}, 0, ${groundRight.z.toFixed(2)}) + +To place entities relative to user (${distance}m away, at eye level y=${position.y.toFixed(1)}): +• FORWARD: (${frontPos.x.toFixed(2)}, ${frontPos.y.toFixed(2)}, ${frontPos.z.toFixed(2)}) +• BACK: (${backPos.x.toFixed(2)}, ${backPos.y.toFixed(2)}, ${backPos.z.toFixed(2)}) +• RIGHT: (${rightPos.x.toFixed(2)}, ${rightPos.y.toFixed(2)}, ${rightPos.z.toFixed(2)}) +• LEFT: (${leftPos.x.toFixed(2)}, ${leftPos.y.toFixed(2)}, ${leftPos.z.toFixed(2)}) + +Formula: position + (groundForward * distance) for forward, position + (groundRight * distance) for right, etc.`; resolve({ toolName: 'get_camera_position', diff --git a/src/react/types/chatTypes.ts b/src/react/types/chatTypes.ts index 22ed7a6..7e10d5e 100644 --- a/src/react/types/chatTypes.ts +++ b/src/react/types/chatTypes.ts @@ -46,6 +46,10 @@ export interface ClearDiagramParams { confirmed: boolean; } +export interface SetModelParams { + model_id: string; +} + export type DiagramToolCall = | { name: 'create_entity'; input: CreateEntityParams } | { name: 'connect_entities'; input: ConnectEntitiesParams } @@ -53,7 +57,10 @@ export type DiagramToolCall = | { name: 'modify_entity'; input: ModifyEntityParams } | { name: 'list_entities'; input: Record } | { name: 'clear_diagram'; input: ClearDiagramParams } - | { name: 'get_camera_position'; input: Record }; + | { name: 'get_camera_position'; input: Record } + | { name: 'list_models'; input: Record } + | { name: 'get_current_model'; input: Record } + | { name: 'set_model'; input: SetModelParams }; export const SHAPE_TO_TEMPLATE: Record = { box: DiagramTemplates.BOX,