Add model selection and ground-projected directional placement

Model management:
- Add list_models, get_current_model, set_model tools
- Support Claude Sonnet 4, Opus 4, and Haiku 3.5
- Model selection persists for session duration

Directional placement improvements:
- Compute ground-projected forward/right vectors from camera
- Accounts for camera being parented to moving platform
- "Forward" means forward on ground plane, ignoring vertical look angle
- Pre-calculate example positions for left/right/forward/back
- Update system prompt to use get_camera_position for relative directions

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

src/diagram/diagramManager.ts

@@ -1,4 +1,4 @@
-import {AbstractActionManager, AbstractMesh, ActionManager, Observable, Scene, WebXRDefaultExperience} from "@babylonjs/core";
+import {AbstractActionManager, AbstractMesh, ActionManager, Observable, Scene, Vector3, WebXRDefaultExperience} from "@babylonjs/core";
 import {DiagramEntity, DiagramEntityType, DiagramEvent, DiagramEventType} from "./types/diagramEntity";
 import log from "loglevel";
 
@@ -220,6 +220,7 @@ export class DiagramManager {
         });
 
         // Get current camera position and orientation
+        // Camera may be parented to a platform, so we use world-space coordinates
         document.addEventListener('chatGetCamera', () => {
             this._logger.debug('chatGetCamera');
             const camera = this._scene.activeCamera;
@@ -228,15 +229,36 @@ export class DiagramManager {
                 return;
             }
 
+            // World-space position (accounts for parent transforms)
             const position = camera.globalPosition;
+
+            // World-space forward direction (where camera is looking)
             const forward = camera.getForwardRay(1).direction;
-            const up = camera.upVector || {x: 0, y: 1, z: 0};
+
+            // World up vector
+            const worldUp = new Vector3(0, 1, 0);
+
+            // Compute ground-projected forward (for intuitive forward/back movement)
+            // This ignores pitch so looking up/down doesn't affect horizontal movement
+            const groundForward = new Vector3(forward.x, 0, forward.z);
+            const groundForwardLength = groundForward.length();
+            if (groundForwardLength > 0.001) {
+                groundForward.scaleInPlace(1 / groundForwardLength);
+            } else {
+                // Looking straight up/down - use a fallback forward
+                groundForward.set(0, 0, -1);
+            }
+
+            // Compute right vector (perpendicular to groundForward in XZ plane)
+            // Right = Cross(groundForward, worldUp) gives left, so we negate or swap
+            const groundRight = Vector3.Cross(worldUp, groundForward).normalize();
 
             const responseEvent = new CustomEvent('chatGetCameraResponse', {
                 detail: {
                     position: {x: position.x, y: position.y, z: position.z},
                     forward: {x: forward.x, y: forward.y, z: forward.z},
-                    up: {x: up.x, y: up.y, z: up.z}
+                    groundForward: {x: groundForward.x, y: groundForward.y, z: groundForward.z},
+                    groundRight: {x: groundRight.x, y: groundRight.y, z: groundRight.z}
                 },
                 bubbles: true
             });

src/react/services/diagramAI.ts

@@ -9,6 +9,61 @@ logger.setLevel('debug');
 // Session management
 let currentSessionId: string | null = null;
 
+// Model management
+export interface ModelInfo {
+    id: string;
+    name: string;
+    description: string;
+}
+
+const AVAILABLE_MODELS: ModelInfo[] = [
+    {
+        id: 'claude-sonnet-4-20250514',
+        name: 'Claude Sonnet 4',
+        description: 'Balanced performance and speed (default)'
+    },
+    {
+        id: 'claude-opus-4-20250514',
+        name: 'Claude Opus 4',
+        description: 'Most capable, best for complex tasks'
+    },
+    {
+        id: 'claude-haiku-3-5-20241022',
+        name: 'Claude Haiku 3.5',
+        description: 'Fastest responses, good for simple tasks'
+    }
+];
+
+let currentModelId: string = 'claude-sonnet-4-20250514';
+
+/**
+ * Get available models
+ */
+export function getAvailableModels(): ModelInfo[] {
+    return [...AVAILABLE_MODELS];
+}
+
+/**
+ * Get current model
+ */
+export function getCurrentModel(): ModelInfo {
+    return AVAILABLE_MODELS.find(m => m.id === currentModelId) || AVAILABLE_MODELS[0];
+}
+
+/**
+ * Set current model
+ */
+export function setCurrentModel(modelId: string): boolean {
+    const model = AVAILABLE_MODELS.find(m => m.id === modelId);
+    if (model) {
+        currentModelId = modelId;
+        logger.info('Model changed to:', model.name);
+        return true;
+    }
+    logger.warn('Invalid model ID:', modelId);
+    return false;
+}
+
 /**
  * Create a new session or resume existing one for a diagram
  */
@@ -90,15 +145,24 @@ const SYSTEM_PROMPT = `You are a 3D diagram assistant helping users create and m
 Available entity shapes: box, sphere, cylinder, cone, plane, person
 Available colors: red, blue, green, yellow, orange, purple, cyan, pink, white, black, brown, gray (or any hex color like #ff5500)
 
-Position coordinates:
+IMPORTANT - Handling directional commands (left, right, forward, back, in front of me, etc.):
-- x: left/right (negative = left, positive = right)
+When the user uses relative directions, you MUST first call get_camera_position to get their current view.
+The response provides:
+- Their world position
+- Ground-projected forward/right vectors (accounts for camera being on a moving platform)
+- Pre-calculated positions for forward/back/left/right at 2m distance
+
+Use the ground-plane directions for intuitive placement - these ignore vertical look angle so "forward" means "forward on the ground" not "where I'm looking vertically".
+
+Position coordinates (when not using relative directions):
+- x: world left/right
 - y: up/down (1.5 is eye level, 0 is floor)
-- z: forward/backward (positive = toward user, negative = away)
+- z: world forward/backward
 
 When creating diagrams, think about good spatial layout:
 - Spread entities apart to avoid overlap (at least 0.5 units)
 - Use y=1.5 for entities at eye level
-- Use z=2 to z=4 for comfortable viewing distance
+- For relative placement, use get_camera_position first
 
 Always use the provided tools to create, modify, or interact with entities. Be concise in your responses.`;
 
@@ -230,6 +294,36 @@ const TOOLS = [
             type: "object",
             properties: {}
         }
+    },
+    {
+        name: "list_models",
+        description: "List all available AI models that can be used for this conversation.",
+        input_schema: {
+            type: "object",
+            properties: {}
+        }
+    },
+    {
+        name: "get_current_model",
+        description: "Get information about the currently active AI model.",
+        input_schema: {
+            type: "object",
+            properties: {}
+        }
+    },
+    {
+        name: "set_model",
+        description: "Change the AI model used for this conversation. Use list_models first to see available options.",
+        input_schema: {
+            type: "object",
+            properties: {
+                model_id: {
+                    type: "string",
+                    description: "The model ID to switch to (e.g., 'claude-sonnet-4-20250514', 'claude-opus-4-20250514', 'claude-haiku-3-5-20241022')"
+                }
+            },
+            required: ["model_id"]
+        }
     }
 ];
 
@@ -285,6 +379,47 @@ async function executeToolCall(toolCall: DiagramToolCall): Promise<ToolResult> {
         case 'get_camera_position':
             result = await getCameraPosition();
             break;
+        case 'list_models': {
+            const models = getAvailableModels();
+            const current = getCurrentModel();
+            const modelList = models.map(m =>
+                `• ${m.name} (${m.id})${m.id === current.id ? ' [CURRENT]' : ''}\n  ${m.description}`
+            ).join('\n\n');
+            result = {
+                toolName: 'list_models',
+                success: true,
+                message: `Available models:\n\n${modelList}`
+            };
+            break;
+        }
+        case 'get_current_model': {
+            const model = getCurrentModel();
+            result = {
+                toolName: 'get_current_model',
+                success: true,
+                message: `Current model: ${model.name} (${model.id})\n${model.description}`
+            };
+            break;
+        }
+        case 'set_model': {
+            const success = setCurrentModel(toolCall.input.model_id);
+            if (success) {
+                const model = getCurrentModel();
+                result = {
+                    toolName: 'set_model',
+                    success: true,
+                    message: `Model changed to: ${model.name}\n${model.description}\n\nNote: The new model will be used starting from the next message.`
+                };
+            } else {
+                const models = getAvailableModels();
+                result = {
+                    toolName: 'set_model',
+                    success: false,
+                    message: `Invalid model ID: "${toolCall.input.model_id}"\n\nAvailable models: ${models.map(m => m.id).join(', ')}`
+                };
+            }
+            break;
+        }
         default:
             result = {
                 toolName: 'unknown',
@@ -332,7 +467,7 @@ export async function sendMessage(
         logger.debug(`[sendMessage] Loop iteration ${loopCount}`);
 
         const requestBody = {
-            model: 'claude-sonnet-4-20250514',
+            model: currentModelId,
             max_tokens: 1024,
             system: SYSTEM_PROMPT,
             tools: TOOLS,

src/react/services/entityBridge.ts

@@ -313,21 +313,55 @@ export async function clearDiagram(params: ClearDiagramParams): Promise<ToolResu
 
 /**
  * Get the current camera position and orientation
+ * Returns world-space coordinates with ground-projected directions for intuitive placement
  */
 export function getCameraPosition(): Promise<ToolResult> {
     logger.debug('[getCameraPosition] Getting camera position...');
     return new Promise((resolve) => {
         const responseHandler = (e: CustomEvent) => {
             document.removeEventListener('chatGetCameraResponse', responseHandler as EventListener);
-            const {position, forward, up} = e.detail;
+            const {position, forward, groundForward, groundRight} = e.detail;
             logger.debug('[getCameraPosition] Got response:', e.detail);
 
-            const message = `Camera position: (${position.x.toFixed(2)}, ${position.y.toFixed(2)}, ${position.z.toFixed(2)})
+            // Compute example positions for each direction
-Forward direction: (${forward.x.toFixed(2)}, ${forward.y.toFixed(2)}, ${forward.z.toFixed(2)})
+            const distance = 2;
-Up direction: (${up.x.toFixed(2)}, ${up.y.toFixed(2)}, ${up.z.toFixed(2)})
+            const frontPos = {
+                x: position.x + groundForward.x * distance,
+                y: position.y,
+                z: position.z + groundForward.z * distance
+            };
+            const rightPos = {
+                x: position.x + groundRight.x * distance,
+                y: position.y,
+                z: position.z + groundRight.z * distance
+            };
+            const leftPos = {
+                x: position.x - groundRight.x * distance,
+                y: position.y,
+                z: position.z - groundRight.z * distance
+            };
+            const backPos = {
+                x: position.x - groundForward.x * distance,
+                y: position.y,
+                z: position.z - groundForward.z * distance
+            };
 
-To place an entity in front of the camera, add the forward direction scaled by desired distance to the camera position.
+            const message = `User's current view (world coordinates):
-Example: For an entity 2 units in front: position = (${(position.x + forward.x * 2).toFixed(2)}, ${(position.y + forward.y * 2).toFixed(2)}, ${(position.z + forward.z * 2).toFixed(2)})`;
+
+Position: (${position.x.toFixed(2)}, ${position.y.toFixed(2)}, ${position.z.toFixed(2)})
+Looking direction: (${forward.x.toFixed(2)}, ${forward.y.toFixed(2)}, ${forward.z.toFixed(2)})
+
+Ground-plane directions (use these for left/right/forward/back):
+• Forward: (${groundForward.x.toFixed(2)}, 0, ${groundForward.z.toFixed(2)})
+• Right: (${groundRight.x.toFixed(2)}, 0, ${groundRight.z.toFixed(2)})
+
+To place entities relative to user (${distance}m away, at eye level y=${position.y.toFixed(1)}):
+• FORWARD: (${frontPos.x.toFixed(2)}, ${frontPos.y.toFixed(2)}, ${frontPos.z.toFixed(2)})
+• BACK: (${backPos.x.toFixed(2)}, ${backPos.y.toFixed(2)}, ${backPos.z.toFixed(2)})
+• RIGHT: (${rightPos.x.toFixed(2)}, ${rightPos.y.toFixed(2)}, ${rightPos.z.toFixed(2)})
+• LEFT: (${leftPos.x.toFixed(2)}, ${leftPos.y.toFixed(2)}, ${leftPos.z.toFixed(2)})
+
+Formula: position + (groundForward * distance) for forward, position + (groundRight * distance) for right, etc.`;
 
             resolve({
                 toolName: 'get_camera_position',

src/react/types/chatTypes.ts

@@ -46,6 +46,10 @@ export interface ClearDiagramParams {
     confirmed: boolean;
 }
 
+export interface SetModelParams {
+    model_id: string;
+}
+
 export type DiagramToolCall =
     | { name: 'create_entity'; input: CreateEntityParams }
     | { name: 'connect_entities'; input: ConnectEntitiesParams }
@@ -53,7 +57,10 @@ export type DiagramToolCall =
     | { name: 'modify_entity'; input: ModifyEntityParams }
     | { name: 'list_entities'; input: Record<string, never> }
     | { name: 'clear_diagram'; input: ClearDiagramParams }
-    | { name: 'get_camera_position'; input: Record<string, never> };
+    | { name: 'get_camera_position'; input: Record<string, never> }
+    | { name: 'list_models'; input: Record<string, never> }
+    | { name: 'get_current_model'; input: Record<string, never> }
+    | { name: 'set_model'; input: SetModelParams };
 
 export const SHAPE_TO_TEMPLATE: Record<CreateEntityParams['shape'], DiagramTemplates> = {
     box: DiagramTemplates.BOX,