Add model selection and ground-projected directional placement
Model management: - Add list_models, get_current_model, set_model tools - Support Claude Sonnet 4, Opus 4, and Haiku 3.5 - Model selection persists for session duration Directional placement improvements: - Compute ground-projected forward/right vectors from camera - Accounts for camera being parented to moving platform - "Forward" means forward on ground plane, ignoring vertical look angle - Pre-calculate example positions for left/right/forward/back - Update system prompt to use get_camera_position for relative directions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
c9dc61b918
commit
a7aa385d98
@ -1,4 +1,4 @@
|
||||
import {AbstractActionManager, AbstractMesh, ActionManager, Observable, Scene, WebXRDefaultExperience} from "@babylonjs/core";
|
||||
import {AbstractActionManager, AbstractMesh, ActionManager, Observable, Scene, Vector3, WebXRDefaultExperience} from "@babylonjs/core";
|
||||
import {DiagramEntity, DiagramEntityType, DiagramEvent, DiagramEventType} from "./types/diagramEntity";
|
||||
import log from "loglevel";
|
||||
|
||||
@ -220,6 +220,7 @@ export class DiagramManager {
|
||||
});
|
||||
|
||||
// Get current camera position and orientation
|
||||
// Camera may be parented to a platform, so we use world-space coordinates
|
||||
document.addEventListener('chatGetCamera', () => {
|
||||
this._logger.debug('chatGetCamera');
|
||||
const camera = this._scene.activeCamera;
|
||||
@ -228,15 +229,36 @@ export class DiagramManager {
|
||||
return;
|
||||
}
|
||||
|
||||
// World-space position (accounts for parent transforms)
|
||||
const position = camera.globalPosition;
|
||||
|
||||
// World-space forward direction (where camera is looking)
|
||||
const forward = camera.getForwardRay(1).direction;
|
||||
const up = camera.upVector || {x: 0, y: 1, z: 0};
|
||||
|
||||
// World up vector
|
||||
const worldUp = new Vector3(0, 1, 0);
|
||||
|
||||
// Compute ground-projected forward (for intuitive forward/back movement)
|
||||
// This ignores pitch so looking up/down doesn't affect horizontal movement
|
||||
const groundForward = new Vector3(forward.x, 0, forward.z);
|
||||
const groundForwardLength = groundForward.length();
|
||||
if (groundForwardLength > 0.001) {
|
||||
groundForward.scaleInPlace(1 / groundForwardLength);
|
||||
} else {
|
||||
// Looking straight up/down - use a fallback forward
|
||||
groundForward.set(0, 0, -1);
|
||||
}
|
||||
|
||||
// Compute right vector (perpendicular to groundForward in XZ plane)
|
||||
// Right = Cross(groundForward, worldUp) gives left, so we negate or swap
|
||||
const groundRight = Vector3.Cross(worldUp, groundForward).normalize();
|
||||
|
||||
const responseEvent = new CustomEvent('chatGetCameraResponse', {
|
||||
detail: {
|
||||
position: {x: position.x, y: position.y, z: position.z},
|
||||
forward: {x: forward.x, y: forward.y, z: forward.z},
|
||||
up: {x: up.x, y: up.y, z: up.z}
|
||||
groundForward: {x: groundForward.x, y: groundForward.y, z: groundForward.z},
|
||||
groundRight: {x: groundRight.x, y: groundRight.y, z: groundRight.z}
|
||||
},
|
||||
bubbles: true
|
||||
});
|
||||
|
||||
@ -9,6 +9,61 @@ logger.setLevel('debug');
|
||||
// Session management
|
||||
let currentSessionId: string | null = null;
|
||||
|
||||
// Model management
|
||||
export interface ModelInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
const AVAILABLE_MODELS: ModelInfo[] = [
|
||||
{
|
||||
id: 'claude-sonnet-4-20250514',
|
||||
name: 'Claude Sonnet 4',
|
||||
description: 'Balanced performance and speed (default)'
|
||||
},
|
||||
{
|
||||
id: 'claude-opus-4-20250514',
|
||||
name: 'Claude Opus 4',
|
||||
description: 'Most capable, best for complex tasks'
|
||||
},
|
||||
{
|
||||
id: 'claude-haiku-3-5-20241022',
|
||||
name: 'Claude Haiku 3.5',
|
||||
description: 'Fastest responses, good for simple tasks'
|
||||
}
|
||||
];
|
||||
|
||||
let currentModelId: string = 'claude-sonnet-4-20250514';
|
||||
|
||||
/**
|
||||
* Get available models
|
||||
*/
|
||||
export function getAvailableModels(): ModelInfo[] {
|
||||
return [...AVAILABLE_MODELS];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current model
|
||||
*/
|
||||
export function getCurrentModel(): ModelInfo {
|
||||
return AVAILABLE_MODELS.find(m => m.id === currentModelId) || AVAILABLE_MODELS[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Set current model
|
||||
*/
|
||||
export function setCurrentModel(modelId: string): boolean {
|
||||
const model = AVAILABLE_MODELS.find(m => m.id === modelId);
|
||||
if (model) {
|
||||
currentModelId = modelId;
|
||||
logger.info('Model changed to:', model.name);
|
||||
return true;
|
||||
}
|
||||
logger.warn('Invalid model ID:', modelId);
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new session or resume existing one for a diagram
|
||||
*/
|
||||
@ -90,15 +145,24 @@ const SYSTEM_PROMPT = `You are a 3D diagram assistant helping users create and m
|
||||
Available entity shapes: box, sphere, cylinder, cone, plane, person
|
||||
Available colors: red, blue, green, yellow, orange, purple, cyan, pink, white, black, brown, gray (or any hex color like #ff5500)
|
||||
|
||||
Position coordinates:
|
||||
- x: left/right (negative = left, positive = right)
|
||||
IMPORTANT - Handling directional commands (left, right, forward, back, in front of me, etc.):
|
||||
When the user uses relative directions, you MUST first call get_camera_position to get their current view.
|
||||
The response provides:
|
||||
- Their world position
|
||||
- Ground-projected forward/right vectors (accounts for camera being on a moving platform)
|
||||
- Pre-calculated positions for forward/back/left/right at 2m distance
|
||||
|
||||
Use the ground-plane directions for intuitive placement - these ignore vertical look angle so "forward" means "forward on the ground" not "where I'm looking vertically".
|
||||
|
||||
Position coordinates (when not using relative directions):
|
||||
- x: world left/right
|
||||
- y: up/down (1.5 is eye level, 0 is floor)
|
||||
- z: forward/backward (positive = toward user, negative = away)
|
||||
- z: world forward/backward
|
||||
|
||||
When creating diagrams, think about good spatial layout:
|
||||
- Spread entities apart to avoid overlap (at least 0.5 units)
|
||||
- Use y=1.5 for entities at eye level
|
||||
- Use z=2 to z=4 for comfortable viewing distance
|
||||
- For relative placement, use get_camera_position first
|
||||
|
||||
Always use the provided tools to create, modify, or interact with entities. Be concise in your responses.`;
|
||||
|
||||
@ -230,6 +294,36 @@ const TOOLS = [
|
||||
type: "object",
|
||||
properties: {}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: "list_models",
|
||||
description: "List all available AI models that can be used for this conversation.",
|
||||
input_schema: {
|
||||
type: "object",
|
||||
properties: {}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: "get_current_model",
|
||||
description: "Get information about the currently active AI model.",
|
||||
input_schema: {
|
||||
type: "object",
|
||||
properties: {}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: "set_model",
|
||||
description: "Change the AI model used for this conversation. Use list_models first to see available options.",
|
||||
input_schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
model_id: {
|
||||
type: "string",
|
||||
description: "The model ID to switch to (e.g., 'claude-sonnet-4-20250514', 'claude-opus-4-20250514', 'claude-haiku-3-5-20241022')"
|
||||
}
|
||||
},
|
||||
required: ["model_id"]
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
@ -285,6 +379,47 @@ async function executeToolCall(toolCall: DiagramToolCall): Promise<ToolResult> {
|
||||
case 'get_camera_position':
|
||||
result = await getCameraPosition();
|
||||
break;
|
||||
case 'list_models': {
|
||||
const models = getAvailableModels();
|
||||
const current = getCurrentModel();
|
||||
const modelList = models.map(m =>
|
||||
`• ${m.name} (${m.id})${m.id === current.id ? ' [CURRENT]' : ''}\n ${m.description}`
|
||||
).join('\n\n');
|
||||
result = {
|
||||
toolName: 'list_models',
|
||||
success: true,
|
||||
message: `Available models:\n\n${modelList}`
|
||||
};
|
||||
break;
|
||||
}
|
||||
case 'get_current_model': {
|
||||
const model = getCurrentModel();
|
||||
result = {
|
||||
toolName: 'get_current_model',
|
||||
success: true,
|
||||
message: `Current model: ${model.name} (${model.id})\n${model.description}`
|
||||
};
|
||||
break;
|
||||
}
|
||||
case 'set_model': {
|
||||
const success = setCurrentModel(toolCall.input.model_id);
|
||||
if (success) {
|
||||
const model = getCurrentModel();
|
||||
result = {
|
||||
toolName: 'set_model',
|
||||
success: true,
|
||||
message: `Model changed to: ${model.name}\n${model.description}\n\nNote: The new model will be used starting from the next message.`
|
||||
};
|
||||
} else {
|
||||
const models = getAvailableModels();
|
||||
result = {
|
||||
toolName: 'set_model',
|
||||
success: false,
|
||||
message: `Invalid model ID: "${toolCall.input.model_id}"\n\nAvailable models: ${models.map(m => m.id).join(', ')}`
|
||||
};
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
result = {
|
||||
toolName: 'unknown',
|
||||
@ -332,7 +467,7 @@ export async function sendMessage(
|
||||
logger.debug(`[sendMessage] Loop iteration ${loopCount}`);
|
||||
|
||||
const requestBody = {
|
||||
model: 'claude-sonnet-4-20250514',
|
||||
model: currentModelId,
|
||||
max_tokens: 1024,
|
||||
system: SYSTEM_PROMPT,
|
||||
tools: TOOLS,
|
||||
|
||||
@ -313,21 +313,55 @@ export async function clearDiagram(params: ClearDiagramParams): Promise<ToolResu
|
||||
|
||||
/**
|
||||
* Get the current camera position and orientation
|
||||
* Returns world-space coordinates with ground-projected directions for intuitive placement
|
||||
*/
|
||||
export function getCameraPosition(): Promise<ToolResult> {
|
||||
logger.debug('[getCameraPosition] Getting camera position...');
|
||||
return new Promise((resolve) => {
|
||||
const responseHandler = (e: CustomEvent) => {
|
||||
document.removeEventListener('chatGetCameraResponse', responseHandler as EventListener);
|
||||
const {position, forward, up} = e.detail;
|
||||
const {position, forward, groundForward, groundRight} = e.detail;
|
||||
logger.debug('[getCameraPosition] Got response:', e.detail);
|
||||
|
||||
const message = `Camera position: (${position.x.toFixed(2)}, ${position.y.toFixed(2)}, ${position.z.toFixed(2)})
|
||||
Forward direction: (${forward.x.toFixed(2)}, ${forward.y.toFixed(2)}, ${forward.z.toFixed(2)})
|
||||
Up direction: (${up.x.toFixed(2)}, ${up.y.toFixed(2)}, ${up.z.toFixed(2)})
|
||||
// Compute example positions for each direction
|
||||
const distance = 2;
|
||||
const frontPos = {
|
||||
x: position.x + groundForward.x * distance,
|
||||
y: position.y,
|
||||
z: position.z + groundForward.z * distance
|
||||
};
|
||||
const rightPos = {
|
||||
x: position.x + groundRight.x * distance,
|
||||
y: position.y,
|
||||
z: position.z + groundRight.z * distance
|
||||
};
|
||||
const leftPos = {
|
||||
x: position.x - groundRight.x * distance,
|
||||
y: position.y,
|
||||
z: position.z - groundRight.z * distance
|
||||
};
|
||||
const backPos = {
|
||||
x: position.x - groundForward.x * distance,
|
||||
y: position.y,
|
||||
z: position.z - groundForward.z * distance
|
||||
};
|
||||
|
||||
To place an entity in front of the camera, add the forward direction scaled by desired distance to the camera position.
|
||||
Example: For an entity 2 units in front: position = (${(position.x + forward.x * 2).toFixed(2)}, ${(position.y + forward.y * 2).toFixed(2)}, ${(position.z + forward.z * 2).toFixed(2)})`;
|
||||
const message = `User's current view (world coordinates):
|
||||
|
||||
Position: (${position.x.toFixed(2)}, ${position.y.toFixed(2)}, ${position.z.toFixed(2)})
|
||||
Looking direction: (${forward.x.toFixed(2)}, ${forward.y.toFixed(2)}, ${forward.z.toFixed(2)})
|
||||
|
||||
Ground-plane directions (use these for left/right/forward/back):
|
||||
• Forward: (${groundForward.x.toFixed(2)}, 0, ${groundForward.z.toFixed(2)})
|
||||
• Right: (${groundRight.x.toFixed(2)}, 0, ${groundRight.z.toFixed(2)})
|
||||
|
||||
To place entities relative to user (${distance}m away, at eye level y=${position.y.toFixed(1)}):
|
||||
• FORWARD: (${frontPos.x.toFixed(2)}, ${frontPos.y.toFixed(2)}, ${frontPos.z.toFixed(2)})
|
||||
• BACK: (${backPos.x.toFixed(2)}, ${backPos.y.toFixed(2)}, ${backPos.z.toFixed(2)})
|
||||
• RIGHT: (${rightPos.x.toFixed(2)}, ${rightPos.y.toFixed(2)}, ${rightPos.z.toFixed(2)})
|
||||
• LEFT: (${leftPos.x.toFixed(2)}, ${leftPos.y.toFixed(2)}, ${leftPos.z.toFixed(2)})
|
||||
|
||||
Formula: position + (groundForward * distance) for forward, position + (groundRight * distance) for right, etc.`;
|
||||
|
||||
resolve({
|
||||
toolName: 'get_camera_position',
|
||||
|
||||
@ -46,6 +46,10 @@ export interface ClearDiagramParams {
|
||||
confirmed: boolean;
|
||||
}
|
||||
|
||||
export interface SetModelParams {
|
||||
model_id: string;
|
||||
}
|
||||
|
||||
export type DiagramToolCall =
|
||||
| { name: 'create_entity'; input: CreateEntityParams }
|
||||
| { name: 'connect_entities'; input: ConnectEntitiesParams }
|
||||
@ -53,7 +57,10 @@ export type DiagramToolCall =
|
||||
| { name: 'modify_entity'; input: ModifyEntityParams }
|
||||
| { name: 'list_entities'; input: Record<string, never> }
|
||||
| { name: 'clear_diagram'; input: ClearDiagramParams }
|
||||
| { name: 'get_camera_position'; input: Record<string, never> };
|
||||
| { name: 'get_camera_position'; input: Record<string, never> }
|
||||
| { name: 'list_models'; input: Record<string, never> }
|
||||
| { name: 'get_current_model'; input: Record<string, never> }
|
||||
| { name: 'set_model'; input: SetModelParams };
|
||||
|
||||
export const SHAPE_TO_TEMPLATE: Record<CreateEntityParams['shape'], DiagramTemplates> = {
|
||||
box: DiagramTemplates.BOX,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user