From 719c969f72a9cddff5a405645f4347f0348c38b3 Mon Sep 17 00:00:00 2001 From: Michael Mainguy Date: Wed, 14 Jan 2026 14:45:42 -0600 Subject: [PATCH] Add WebXR system keyboard support and improve entity positioning - Add SystemKeyboardInput class for Meta Quest native keyboard in VR - Integrate system keyboard into DiagramMenuManager with auto-detection - Falls back to virtual keyboard when system keyboard not supported - Update AI system prompt with entity positioning guidelines: - Default position near (0, 1.5, 0) at eye level - Minimum 0.2m spacing between entities - Entities placed in front of camera with positive z values Co-Authored-By: Claude Opus 4.5 --- src/diagram/diagramMenuManager.ts | 36 +++++- src/information/systemKeyboardInput.ts | 158 +++++++++++++++++++++++++ src/react/services/diagramAI.ts | 54 ++++++--- 3 files changed, 226 insertions(+), 22 deletions(-) create mode 100644 src/information/systemKeyboardInput.ts diff --git a/src/diagram/diagramMenuManager.ts b/src/diagram/diagramMenuManager.ts index 4e491a4..7b1ac3f 100644 --- a/src/diagram/diagramMenuManager.ts +++ b/src/diagram/diagramMenuManager.ts @@ -1,6 +1,7 @@ import {DiagramEntityType, DiagramEvent, DiagramEventType} from "./types/diagramEntity"; import {AbstractMesh, ActionEvent, Observable, Scene, Vector3, WebXRDefaultExperience, WebXRInputSource} from "@babylonjs/core"; import {InputTextView} from "../information/inputTextView"; +import {SystemKeyboardInput} from "../information/systemKeyboardInput"; import {DefaultScene} from "../defaultScene"; import log from "loglevel"; import {Toolbox} from "../toolbox/toolbox"; @@ -19,6 +20,8 @@ export class DiagramMenuManager { public readonly toolbox: Toolbox; private readonly _notifier: Observable; private readonly _inputTextView: InputTextView; + private readonly _systemKeyboardInput: SystemKeyboardInput; + private _useSystemKeyboard: boolean = false; private readonly _vrConfigPanel: VRConfigPanel; private _groupMenu: GroupMenu; private readonly _scene: Scene; @@ -31,16 +34,22 @@ export class DiagramMenuManager { this._scene = DefaultScene.Scene; this._notifier = notifier; this._inputTextView = new InputTextView(controllerObservable); + this._systemKeyboardInput = new SystemKeyboardInput(); this._vrConfigPanel = new VRConfigPanel(this._scene); //this.configMenu = new ConfigMenu(config); - this._inputTextView.onTextObservable.add((evt) => { + // Handler for text input changes (shared between both input methods) + const handleTextEvent = (evt) => { const event = { type: DiagramEventType.MODIFY, entity: {id: evt.id, text: evt.text, type: DiagramEntityType.ENTITY} } this._notifier.notifyObservers(event, DiagramEventObserverMask.FROM_DB); - }); + }; + + this._inputTextView.onTextObservable.add(handleTextEvent); + this._systemKeyboardInput.onTextObservable.add(handleTextEvent); + this.toolbox = new Toolbox(readyObservable); if (viewOnly()) { @@ -87,7 +96,13 @@ export class DiagramMenuManager { } public editText(mesh: AbstractMesh) { - this._inputTextView.show(mesh); + if (this._useSystemKeyboard) { + this._logger.debug('Using system keyboard for text input'); + this._systemKeyboardInput.show(mesh); + } else { + this._logger.debug('Using virtual keyboard for text input'); + this._inputTextView.show(mesh); + } } public activateResizeGizmo(mesh: AbstractMesh) { @@ -164,6 +179,21 @@ export class DiagramMenuManager { public setXR(xr: WebXRDefaultExperience): void { this._xr = xr; this.toolbox.setXR(xr, this); + + // Listen for XR session start to detect system keyboard support + xr.baseExperience.onStateChangedObservable.add((state) => { + if (state === 2) { // WebXRState.IN_XR + const session = xr.baseExperience.sessionManager.session; + this._useSystemKeyboard = SystemKeyboardInput.isSupported(session); + this._logger.debug('System keyboard supported:', this._useSystemKeyboard); + + if (this._useSystemKeyboard) { + this._systemKeyboardInput.setSession(session); + } + } else if (state === 0) { // WebXRState.NOT_IN_XR + this._useSystemKeyboard = false; + } + }); } public toggleVRConfigPanel(): void { diff --git a/src/information/systemKeyboardInput.ts b/src/information/systemKeyboardInput.ts new file mode 100644 index 0000000..a349163 --- /dev/null +++ b/src/information/systemKeyboardInput.ts @@ -0,0 +1,158 @@ +import {AbstractMesh, Observable} from "@babylonjs/core"; +import log, {Logger} from "loglevel"; +import {TextEvent} from "./inputTextView"; + +/** + * SystemKeyboardInput provides text input using the WebXR system keyboard (Meta Quest native keyboard). + * This is an alternative to the VirtualKeyboard-based InputTextView for devices that support + * the WebXR system keyboard API. + * + * Usage: + * - Check isSupported() before using + * - Call show(mesh) to display the system keyboard + * - Listen to onTextObservable for text changes + * - Call dispose() when XR session ends + */ +export class SystemKeyboardInput { + private logger: Logger = log.getLogger('SystemKeyboardInput'); + public readonly onTextObservable: Observable = new Observable(); + + private inputElement: HTMLInputElement | null = null; + private currentMeshId: string | null = null; + private xrSession: XRSession | null = null; + private visibilityHandler: ((ev: XRSessionEvent) => void) | null = null; + + constructor() { + this.logger.debug('SystemKeyboardInput created'); + } + + /** + * Check if the system keyboard is supported for the given XR session. + */ + public static isSupported(session: XRSession | null): boolean { + if (!session) { + return false; + } + // TypeScript doesn't have this property in the type definitions yet + return (session as any).isSystemKeyboardSupported === true; + } + + /** + * Initialize with an XR session. Should be called when entering immersive mode. + */ + public setSession(session: XRSession): void { + this.xrSession = session; + + // Set up visibility change listener for keyboard lifecycle + this.visibilityHandler = (ev: XRSessionEvent) => { + const visState = (ev.target as XRSession).visibilityState; + this.logger.debug('XR visibility changed:', visState); + + if (visState === 'visible' && this.inputElement) { + // Keyboard was dismissed, finalize input + this.finalizeInput(); + } + }; + + session.addEventListener('visibilitychange', this.visibilityHandler); + + // Clean up when session ends + session.addEventListener('end', () => { + this.dispose(); + }); + } + + /** + * Show the system keyboard for editing the given mesh's text. + */ + public show(mesh: AbstractMesh): void { + if (!this.xrSession) { + this.logger.warn('Cannot show system keyboard: no XR session'); + return; + } + + this.currentMeshId = mesh.id; + + // Create input element if it doesn't exist + if (!this.inputElement) { + this.inputElement = document.createElement('input'); + this.inputElement.type = 'text'; + this.inputElement.id = 'system-keyboard-input'; + + // Position off-screen to avoid visual interference + // but keep it in DOM for keyboard to work + this.inputElement.style.position = 'fixed'; + this.inputElement.style.left = '-9999px'; + this.inputElement.style.top = '0'; + this.inputElement.style.opacity = '0'; + this.inputElement.style.pointerEvents = 'none'; + + // Handle blur event (keyboard dismissed) + this.inputElement.onblur = () => { + this.logger.debug('Input blurred'); + this.finalizeInput(); + }; + + document.body.appendChild(this.inputElement); + } + + // Set initial value from mesh metadata + const initialText = mesh.metadata?.text || ''; + this.inputElement.value = initialText; + + this.logger.debug('Showing system keyboard for mesh:', mesh.id, 'initial text:', initialText); + + // Focus to trigger the system keyboard + this.inputElement.focus(); + } + + /** + * Hide the keyboard and discard any changes. + */ + public hide(): void { + if (this.inputElement) { + this.inputElement.blur(); + } + this.currentMeshId = null; + } + + /** + * Finalize the input and notify observers. + */ + private finalizeInput(): void { + if (!this.currentMeshId || !this.inputElement) { + return; + } + + const text = this.inputElement.value.trim(); + this.logger.debug('Finalizing input:', text, 'for mesh:', this.currentMeshId); + + // Notify observers with the final text (or null if empty) + this.onTextObservable.notifyObservers({ + id: this.currentMeshId, + text: text.length > 0 ? text : null + }); + + this.currentMeshId = null; + } + + /** + * Clean up resources. Should be called when XR session ends. + */ + public dispose(): void { + this.logger.debug('Disposing SystemKeyboardInput'); + + if (this.visibilityHandler && this.xrSession) { + this.xrSession.removeEventListener('visibilitychange', this.visibilityHandler); + this.visibilityHandler = null; + } + + if (this.inputElement) { + this.inputElement.remove(); + this.inputElement = null; + } + + this.xrSession = null; + this.currentMeshId = null; + } +} diff --git a/src/react/services/diagramAI.ts b/src/react/services/diagramAI.ts index 6273fd0..0010daf 100644 --- a/src/react/services/diagramAI.ts +++ b/src/react/services/diagramAI.ts @@ -17,84 +17,97 @@ export interface ModelInfo { name: string; description: string; provider: AIProvider; + contextLimit: number; // Maximum context window size in tokens } const AVAILABLE_MODELS: ModelInfo[] = [ - // Claude models + // Claude models - 200K context { id: 'claude-sonnet-4-20250514', name: 'Claude Sonnet 4', description: 'Balanced performance and speed (default)', - provider: 'claude' + provider: 'claude', + contextLimit: 200000 }, { id: 'claude-opus-4-20250514', name: 'Claude Opus 4', description: 'Most capable, best for complex tasks', - provider: 'claude' + provider: 'claude', + contextLimit: 200000 }, { id: 'claude-haiku-3-5-20241022', name: 'Claude Haiku 3.5', description: 'Fastest responses, good for simple tasks', - provider: 'claude' + provider: 'claude', + contextLimit: 200000 }, // Cloudflare Workers AI models - with tool support { id: '@cf/mistralai/mistral-small-3.1-24b-instruct', name: 'Mistral Small 3.1 (CF)', description: 'Best CF model - supports diagram tools', - provider: 'cloudflare' + provider: 'cloudflare', + contextLimit: 32000 }, { id: '@hf/nousresearch/hermes-2-pro-mistral-7b', name: 'Hermes 2 Pro (CF)', description: 'Lightweight - supports diagram tools', - provider: 'cloudflare' + provider: 'cloudflare', + contextLimit: 8000 }, // Cloudflare models WITHOUT tool support - chat only { id: '@cf/meta/llama-3.3-70b-instruct-fp8-fast', name: 'Llama 3.3 70B (CF)', description: 'Powerful but NO tool support', - provider: 'cloudflare' + provider: 'cloudflare', + contextLimit: 128000 }, { id: '@cf/meta/llama-3.1-8b-instruct', name: 'Llama 3.1 8B (CF)', description: 'Fast/cheap but NO tool support', - provider: 'cloudflare' + provider: 'cloudflare', + contextLimit: 128000 }, { id: '@cf/deepseek-ai/deepseek-r1-distill-qwen-32b', name: 'DeepSeek R1 (CF)', description: 'Reasoning but NO tool support', - provider: 'cloudflare' + provider: 'cloudflare', + contextLimit: 32000 }, { id: '@cf/qwen/qwen2.5-coder-32b-instruct', name: 'Qwen 2.5 Coder (CF)', description: 'Code-focused but NO tool support', - provider: 'cloudflare' + provider: 'cloudflare', + contextLimit: 32000 }, - // Ollama models (local) + // Ollama models (local) - context depends on local config, using defaults { id: 'llama3.1', name: 'Llama 3.1', description: 'Local model with function calling support', - provider: 'ollama' + provider: 'ollama', + contextLimit: 128000 }, { id: 'mistral', name: 'Mistral', description: 'Fast local model with good tool support', - provider: 'ollama' + provider: 'ollama', + contextLimit: 32000 }, { id: 'qwen2.5', name: 'Qwen 2.5', description: 'Capable local model with function calling', - provider: 'ollama' + provider: 'ollama', + contextLimit: 32000 } ]; @@ -413,9 +426,12 @@ Example positions: - (0, 3, 2) = directly in front, above eye level ## Layout Guidelines -- Spread entities apart (at least 0.5 units) -- Use y=1.5 for eye-level entities -- For relative directions, call get_camera_position first +- **Default position**: Place new entities in front of the camera, centered around (0, 1.5, 0) which is eye level +- **Minimum spacing**: Entities must NEVER be closer than 0.2 meters (20cm) from each other +- **Spread layout**: When creating multiple entities, spread them horizontally (x-axis) and vertically (y-axis) with at least 0.5m between them +- Use y=1.5 for eye-level entities (comfortable viewing height) +- Use positive z values (e.g., z=1 to z=3) to place entities in front of the user +- For relative directions (left, right, forward), call get_camera_position first Be concise. Call tools immediately when the user requests an action.`; @@ -444,9 +460,9 @@ const TOOLS = [ properties: { x: {type: "number", description: "Left (-) / Right (+) from camera view"}, y: {type: "number", description: "Down (-) / Up (+), 0=floor, 1.5=eye level"}, - z: {type: "number", description: "Backward (-) / Forward (+) from camera view"} + z: {type: "number", description: "Backward (-) / Forward (+) from camera view, use positive values (1-3) to place in front"} }, - description: "3D position from camera perspective. Example: (0, 1.5, 2) = in front at eye level" + description: "3D position. Default around (0, 1.5, 0). Keep entities at least 0.2m apart. Example: (0, 1.5, 2) = in front at eye level" } }, required: ["shape"]