mmainguy/immersive2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add WebXR system keyboard support and improve entity positioning

Browse Source 
- Add SystemKeyboardInput class for Meta Quest native keyboard in VR
- Integrate system keyboard into DiagramMenuManager with auto-detection
- Falls back to virtual keyboard when system keyboard not supported
- Update AI system prompt with entity positioning guidelines:
  - Default position near (0, 1.5, 0) at eye level
  - Minimum 0.2m spacing between entities
  - Entities placed in front of camera with positive z values

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Michael Mainguy 2026-01-14 14:45:42 -06:00
parent 4ca98cf980
commit 719c969f72
3 changed files with 226 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
src/diagram/diagramMenuManager.ts
View File

@ -1,6 +1,7 @@
import {DiagramEntityType, DiagramEvent, DiagramEventType} from "./types/diagramEntity";
import {AbstractMesh, ActionEvent, Observable, Scene, Vector3, WebXRDefaultExperience, WebXRInputSource} from "@babylonjs/core";
import {InputTextView} from "../information/inputTextView";
import {SystemKeyboardInput} from "../information/systemKeyboardInput";
import {DefaultScene} from "../defaultScene";
import log from "loglevel";
import {Toolbox} from "../toolbox/toolbox";
@ -19,6 +20,8 @@ export class DiagramMenuManager {
public readonly toolbox: Toolbox;
private readonly _notifier: Observable<DiagramEvent>;
private readonly _inputTextView: InputTextView;
private readonly _systemKeyboardInput: SystemKeyboardInput;
private _useSystemKeyboard: boolean = false;
private readonly _vrConfigPanel: VRConfigPanel;
private _groupMenu: GroupMenu;
private readonly _scene: Scene;
@ -31,16 +34,22 @@ export class DiagramMenuManager {
this._scene = DefaultScene.Scene;
this._notifier = notifier;
this._inputTextView = new InputTextView(controllerObservable);
this._systemKeyboardInput = new SystemKeyboardInput();
this._vrConfigPanel = new VRConfigPanel(this._scene);
//this.configMenu = new ConfigMenu(config);
this._inputTextView.onTextObservable.add((evt) => {
// Handler for text input changes (shared between both input methods)
const handleTextEvent = (evt) => {
const event = {
type: DiagramEventType.MODIFY,
entity: {id: evt.id, text: evt.text, type: DiagramEntityType.ENTITY}
}
this._notifier.notifyObservers(event, DiagramEventObserverMask.FROM_DB);
});
};
this._inputTextView.onTextObservable.add(handleTextEvent);
this._systemKeyboardInput.onTextObservable.add(handleTextEvent);
this.toolbox = new Toolbox(readyObservable);
if (viewOnly()) {
@ -87,8 +96,14 @@ export class DiagramMenuManager {
}
public editText(mesh: AbstractMesh) {
if (this._useSystemKeyboard) {
this._logger.debug('Using system keyboard for text input');
this._systemKeyboardInput.show(mesh);
} else {
this._logger.debug('Using virtual keyboard for text input');
this._inputTextView.show(mesh);
}
}
public activateResizeGizmo(mesh: AbstractMesh) {
// Dispose existing gizmo if any
@ -164,6 +179,21 @@ export class DiagramMenuManager {
public setXR(xr: WebXRDefaultExperience): void {
this._xr = xr;
this.toolbox.setXR(xr, this);
// Listen for XR session start to detect system keyboard support
xr.baseExperience.onStateChangedObservable.add((state) => {
if (state === 2) { // WebXRState.IN_XR
const session = xr.baseExperience.sessionManager.session;
this._useSystemKeyboard = SystemKeyboardInput.isSupported(session);
this._logger.debug('System keyboard supported:', this._useSystemKeyboard);
if (this._useSystemKeyboard) {
this._systemKeyboardInput.setSession(session);
}
} else if (state === 0) { // WebXRState.NOT_IN_XR
this._useSystemKeyboard = false;
}
});
}
public toggleVRConfigPanel(): void {

158
src/information/systemKeyboardInput.ts Normal file
View File

@ -0,0 +1,158 @@
import {AbstractMesh, Observable} from "@babylonjs/core";
import log, {Logger} from "loglevel";
import {TextEvent} from "./inputTextView";
/**
* SystemKeyboardInput provides text input using the WebXR system keyboard (Meta Quest native keyboard).
* This is an alternative to the VirtualKeyboard-based InputTextView for devices that support
* the WebXR system keyboard API.
*
* Usage:
* - Check isSupported() before using
* - Call show(mesh) to display the system keyboard
* - Listen to onTextObservable for text changes
* - Call dispose() when XR session ends
*/
export class SystemKeyboardInput {
private logger: Logger = log.getLogger('SystemKeyboardInput');
public readonly onTextObservable: Observable<TextEvent> = new Observable<TextEvent>();
private inputElement: HTMLInputElement | null = null;
private currentMeshId: string | null = null;
private xrSession: XRSession | null = null;
private visibilityHandler: ((ev: XRSessionEvent) => void) | null = null;
constructor() {
this.logger.debug('SystemKeyboardInput created');
}
/**
* Check if the system keyboard is supported for the given XR session.
*/
public static isSupported(session: XRSession | null): boolean {
if (!session) {
return false;
}
// TypeScript doesn't have this property in the type definitions yet
return (session as any).isSystemKeyboardSupported === true;
}
/**
* Initialize with an XR session. Should be called when entering immersive mode.
*/
public setSession(session: XRSession): void {
this.xrSession = session;
// Set up visibility change listener for keyboard lifecycle
this.visibilityHandler = (ev: XRSessionEvent) => {
const visState = (ev.target as XRSession).visibilityState;
this.logger.debug('XR visibility changed:', visState);
if (visState === 'visible' && this.inputElement) {
// Keyboard was dismissed, finalize input
this.finalizeInput();
}
};
session.addEventListener('visibilitychange', this.visibilityHandler);
// Clean up when session ends
session.addEventListener('end', () => {
this.dispose();
});
}
/**
* Show the system keyboard for editing the given mesh's text.
*/
public show(mesh: AbstractMesh): void {
if (!this.xrSession) {
this.logger.warn('Cannot show system keyboard: no XR session');
return;
}
this.currentMeshId = mesh.id;
// Create input element if it doesn't exist
if (!this.inputElement) {
this.inputElement = document.createElement('input');
this.inputElement.type = 'text';
this.inputElement.id = 'system-keyboard-input';
// Position off-screen to avoid visual interference
// but keep it in DOM for keyboard to work
this.inputElement.style.position = 'fixed';
this.inputElement.style.left = '-9999px';
this.inputElement.style.top = '0';
this.inputElement.style.opacity = '0';
this.inputElement.style.pointerEvents = 'none';
// Handle blur event (keyboard dismissed)
this.inputElement.onblur = () => {
this.logger.debug('Input blurred');
this.finalizeInput();
};
document.body.appendChild(this.inputElement);
}
// Set initial value from mesh metadata
const initialText = mesh.metadata?.text || '';
this.inputElement.value = initialText;
this.logger.debug('Showing system keyboard for mesh:', mesh.id, 'initial text:', initialText);
// Focus to trigger the system keyboard
this.inputElement.focus();
}
/**
* Hide the keyboard and discard any changes.
*/
public hide(): void {
if (this.inputElement) {
this.inputElement.blur();
}
this.currentMeshId = null;
}
/**
* Finalize the input and notify observers.
*/
private finalizeInput(): void {
if (!this.currentMeshId || !this.inputElement) {
return;
}
const text = this.inputElement.value.trim();
this.logger.debug('Finalizing input:', text, 'for mesh:', this.currentMeshId);
// Notify observers with the final text (or null if empty)
this.onTextObservable.notifyObservers({
id: this.currentMeshId,
text: text.length > 0 ? text : null
});
this.currentMeshId = null;
}
/**
* Clean up resources. Should be called when XR session ends.
*/
public dispose(): void {
this.logger.debug('Disposing SystemKeyboardInput');
if (this.visibilityHandler && this.xrSession) {
this.xrSession.removeEventListener('visibilitychange', this.visibilityHandler);
this.visibilityHandler = null;
}
if (this.inputElement) {
this.inputElement.remove();
this.inputElement = null;
}
this.xrSession = null;
this.currentMeshId = null;
}
}

54
src/react/services/diagramAI.ts
View File

@ -17,84 +17,97 @@ export interface ModelInfo {
name: string;
description: string;
provider: AIProvider;
contextLimit: number; // Maximum context window size in tokens
}
const AVAILABLE_MODELS: ModelInfo[] = [
// Claude models
// Claude models - 200K context
{
id: 'claude-sonnet-4-20250514',
name: 'Claude Sonnet 4',
description: 'Balanced performance and speed (default)',
provider: 'claude'
provider: 'claude',
contextLimit: 200000
},
{
id: 'claude-opus-4-20250514',
name: 'Claude Opus 4',
description: 'Most capable, best for complex tasks',
provider: 'claude'
provider: 'claude',
contextLimit: 200000
},
{
id: 'claude-haiku-3-5-20241022',
name: 'Claude Haiku 3.5',
description: 'Fastest responses, good for simple tasks',
provider: 'claude'
provider: 'claude',
contextLimit: 200000
},
// Cloudflare Workers AI models - with tool support
{
id: '@cf/mistralai/mistral-small-3.1-24b-instruct',
name: 'Mistral Small 3.1 (CF)',
description: 'Best CF model - supports diagram tools',
provider: 'cloudflare'
provider: 'cloudflare',
contextLimit: 32000
},
{
id: '@hf/nousresearch/hermes-2-pro-mistral-7b',
name: 'Hermes 2 Pro (CF)',
description: 'Lightweight - supports diagram tools',
provider: 'cloudflare'
provider: 'cloudflare',
contextLimit: 8000
},
// Cloudflare models WITHOUT tool support - chat only
{
id: '@cf/meta/llama-3.3-70b-instruct-fp8-fast',
name: 'Llama 3.3 70B (CF)',
description: 'Powerful but NO tool support',
provider: 'cloudflare'
provider: 'cloudflare',
contextLimit: 128000
},
{
id: '@cf/meta/llama-3.1-8b-instruct',
name: 'Llama 3.1 8B (CF)',
description: 'Fast/cheap but NO tool support',
provider: 'cloudflare'
provider: 'cloudflare',
contextLimit: 128000
},
{
id: '@cf/deepseek-ai/deepseek-r1-distill-qwen-32b',
name: 'DeepSeek R1 (CF)',
description: 'Reasoning but NO tool support',
provider: 'cloudflare'
provider: 'cloudflare',
contextLimit: 32000
},
{
id: '@cf/qwen/qwen2.5-coder-32b-instruct',
name: 'Qwen 2.5 Coder (CF)',
description: 'Code-focused but NO tool support',
provider: 'cloudflare'
provider: 'cloudflare',
contextLimit: 32000
},
// Ollama models (local)
// Ollama models (local) - context depends on local config, using defaults
{
id: 'llama3.1',
name: 'Llama 3.1',
description: 'Local model with function calling support',
provider: 'ollama'
provider: 'ollama',
contextLimit: 128000
},
{
id: 'mistral',
name: 'Mistral',
description: 'Fast local model with good tool support',
provider: 'ollama'
provider: 'ollama',
contextLimit: 32000
},
{
id: 'qwen2.5',
name: 'Qwen 2.5',
description: 'Capable local model with function calling',
provider: 'ollama'
provider: 'ollama',
contextLimit: 32000
}
];
@ -413,9 +426,12 @@ Example positions:
- (0, 3, 2) = directly in front, above eye level
## Layout Guidelines
- Spread entities apart (at least 0.5 units)
- Use y=1.5 for eye-level entities
- For relative directions, call get_camera_position first
- **Default position**: Place new entities in front of the camera, centered around (0, 1.5, 0) which is eye level
- **Minimum spacing**: Entities must NEVER be closer than 0.2 meters (20cm) from each other
- **Spread layout**: When creating multiple entities, spread them horizontally (x-axis) and vertically (y-axis) with at least 0.5m between them
- Use y=1.5 for eye-level entities (comfortable viewing height)
- Use positive z values (e.g., z=1 to z=3) to place entities in front of the user
- For relative directions (left, right, forward), call get_camera_position first
Be concise. Call tools immediately when the user requests an action.`;
@ -444,9 +460,9 @@ const TOOLS = [
properties: {
x: {type: "number", description: "Left (-) / Right (+) from camera view"},
y: {type: "number", description: "Down (-) / Up (+), 0=floor, 1.5=eye level"},
z: {type: "number", description: "Backward (-) / Forward (+) from camera view"}
z: {type: "number", description: "Backward (-) / Forward (+) from camera view, use positive values (1-3) to place in front"}
},
description: "3D position from camera perspective. Example: (0, 1.5, 2) = in front at eye level"
description: "3D position. Default around (0, 1.5, 0). Keep entities at least 0.2m apart. Example: (0, 1.5, 2) = in front at eye level"
}
},
required: ["shape"]