mmainguy/immersive2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added chat interface.

Browse Source
This commit is contained in:
Michael Mainguy 2025-12-20 11:25:14 -06:00
parent 54e5017c38
commit e714c3d3df
5 changed files with 794 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
src/react/components/ChatMessage.tsx Normal file
View File

@ -0,0 +1,94 @@
import React from "react";
import {Box, Text, Loader, Badge, Stack} from "@mantine/core";
import {IconCheck, IconX, IconRobot, IconUser} from "@tabler/icons-react";
import {ChatMessage as ChatMessageType} from "../types/chatTypes";
interface ChatMessageProps {
message: ChatMessageType;
}
export default function ChatMessage({message}: ChatMessageProps) {
const isUser = message.role === 'user';
if (message.isLoading) {
return (
<Box
style={{
display: 'flex',
alignItems: 'center',
gap: '0.5rem',
padding: '0.75rem',
marginBottom: '0.5rem',
}}
>
<IconRobot size={20} style={{opacity: 0.7}}/>
<Loader size="sm" type="dots"/>
</Box>
);
}
return (
<Box
style={{
display: 'flex',
flexDirection: 'column',
alignItems: isUser ? 'flex-end' : 'flex-start',
marginBottom: '0.75rem',
}}
>
<Box
style={{
display: 'flex',
alignItems: 'flex-start',
gap: '0.5rem',
flexDirection: isUser ? 'row-reverse' : 'row',
maxWidth: '90%',
}}
>
<Box
style={{
width: 28,
height: 28,
borderRadius: '50%',
backgroundColor: isUser ? 'var(--mantine-color-blue-6)' : 'var(--mantine-color-gray-7)',
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
flexShrink: 0,
}}
>
{isUser ? <IconUser size={16}/> : <IconRobot size={16}/>}
</Box>
<Box
style={{
backgroundColor: isUser ? 'var(--mantine-color-blue-9)' : 'var(--mantine-color-dark-6)',
padding: '0.75rem 1rem',
borderRadius: isUser ? '1rem 1rem 0 1rem' : '1rem 1rem 1rem 0',
}}
>
<Text size="sm" style={{whiteSpace: 'pre-wrap'}}>
{message.content}
</Text>
</Box>
</Box>
{message.toolResults && message.toolResults.length > 0 && (
<Stack gap="xs" style={{marginTop: '0.5rem', marginLeft: '2.25rem'}}>
{message.toolResults.map((result, index) => (
<Badge
key={index}
leftSection={result.success ? <IconCheck size={12}/> : <IconX size={12}/>}
color={result.success ? 'green' : 'red'}
variant="light"
size="sm"
>
{result.message.length > 50
? result.message.substring(0, 50) + '...'
: result.message}
</Badge>
))}
</Stack>
)}
</Box>
);
}

178
src/react/components/ChatPanel.tsx Normal file
View File

@ -0,0 +1,178 @@
import React, {useState, useRef, useEffect} from "react";
import {
Paper,
ScrollArea,
Textarea,
ActionIcon,
Box,
Text,
Group,
CloseButton,
Loader,
Alert
} from "@mantine/core";
import {IconSend, IconRobot, IconAlertCircle} from "@tabler/icons-react";
import ChatMessage from "./ChatMessage";
import {ChatMessage as ChatMessageType, ToolResult} from "../types/chatTypes";
import {sendMessage, createUserMessage, createAssistantMessage, createLoadingMessage} from "../services/diagramAI";
import log from "loglevel";
const logger = log.getLogger('ChatPanel');
interface ChatPanelProps {
width?: number;
onClose?: () => void;
}
export default function ChatPanel({width = 400, onClose}: ChatPanelProps) {
const [messages, setMessages] = useState<ChatMessageType[]>([
createAssistantMessage("Hello! I can help you create and modify 3D diagrams. Try saying things like:\n\n• \"Add a blue box labeled 'Server'\"\n• \"Create a red sphere called 'Database'\"\n• \"Connect Server to Database\"\n• \"What's in my diagram?\"\n\nHow can I help you today?")
]);
const [inputValue, setInputValue] = useState('');
const [isLoading, setIsLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const scrollAreaRef = useRef<HTMLDivElement>(null);
const textareaRef = useRef<HTMLTextAreaElement>(null);
useEffect(() => {
if (scrollAreaRef.current) {
scrollAreaRef.current.scrollTo({
top: scrollAreaRef.current.scrollHeight,
behavior: 'smooth'
});
}
}, [messages]);
const handleSend = async () => {
const trimmedInput = inputValue.trim();
if (!trimmedInput || isLoading) return;
setError(null);
const userMessage = createUserMessage(trimmedInput);
setMessages(prev => [...prev, userMessage]);
setInputValue('');
setIsLoading(true);
const loadingMessage = createLoadingMessage();
setMessages(prev => [...prev, loadingMessage]);
try {
const allToolResults: ToolResult[] = [];
const {response, toolResults} = await sendMessage(
trimmedInput,
messages,
(result) => {
allToolResults.push(result);
logger.debug('Tool executed:', result);
}
);
setMessages(prev => {
const filtered = prev.filter(m => m.id !== loadingMessage.id);
return [...filtered, createAssistantMessage(response, toolResults)];
});
} catch (err) {
logger.error('Chat error:', err);
setMessages(prev => prev.filter(m => m.id !== loadingMessage.id));
setError(err instanceof Error ? err.message : 'Failed to send message');
} finally {
setIsLoading(false);
}
};
const handleKeyDown = (e: React.KeyboardEvent) => {
if (e.key === 'Enter' && !e.shiftKey) {
e.preventDefault();
handleSend();
}
};
return (
<Paper
shadow="xl"
style={{
width,
height: '100vh',
display: 'flex',
flexDirection: 'column',
borderLeft: '1px solid var(--mantine-color-dark-4)',
backgroundColor: 'var(--mantine-color-dark-7)',
}}
>
{/* Header */}
<Box
style={{
padding: '1rem',
borderBottom: '1px solid var(--mantine-color-dark-4)',
flexShrink: 0,
}}
>
<Group justify="space-between">
<Group gap="xs">
<IconRobot size={24}/>
<Text fw={600}>Diagram Assistant</Text>
</Group>
{onClose && <CloseButton onClick={onClose}/>}
</Group>
</Box>
{/* Messages */}
<ScrollArea
style={{flex: 1}}
viewportRef={scrollAreaRef}
p="md"
>
{messages.map((message) => (
<ChatMessage key={message.id} message={message}/>
))}
</ScrollArea>
{/* Error Alert */}
{error && (
<Alert
color="red"
icon={<IconAlertCircle size={16}/>}
withCloseButton
onClose={() => setError(null)}
style={{margin: '0 1rem'}}
>
{error}
</Alert>
)}
{/* Input */}
<Box
style={{
padding: '1rem',
borderTop: '1px solid var(--mantine-color-dark-4)',
flexShrink: 0,
}}
>
<Group gap="xs" align="flex-end">
<Textarea
ref={textareaRef}
placeholder="Describe what you want to create..."
value={inputValue}
onChange={(e) => setInputValue(e.currentTarget.value)}
onKeyDown={handleKeyDown}
autosize
minRows={1}
maxRows={4}
disabled={isLoading}
style={{flex: 1}}
/>
<ActionIcon
size="lg"
variant="filled"
onClick={handleSend}
disabled={!inputValue.trim() || isLoading}
loading={isLoading}
>
<IconSend size={18}/>
</ActionIcon>
</Group>
</Box>
</Paper>
);
}

283
src/react/services/diagramAI.ts Normal file
View File

@ -0,0 +1,283 @@
import {ChatMessage, DiagramToolCall, ToolResult} from "../types/chatTypes";
import {createEntity, connectEntities, removeEntity, modifyEntity, listEntities} from "./entityBridge";
import {v4 as uuidv4} from 'uuid';
const SYSTEM_PROMPT = `You are a 3D diagram assistant helping users create and modify diagrams in a virtual reality environment.
Available entity shapes: box, sphere, cylinder, cone, plane, person
Available colors: red, blue, green, yellow, orange, purple, cyan, pink, white, black, brown, gray (or any hex color like #ff5500)
Position coordinates:
- x: left/right (negative = left, positive = right)
- y: up/down (1.5 is eye level, 0 is floor)
- z: forward/backward (positive = toward user, negative = away)
When creating diagrams, think about good spatial layout:
- Spread entities apart to avoid overlap (at least 0.5 units)
- Use y=1.5 for entities at eye level
- Use z=2 to z=4 for comfortable viewing distance
Always use the provided tools to create, modify, or interact with entities. Be concise in your responses.`;
const TOOLS = [
{
name: "create_entity",
description: "Create a 3D shape in the diagram. Use this to add new elements like boxes, spheres, cylinders, etc.",
input_schema: {
type: "object",
properties: {
shape: {
type: "string",
enum: ["box", "sphere", "cylinder", "cone", "plane", "person"],
description: "The type of 3D shape to create"
},
color: {
type: "string",
description: "Color name (red, blue, green, etc.) or hex code (#ff0000)"
},
label: {
type: "string",
description: "Text label to display on or near the entity"
},
position: {
type: "object",
properties: {
x: {type: "number", description: "Left/right position"},
y: {type: "number", description: "Up/down position (1.5 = eye level)"},
z: {type: "number", description: "Forward/backward position"}
},
description: "3D position. If not specified, defaults to (0, 1.5, 2)"
}
},
required: ["shape"]
}
},
{
name: "connect_entities",
description: "Draw a connection line between two entities. Use entity IDs or labels to identify them.",
input_schema: {
type: "object",
properties: {
from: {
type: "string",
description: "ID or label of the source entity"
},
to: {
type: "string",
description: "ID or label of the target entity"
},
color: {
type: "string",
description: "Color of the connection line"
}
},
required: ["from", "to"]
}
},
{
name: "list_entities",
description: "List all entities currently in the diagram. Use this to see what exists before connecting or modifying.",
input_schema: {
type: "object",
properties: {}
}
},
{
name: "remove_entity",
description: "Remove an entity from the diagram by its ID or label.",
input_schema: {
type: "object",
properties: {
target: {
type: "string",
description: "ID or label of the entity to remove"
}
},
required: ["target"]
}
},
{
name: "modify_entity",
description: "Modify an existing entity's properties like color, label, or position.",
input_schema: {
type: "object",
properties: {
target: {
type: "string",
description: "ID or label of the entity to modify"
},
color: {
type: "string",
description: "New color for the entity"
},
label: {
type: "string",
description: "New label text"
},
position: {
type: "object",
properties: {
x: {type: "number"},
y: {type: "number"},
z: {type: "number"}
}
}
},
required: ["target"]
}
}
];
interface ClaudeMessage {
role: 'user' | 'assistant';
content: string | ClaudeContentBlock[];
}
interface ClaudeContentBlock {
type: 'text' | 'tool_use' | 'tool_result';
text?: string;
id?: string;
name?: string;
input?: Record<string, unknown>;
tool_use_id?: string;
content?: string;
}
interface ClaudeResponse {
content: ClaudeContentBlock[];
stop_reason: 'end_turn' | 'tool_use' | 'max_tokens';
}
async function executeToolCall(toolCall: DiagramToolCall): Promise<ToolResult> {
switch (toolCall.name) {
case 'create_entity':
return createEntity(toolCall.input);
case 'connect_entities':
return connectEntities(toolCall.input);
case 'remove_entity':
return removeEntity(toolCall.input);
case 'modify_entity':
return modifyEntity(toolCall.input);
case 'list_entities':
return await listEntities();
default:
return {
toolName: 'unknown',
success: false,
message: 'Unknown tool'
};
}
}
export async function sendMessage(
userMessage: string,
conversationHistory: ChatMessage[],
onToolResult?: (result: ToolResult) => void
): Promise<{response: string; toolResults: ToolResult[]}> {
const messages: ClaudeMessage[] = conversationHistory
.filter(m => !m.isLoading)
.map(m => ({
role: m.role,
content: m.content
}));
messages.push({role: 'user', content: userMessage});
const allToolResults: ToolResult[] = [];
let finalResponse = '';
let continueLoop = true;
while (continueLoop) {
const response = await fetch('/api/claude/v1/messages', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
model: 'claude-sonnet-4-20250514',
max_tokens: 1024,
system: SYSTEM_PROMPT,
tools: TOOLS,
messages
})
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`API error: ${response.status} - ${errorText}`);
}
const data: ClaudeResponse = await response.json();
const textBlocks = data.content.filter(b => b.type === 'text');
const toolBlocks = data.content.filter(b => b.type === 'tool_use');
if (textBlocks.length > 0) {
finalResponse = textBlocks.map(b => b.text).join('\n');
}
if (data.stop_reason === 'tool_use' && toolBlocks.length > 0) {
messages.push({
role: 'assistant',
content: data.content
});
const toolResults: ClaudeContentBlock[] = [];
for (const toolBlock of toolBlocks) {
const toolCall: DiagramToolCall = {
name: toolBlock.name as DiagramToolCall['name'],
input: toolBlock.input as DiagramToolCall['input']
};
const result = await executeToolCall(toolCall);
allToolResults.push(result);
onToolResult?.(result);
toolResults.push({
type: 'tool_result',
tool_use_id: toolBlock.id,
content: result.message
});
}
messages.push({
role: 'user',
content: toolResults
});
} else {
continueLoop = false;
}
}
return {response: finalResponse, toolResults: allToolResults};
}
export function createUserMessage(content: string): ChatMessage {
return {
id: uuidv4(),
role: 'user',
content,
timestamp: new Date()
};
}
export function createAssistantMessage(content: string, toolResults?: ToolResult[]): ChatMessage {
return {
id: uuidv4(),
role: 'assistant',
content,
timestamp: new Date(),
toolResults
};
}
export function createLoadingMessage(): ChatMessage {
return {
id: uuidv4(),
role: 'assistant',
content: '',
timestamp: new Date(),
isLoading: true
};
}

164
src/react/services/entityBridge.ts Normal file
View File

@ -0,0 +1,164 @@
import {DiagramEntity, DiagramEntityType, DiagramTemplates} from "../../diagram/types/diagramEntity";
import {
CreateEntityParams,
ConnectEntitiesParams,
RemoveEntityParams,
ModifyEntityParams,
SHAPE_TO_TEMPLATE,
COLOR_NAME_TO_HEX,
ToolResult
} from "../types/chatTypes";
import {v4 as uuidv4} from 'uuid';
function resolveColor(color?: string): string {
if (!color) return '#0000ff';
const lower = color.toLowerCase();
if (COLOR_NAME_TO_HEX[lower]) {
return COLOR_NAME_TO_HEX[lower];
}
if (color.startsWith('#')) {
return color;
}
return '#0000ff';
}
export function createEntity(params: CreateEntityParams): ToolResult {
const id = 'id' + uuidv4();
const template = SHAPE_TO_TEMPLATE[params.shape];
const color = resolveColor(params.color);
const position = params.position || {x: 0, y: 1.5, z: 2};
const entity: DiagramEntity = {
id,
template,
type: DiagramEntityType.ENTITY,
color,
text: params.label || '',
position,
rotation: {x: 0, y: Math.PI, z: 0},
scale: {x: 0.1, y: 0.1, z: 0.1},
};
const event = new CustomEvent('chatCreateEntity', {
detail: {entity},
bubbles: true
});
document.dispatchEvent(event);
return {
toolName: 'create_entity',
success: true,
message: `Created ${params.shape}${params.label ? ` labeled "${params.label}"` : ''} at position (${position.x.toFixed(1)}, ${position.y.toFixed(1)}, ${position.z.toFixed(1)})`,
entityId: id
};
}
export function connectEntities(params: ConnectEntitiesParams): ToolResult {
const id = 'id' + uuidv4();
const color = resolveColor(params.color);
const entity: DiagramEntity = {
id,
template: DiagramTemplates.CONNECTION,
type: DiagramEntityType.ENTITY,
color,
from: params.from,
to: params.to,
};
const event = new CustomEvent('chatCreateEntity', {
detail: {entity},
bubbles: true
});
document.dispatchEvent(event);
return {
toolName: 'connect_entities',
success: true,
message: `Connected "${params.from}" to "${params.to}"`,
entityId: id
};
}
export function removeEntity(params: RemoveEntityParams): ToolResult {
const event = new CustomEvent('chatRemoveEntity', {
detail: {target: params.target},
bubbles: true
});
document.dispatchEvent(event);
return {
toolName: 'remove_entity',
success: true,
message: `Removed entity "${params.target}"`
};
}
export function modifyEntity(params: ModifyEntityParams): ToolResult {
const updates: Partial<DiagramEntity> = {};
if (params.color) {
updates.color = resolveColor(params.color);
}
if (params.label !== undefined) {
updates.text = params.label;
}
if (params.position) {
updates.position = params.position;
}
const event = new CustomEvent('chatModifyEntity', {
detail: {target: params.target, updates},
bubbles: true
});
document.dispatchEvent(event);
return {
toolName: 'modify_entity',
success: true,
message: `Modified entity "${params.target}"`
};
}
export function listEntities(): Promise<ToolResult> {
return new Promise((resolve) => {
const responseHandler = (e: CustomEvent) => {
document.removeEventListener('chatListEntitiesResponse', responseHandler as EventListener);
const entities = e.detail.entities as Array<{id: string; template: string; text: string; position: {x: number; y: number; z: number}}>;
if (entities.length === 0) {
resolve({
toolName: 'list_entities',
success: true,
message: 'The diagram is empty.'
});
return;
}
const list = entities.map(e => {
const shape = e.template.replace('#', '').replace('-template', '');
return `- ${e.text || '(no label)'} (${shape}) at (${e.position?.x?.toFixed(1) || 0}, ${e.position?.y?.toFixed(1) || 0}, ${e.position?.z?.toFixed(1) || 0}) [id: ${e.id}]`;
}).join('\n');
resolve({
toolName: 'list_entities',
success: true,
message: `Current entities in the diagram:\n${list}`
});
};
document.addEventListener('chatListEntitiesResponse', responseHandler as EventListener);
const event = new CustomEvent('chatListEntities', {bubbles: true});
document.dispatchEvent(event);
setTimeout(() => {
document.removeEventListener('chatListEntitiesResponse', responseHandler as EventListener);
resolve({
toolName: 'list_entities',
success: false,
message: 'Failed to list entities (timeout)'
});
}, 5000);
});
}

75
src/react/types/chatTypes.ts Normal file
View File

@ -0,0 +1,75 @@
import {DiagramEntity, DiagramTemplates} from "../../diagram/types/diagramEntity";
export type ChatRole = 'user' | 'assistant';
export interface ChatMessage {
id: string;
role: ChatRole;
content: string;
timestamp: Date;
toolResults?: ToolResult[];
isLoading?: boolean;
}
export interface ToolResult {
toolName: string;
success: boolean;
message: string;
entityId?: string;
}
export interface CreateEntityParams {
shape: 'box' | 'sphere' | 'cylinder' | 'cone' | 'plane' | 'person';
color?: string;
label?: string;
position?: { x: number; y: number; z: number };
}
export interface ConnectEntitiesParams {
from: string;
to: string;
color?: string;
}
export interface RemoveEntityParams {
target: string;
}
export interface ModifyEntityParams {
target: string;
color?: string;
label?: string;
position?: { x: number; y: number; z: number };
}
export type DiagramToolCall =
| { name: 'create_entity'; input: CreateEntityParams }
| { name: 'connect_entities'; input: ConnectEntitiesParams }
| { name: 'remove_entity'; input: RemoveEntityParams }
| { name: 'modify_entity'; input: ModifyEntityParams }
| { name: 'list_entities'; input: Record<string, never> };
export const SHAPE_TO_TEMPLATE: Record<CreateEntityParams['shape'], DiagramTemplates> = {
box: DiagramTemplates.BOX,
sphere: DiagramTemplates.SPHERE,
cylinder: DiagramTemplates.CYLINDER,
cone: DiagramTemplates.CONE,
plane: DiagramTemplates.PLANE,
person: DiagramTemplates.PERSON,
};
export const COLOR_NAME_TO_HEX: Record<string, string> = {
red: '#ff0000',
green: '#00ff00',
blue: '#0000ff',
yellow: '#ffff00',
orange: '#ffa500',
purple: '#4b0082',
cyan: '#00ffff',
pink: '#ff69b4',
white: '#ffffff',
black: '#222222',
brown: '#8b4513',
gray: '#778899',
grey: '#778899',
};