mmainguy/immersive2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added Initial Voice Command Handler, fixed connection issues.

Browse Source
This commit is contained in:
Michael Mainguy 2023-08-14 13:27:20 -05:00
parent 8251681035
commit 64060ef573
8 changed files with 64 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
functions/api/voice/token.ts
View File

@ -1,11 +0,0 @@
interface Env {
VOICE_TOKEN: string;
}
const handler: ExportedHandler<Env> = {
async fetch(request, context) {
console.log(context.VOICE_TOKEN);
return new Response('Hello World 2!');
}
}
export default handler;

2
index.html
View File

@ -25,7 +25,7 @@
</style> </style>
</head> </head>
<body> <body>
<div class="loader">Loading...</div> <div class="loader" id="loader">Loading...</div>
<div id="graphContainer"></div> <div id="graphContainer"></div>
<script type="module" src="./src/app.ts"></script> <script type="module" src="./src/app.ts"></script>
</body> </body>

27
src/app.ts
View File

@ -13,8 +13,6 @@ import {DiagramManager} from "./diagram/diagramManager";
import {Toolbox} from "./toolbox/toolbox"; import {Toolbox} from "./toolbox/toolbox";
import log from "loglevel"; import log from "loglevel";
import {AppConfig} from "./util/appConfig"; import {AppConfig} from "./util/appConfig";
import {PeerjsNetworkConnection} from "./integration/peerjsNetworkConnection";
import {InputTextView} from "./information/inputTextView";
import {GamepadManager} from "./controllers/gamepadManager"; import {GamepadManager} from "./controllers/gamepadManager";
import {CustomEnvironment} from "./util/customEnvironment"; import {CustomEnvironment} from "./util/customEnvironment";
import {DrawioManager} from "./integration/drawioManager"; import {DrawioManager} from "./integration/drawioManager";
@ -46,6 +44,10 @@ export class App {
log.debug('App', 'gameCanvas created'); log.debug('App', 'gameCanvas created');
this.initialize(canvas).then(() => { this.initialize(canvas).then(() => {
logger.debug('App', 'Scene Initialized'); logger.debug('App', 'Scene Initialized');
const loader = document.querySelector('#loader');
if (loader) {
loader.remove();
}
}); });
} }
@ -56,24 +58,6 @@ export class App {
const scene = new Scene(engine); const scene = new Scene(engine);
const environment = new CustomEnvironment(scene); const environment = new CustomEnvironment(scene);
const query = Object.fromEntries(new URLSearchParams(window.location.search));
logger.debug('Query', query);
if (query.shareCode) {
scene.onReadyObservable.addOnce(() => {
logger.debug('Scene ready');
const identityView = new InputTextView({scene: scene, text: ""});
identityView.onTextObservable.add((text) => {
if (text?.text?.trim() != "") {
logger.debug('Identity', text.text);
const network = new PeerjsNetworkConnection(query.shareCode, text.text);
if (query.host) {
network.connect(query.host);
}
}
});
identityView.show();
});
}
const camera: ArcRotateCamera = new ArcRotateCamera("Camera", Math.PI / 2, Math.PI / 2, 2, const camera: ArcRotateCamera = new ArcRotateCamera("Camera", Math.PI / 2, Math.PI / 2, 2,
new Vector3(0, 1.6, 0), scene); new Vector3(0, 1.6, 0), scene);
@ -111,7 +95,7 @@ export class App {
const diagramManager = new DiagramManager(scene, xr.baseExperience); const diagramManager = new DiagramManager(scene, xr.baseExperience);
const rig = new Rigplatform(scene, xr, diagramManager); const rig = new Rigplatform(scene, xr, diagramManager);
const toolbox = new Toolbox(scene, xr.baseExperience, diagramManager); const toolbox = new Toolbox(scene, xr.baseExperience, diagramManager);
const dioManager = new DrawioManager(scene, diagramManager); //const dioManager = new DrawioManager(scene, diagramManager);
import ('./integration/indexdbPersistenceManager').then((module) => { import ('./integration/indexdbPersistenceManager').then((module) => {
const persistenceManager = new module.IndexdbPersistenceManager("diagram"); const persistenceManager = new module.IndexdbPersistenceManager("diagram");
diagramManager.setPersistenceManager(persistenceManager); diagramManager.setPersistenceManager(persistenceManager);
@ -168,6 +152,7 @@ export class App {
logger.info('keydown event listener added, use Ctrl+Shift+Alt+I to toggle debug layer'); logger.info('keydown event listener added, use Ctrl+Shift+Alt+I to toggle debug layer');
engine.runRenderLoop(() => { engine.runRenderLoop(() => {
scene.render(); scene.render();
}); });

2
src/diagram/diagramConnection.ts
View File

@ -24,6 +24,7 @@ export class DiagramConnection {
if (fromMesh) { if (fromMesh) {
this.fromAnchor = fromMesh; this.fromAnchor = fromMesh;
} }
const toMesh = this.scene.getMeshById(to); const toMesh = this.scene.getMeshById(to);
if (toMesh) { if (toMesh) {
this.toAnchor = toMesh; this.toAnchor = toMesh;
@ -142,6 +143,7 @@ export class DiagramConnection {
this.fromAnchor = null; this.fromAnchor = null;
} }
if (this._mesh) { if (this._mesh) {
this._mesh.dispose();
this._mesh = null; this._mesh = null;
} }
if (this.scene) { if (this.scene) {

18
src/diagram/diagramManager.ts
View File

@ -66,6 +66,21 @@ export class DiagramManager {
} }
this.onDiagramEventObservable.add(this.onDiagramEvent, -1, true, this); this.onDiagramEventObservable.add(this.onDiagramEvent, -1, true, this);
this.logger.debug("DiagramManager constructed"); this.logger.debug("DiagramManager constructed");
scene.onMeshRemovedObservable.add((mesh) => {
if (mesh?.metadata?.template) {
if (mesh.metadata.template != '#connection-template') {
scene.meshes.forEach((m) => {
if (m?.metadata?.to == mesh.id || m?.metadata?.from == mesh.id) {
this.logger.debug("removing connection", m.id);
this.onDiagramEventObservable.notifyObservers({
type: DiagramEventType.REMOVE,
entity: MeshConverter.toDiagramEntity(m)
});
}
});
}
}
});
} }
public createCopy(mesh: AbstractMesh, copy: boolean = false): AbstractMesh { public createCopy(mesh: AbstractMesh, copy: boolean = false): AbstractMesh {
let newMesh; let newMesh;
@ -114,7 +129,7 @@ export class DiagramManager {
private onRemoteEvent(event: DiagramEntity) { private onRemoteEvent(event: DiagramEntity) {
this.logger.debug(event); this.logger.debug(event);
const toolMesh = this.scene.getMeshById("tool-" + event.template + "-" + event.color); const toolMesh = this.scene.getMeshById("tool-" + event.template + "-" + event.color);
if (!toolMesh) { if (!toolMesh && (event.template != '#connection-template')) {
log.debug('no mesh found for ' + event.template + "-" + event.color, 'adding it'); log.debug('no mesh found for ' + event.template + "-" + event.color, 'adding it');
this.onDiagramEventObservable.notifyObservers({ this.onDiagramEventObservable.notifyObservers({
type: DiagramEventType.CHANGECOLOR, type: DiagramEventType.CHANGECOLOR,
@ -203,6 +218,7 @@ export class DiagramManager {
mesh?.physicsBody?.dispose(); mesh?.physicsBody?.dispose();
mesh.dispose(); mesh.dispose();
DiaSounds.instance.exit.play(); DiaSounds.instance.exit.play();
} }
break; break;
} }

8
src/diagram/meshConverter.ts
View File

@ -35,10 +35,10 @@ export class MeshConverter {
return entity; return entity;
} }
public static fromDiagramEntity(entity: DiagramEntity, scene: Scene): AbstractMesh { public static fromDiagramEntity(entity: DiagramEntity, scene: Scene): AbstractMesh {
if (!entity) { if (!entity) {
this.logger.error("fromDiagramEntity: entity is null"); this.logger.error("fromDiagramEntity: entity is null");
return null; return null;
} }
if (!entity.id) { if (!entity.id) {
entity.id = "id" + uuidv4(); entity.id = "id" + uuidv4();
} }

2
src/integration/peerjsNetworkConnection.ts
View File

@ -6,7 +6,6 @@ export class PeerjsNetworkConnection {
private dataChannel: P2PDataChannel<any>; private dataChannel: P2PDataChannel<any>;
constructor(dataChannel: string, identity: string) { constructor(dataChannel: string, identity: string) {
const config = { const config = {
debug: false, debug: false,
dataChannel: dataChannel, dataChannel: dataChannel,
@ -15,7 +14,6 @@ export class PeerjsNetworkConnection {
pingTimeout: 8000 pingTimeout: 8000
} }
this.dataChannel = new P2PDataChannel(identity, config); this.dataChannel = new P2PDataChannel(identity, config);
this.dataChannel.onConnected((peerId) => { this.dataChannel.onConnected((peerId) => {
this.logger.debug(peerId, ' connected'); this.logger.debug(peerId, ' connected');
}); });

54
src/menus/editMenu.ts
View File

@ -34,6 +34,7 @@ export class EditMenu {
private readonly xr: WebXRExperienceHelper; private readonly xr: WebXRExperienceHelper;
private readonly diagramManager: DiagramManager; private readonly diagramManager: DiagramManager;
private connection: DiagramConnection = null; private connection: DiagramConnection = null;
private panel: StackPanel3D;
constructor(scene: Scene, xr: WebXRExperienceHelper, diagramManager: DiagramManager) { constructor(scene: Scene, xr: WebXRExperienceHelper, diagramManager: DiagramManager) {
this.scene = scene; this.scene = scene;
@ -46,7 +47,20 @@ export class EditMenu {
this.gizmoManager.gizmos.boundingBoxGizmo.scaleDragSpeed = 2; this.gizmoManager.gizmos.boundingBoxGizmo.scaleDragSpeed = 2;
this.gizmoManager.clearGizmoOnEmptyPointerEvent = true; this.gizmoManager.clearGizmoOnEmptyPointerEvent = true;
this.gizmoManager.usePointerToAttachGizmos = false; this.gizmoManager.usePointerToAttachGizmos = false;
this.manager = new GUI3DManager(this.scene);
const panel = new StackPanel3D();
this.manager.addControl(panel);
panel.addControl(this.makeButton("Modify", "modify"));
panel.addControl(this.makeButton("Remove", "remove"));
panel.addControl(this.makeButton("Add Label", "label"));
panel.addControl(this.makeButton("Copy", "copy"));
panel.addControl(this.makeButton("Connect", "connect"));
panel.addControl(this.makeButton("Export", "export"));
panel.addControl(this.makeButton("Recolor", "recolor"));
//panel.addControl(this.makeButton("Add Ring Cameras", "addRingCameras"));
this.manager.controlScaling = .5;
this.scene.onPointerObservable.add((pointerInfo) => { this.scene.onPointerObservable.add((pointerInfo) => {
switch (pointerInfo.type) { switch (pointerInfo.type) {
case PointerEventTypes.POINTERPICK: case PointerEventTypes.POINTERPICK:
@ -71,32 +85,32 @@ export class EditMenu {
} }
} }
}); });
this.panel = panel;
this.isVisible = false;
}
private get isVisible(): boolean {
return this.panel.isVisible;
}
private set isVisible(visible: boolean) {
this.panel.isVisible = visible;
this.panel.children.forEach((child) => {
child.isVisible = visible;
});
} }
toggle() { toggle() {
if (this.manager) { if (this.isVisible) {
DiaSounds.instance.exit.play(); DiaSounds.instance.exit.play();
this.manager.dispose(); this.isVisible = false;
this.manager = null;
} else { } else {
DiaSounds.instance.enter.play(); DiaSounds.instance.enter.play();
this.manager = new GUI3DManager(this.scene); CameraHelper.setMenuPosition(this.manager.rootContainer.children[0].node, this.scene);
const panel = new StackPanel3D(); this.isVisible = true;
this.manager.addControl(panel);
panel.addControl(this.makeButton("Modify", "modify"));
panel.addControl(this.makeButton("Remove", "remove"));
panel.addControl(this.makeButton("Add Label", "label"));
panel.addControl(this.makeButton("Copy", "copy"));
panel.addControl(this.makeButton("Connect", "connect"));
panel.addControl(this.makeButton("Export", "export"));
panel.addControl(this.makeButton("Recolor", "recolor"));
//panel.addControl(this.makeButton("Add Ring Cameras", "addRingCameras"));
this.manager.controlScaling = .5;
CameraHelper.setMenuPosition(panel.node, this.scene);
} }
} }
private getTool(template: string, color: Color3): Mesh { private getTool(template: string, color: Color3): Mesh {
const baseMeshId = 'tool-' + template + '-' + color.toHexString(); const baseMeshId = 'tool-' + template + '-' + color.toHexString();
return (this.scene.getMeshById(baseMeshId) as Mesh); return (this.scene.getMeshById(baseMeshId) as Mesh);
@ -275,7 +289,7 @@ export class EditMenu {
return; return;
} }
this.manager.dispose();
this.manager = null; this.isVisible = false;
} }
} }