mmainguy/immersive2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Re added quest link.

Browse Source
This commit is contained in:
Michael Mainguy 2024-03-29 15:24:50 -05:00
parent d212281ee8
commit 73dae9c1cd
19 changed files with 1771 additions and 465 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
index.html
View File

@ -18,27 +18,26 @@
<body> <body>
<img id="loadingGrid" src="/assets/grid3.jpg"/> <img id="loadingGrid" src="/assets/grid3.jpg"/>
<script> <script>
/* /*
var SpeechRecognition = SpeechRecognition || webkitSpeechRecognition var SpeechRecognition = SpeechRecognition || webkitSpeechRecognition
var SpeechGrammarList = SpeechGrammarList || window.webkitSpeechGrammarList var SpeechGrammarList = SpeechGrammarList || window.webkitSpeechGrammarList
var SpeechRecognitionEvent = SpeechRecognitionEvent || webkitSpeechRecognitionEvent var SpeechRecognitionEvent = SpeechRecognitionEvent || webkitSpeechRecognitionEvent
var recognition = new SpeechRecognition(); var recognition = new SpeechRecognition();
recognition.continuous = false; recognition.continuous = false;
recognition.lang = 'en-US'; recognition.lang = 'en-US';
recognition.interimResults = true; recognition.interimResults = true;
recognition.maxAlternatives = 1; recognition.maxAlternatives = 1;
recognition.onresult = function(event) { recognition.onresult = function(event) {
console.log(event.results[0][0].transcript); console.log(event.results[0][0].transcript);
} }
recognition.onend = function() { recognition.onend = function() {
console.log("recognition ended"); console.log("recognition ended");
recognition.start(); recognition.start();
} }
console.log("starting recognition"); console.log("starting recognition");
recognition.start(); recognition.start();
*/ */
</script> </script>
<div class="webApp" id="webApp"> <div class="webApp" id="webApp">

1678
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

23
package.json
View File

@ -8,6 +8,7 @@
}, },
"scripts": { "scripts": {
"dev": "cp ./node_modules/@babylonjs/havok/lib/esm/HavokPhysics.wasm ./node_modules/.vite/deps && vite", "dev": "cp ./node_modules/@babylonjs/havok/lib/esm/HavokPhysics.wasm ./node_modules/.vite/deps && vite",
"test": "vitest",
"build": "vite build", "build": "vite build",
"preview": "vite preview", "preview": "vite preview",
"serve": "node server.js", "serve": "node server.js",
@ -15,7 +16,7 @@
"havok": "cp ./node_modules/@babylonjs/havok/lib/esm/HavokPhysics.wasm ./node_modules/.vite/deps" "havok": "cp ./node_modules/@babylonjs/havok/lib/esm/HavokPhysics.wasm ./node_modules/.vite/deps"
}, },
"dependencies": { "dependencies": {
"axios": "^0.24.0", "axios": "^1.6.8",
"@babylonjs/core": "^6.45.1", "@babylonjs/core": "^6.45.1",
"@babylonjs/gui": "^6.45.1", "@babylonjs/gui": "^6.45.1",
"@babylonjs/havok": "1.3.1", "@babylonjs/havok": "1.3.1",
@ -24,31 +25,33 @@
"@babylonjs/materials": "^6.45.1", "@babylonjs/materials": "^6.45.1",
"@babylonjs/procedural-textures": "^6.45.1", "@babylonjs/procedural-textures": "^6.45.1",
"@babylonjs/serializers": "^6.45.1", "@babylonjs/serializers": "^6.45.1",
"@cloudflare/workers-types": "^4.20230821.0",
"@netlify/functions": "^2.3.0",
"events": "^3.3.0", "events": "^3.3.0",
"@typed-mxgraph/typed-mxgraph": "^1.0.8", "@typed-mxgraph/typed-mxgraph": "^1.0.8",
"@types/node": "^18.14.0", "@types/node": "^18.14.0",
"earcut": "^2.2.4", "earcut": "^2.2.4",
"file-saver": "^2.0.5", "file-saver": "^2.0.5",
"@types/file-saver": "^2.0.6", "@types/file-saver": "^2.0.6",
"@picovoice/eagle-web": "^1.0.0",
"@picovoice/web-voice-processor": "^4.0.9",
"@picovoice/cobra-web": "^2.0.3",
"hls.js": "^1.1.4", "hls.js": "^1.1.4",
"loglevel": "^1.8.1", "loglevel": "^1.8.1",
"mxgraph": "^4.2.2",
"niceware": "^4.0.0", "niceware": "^4.0.0",
"pouchdb": "^8.0.1", "pouchdb": "^8.0.1",
"pouchdb-find": "^7.2.2", "pouchdb-find": "^8.0.1",
"query-string": "^8.1.0", "query-string": "^8.1.0",
"recordrtc": "^5.6.0",
"react-dom": "^18.2.0", "react-dom": "^18.2.0",
"@types/react-dom": "^18.2.18", "rfc4648": "^1.5.3",
"@types/react": "^18.2.52", "@types/react-dom": "^18.2.22",
"@types/react": "^18.2.72",
"round": "^2.0.1", "round": "^2.0.1",
"uuid": "^9.0.0" "uuid": "^9.0.1"
}, },
"devDependencies": { "devDependencies": {
"typescript": "^4.9.5", "typescript": "^4.9.5",
"vite": "^4.4.9", "vite": "^5.2.6",
"vite-plugin-api": "^0.1.11", "vitest": "^1.4.0",
"vite-plugin-cp": "^1.0.0" "vite-plugin-cp": "^1.0.0"
} }
} }

BIN
public/voice/eagle_params.pv Normal file
View File

Binary file not shown.

4
server.js
View File

@ -2,14 +2,12 @@ import express from "express";
import ViteExpress from "vite-express"; import ViteExpress from "vite-express";
import dotenv from "dotenv"; import dotenv from "dotenv";
import expressProxy from "express-http-proxy"; import expressProxy from "express-http-proxy";
dotenv.config(); dotenv.config();
const app = express(); const app = express();
app.use("/api", expressProxy("local.immersiveidea.com")); app.use("/api", expressProxy("local.immersiveidea.com"));
ViteExpress.listen(app, process.env.PORT || 3001, () => console.log("Server is listening...")); ViteExpress.listen(app, process.env.PORT || 3001, () => console.log("Server is listening..."));

47
src/diagram/functions/applyScaling.test.ts Normal file
View File

@ -0,0 +1,47 @@
import {afterEach, describe, expect, it, vi} from 'vitest'
import {applyScaling} from './applyScaling'
import {Vector3} from "@babylonjs/core";
describe('applyScaling', () => {
afterEach(() => {
vi.restoreAllMocks();
})
it('should copy scaling', () => {
const oldMesh = {
scaling: {
clone: () => 'cloned'
}
}
const newMesh = {
scaling: null
}
applyScaling(oldMesh as any, newMesh as any, true, 0)
expect(newMesh.scaling).toBe('cloned')
})
it('scaling to be set to 1,1,1 if snap passed as null', () => {
const spy = vi.spyOn(Vector3, 'One');
//expect(spy).toHaveBeenCalledTimes(1);
const oldMesh = {
scaling: {}
}
const newMesh = {
scaling: null
}
applyScaling(oldMesh as any, newMesh as any, false, null)
expect(newMesh.scaling.x).toBe(1);
expect(newMesh.scaling.y).toBe(1);
expect(newMesh.scaling.z).toBe(1);
})
it('scaling to be set to 2,2,2 snap passed as Vector3(2,2,2)', () => {
const oldMesh = {
scaling: {}
}
const newMesh = {
scaling: new Vector3()
}
applyScaling(oldMesh as any, newMesh as any, false, 2)
expect(newMesh.scaling.x).toBe(2);
expect(newMesh.scaling.y).toBe(2);
expect(newMesh.scaling.z).toBe(2);
})
});

73
src/diagram/functions/buildMeshFromDiagramEntity.test.ts Normal file
View File

@ -0,0 +1,73 @@
import {afterEach, describe, expect, it, vi} from 'vitest'
import {buildMeshFromDiagramEntity} from './buildMeshFromDiagramEntity'
import {DiagramEntityType} from "../types/diagramEntity";
import {Vector3} from "@babylonjs/core";
describe('buildMeshFromDiagramEntity', () => {
afterEach(() => {
vi.restoreAllMocks();
})
it('should return null if entity is null', () => {
const scene = {
getMeshById: () => null
}
const entity = buildMeshFromDiagramEntity(null, scene as any);
expect(entity).toBe(null);
});
it('should return existing mesh if id exists in scene', () => {
const material = 'material';
const scene = {
getMeshById: (id) => {
return {
id: id,
material: material
}
}
}
const dEntity = {
type: DiagramEntityType.USER,
}
const entity = buildMeshFromDiagramEntity(dEntity, scene as any);
expect(entity.material).toBe(material);
});
it('should generate new mesh if id is missing', () => {
vi.mock('../diagramConnection', () => {
const DiagramConnection = vi.fn();
DiagramConnection.prototype.mesh =
{
id: 'id',
material: 'material',
getChildren: vi.fn(),
getScene: vi.fn()
}
return {DiagramConnection}
});
const scene = {
getMeshById: () => {
return null;
},
}
const dEntity = {
type: DiagramEntityType.USER,
template: "#connection-template",
color: "$FF00FF",
position: {x: 1, y: 2, z: 3},
rotation: {x: 4, y: 5, z: 6},
scale: {x: 7, y: 8, z: 9},
text: 'new text'
}
const entity = buildMeshFromDiagramEntity(dEntity, scene as any);
expect(entity.id).toBe('id');
expect(entity.material).toBe('material');
expect(entity.position).toEqual(new Vector3(1, 2, 3));
expect(entity.rotation).toEqual(new Vector3(4, 5, 6));
expect(entity.scaling).toEqual(new Vector3(7, 8, 9));
expect(entity.metadata.text).toEqual('new text');
});
});

11
src/diagram/functions/buildMeshFromDiagramEntity.ts
View File

@ -34,6 +34,7 @@ function createNewInstanceIfNecessary(entity: DiagramEntity, scene: Scene): Abst
} else { } else {
if (entity.template == "#connection-template") { if (entity.template == "#connection-template") {
const connection: DiagramConnection = new DiagramConnection(entity.from, entity.to, entity.id, scene); const connection: DiagramConnection = new DiagramConnection(entity.from, entity.to, entity.id, scene);
logger.debug(`connection.mesh = ${connection.mesh.id}`); logger.debug(`connection.mesh = ${connection.mesh.id}`);
newMesh = connection.mesh; newMesh = connection.mesh;
} else { } else {
@ -58,6 +59,9 @@ function generateId(entity: DiagramEntity) {
function mapMetadata(entity: DiagramEntity, newMesh: AbstractMesh, scene: Scene): AbstractMesh { function mapMetadata(entity: DiagramEntity, newMesh: AbstractMesh, scene: Scene): AbstractMesh {
if (newMesh) { if (newMesh) {
if (!newMesh.metadata) {
newMesh.metadata = {};
}
if (entity.position) { if (entity.position) {
newMesh.position = xyztovec(entity.position); newMesh.position = xyztovec(entity.position);
} }
@ -69,7 +73,12 @@ function mapMetadata(entity: DiagramEntity, newMesh: AbstractMesh, scene: Scene)
} }
} }
if (entity.parent) { if (entity.parent) {
newMesh.parent = scene.getNodeById(entity.parent); const parent_node = scene.getNodeById(entity.parent);
if (parent_node) {
newMesh.parent = parent_node;
newMesh.metadata.parent = entity.parent;
}
} }
if (entity.scale) { if (entity.scale) {
newMesh.scaling = xyztovec(entity.scale); newMesh.scaling = xyztovec(entity.scale);

14
src/information/inputTextView.ts
View File

@ -1,9 +1,10 @@
import {MeshBuilder, Observable, Scene, TransformNode, Vector3, WebXRDefaultExperience} from "@babylonjs/core"; import {MeshBuilder, Observable, Scene, Vector3, WebXRDefaultExperience} from "@babylonjs/core";
import log, {Logger} from "loglevel"; import log, {Logger} from "loglevel";
import {AdvancedDynamicTexture, Control, InputText, VirtualKeyboard} from "@babylonjs/gui"; import {AdvancedDynamicTexture, Control, InputText, VirtualKeyboard} from "@babylonjs/gui";
import {ControllerEventType, Controllers} from "../controllers/controllers"; import {ControllerEventType, Controllers} from "../controllers/controllers";
import {setMenuPosition} from "../util/functions/setMenuPosition"; import {setMenuPosition} from "../util/functions/setMenuPosition";
import {DiaSounds} from "../util/diaSounds"; import {DiaSounds} from "../util/diaSounds";
import {Handle} from "../objects/handle";
export type TextEvent = { export type TextEvent = {
text: string; text: string;
@ -37,10 +38,10 @@ export class InputTextView {
public showVirtualKeyboard() { public showVirtualKeyboard() {
const inputBaseNode = new TransformNode("inputBase", this.scene);
const inputMesh = MeshBuilder.CreatePlane("input", {width: 1, height: .5}, this.scene); const inputMesh = MeshBuilder.CreatePlane("input", {width: 1, height: .5}, this.scene);
inputMesh.parent = inputBaseNode; const handle = new Handle(inputMesh);
inputMesh.rotation.y = Math.PI; setMenuPosition(handle.mesh, this.scene, new Vector3(0, .4, 0));
const advancedTexture = AdvancedDynamicTexture.CreateForMesh(inputMesh, 2048, 1024, false); const advancedTexture = AdvancedDynamicTexture.CreateForMesh(inputMesh, 2048, 1024, false);
const input = new InputText(); const input = new InputText();
@ -58,6 +59,7 @@ export class InputTextView {
advancedTexture.addControl(input); advancedTexture.addControl(input);
const keyboard = VirtualKeyboard.CreateDefaultLayout(); const keyboard = VirtualKeyboard.CreateDefaultLayout();
keyboard.scaleY = 2; keyboard.scaleY = 2;
keyboard.scaleX = 2; keyboard.scaleX = 2;
keyboard.transformCenterY = 0; keyboard.transformCenterY = 0;
@ -98,7 +100,8 @@ export class InputTextView {
this.sounds.exit.play(); this.sounds.exit.play();
} }
}); });
setMenuPosition(inputBaseNode, this.scene, new Vector3(0, .4, 0));
this.sounds.enter.play(); this.sounds.enter.play();
} }
@ -154,5 +157,6 @@ export class InputTextView {
}); });
textInput.focus(); textInput.focus();
} }
} }

17
src/integration/functions/syncDoc.ts Normal file
View File

@ -0,0 +1,17 @@
export function syncDoc(info) {
console.log(info);
console.log(this);
if (info.direction == 'pull') {
const docs = info.change.docs;
for (const doc of docs) {
if (doc._deleted) {
console.log(doc);
this.removeObserver.notifyObservers({id: doc._id, template: doc.template}, 1);
} else {
this.updateObserver.notifyObservers(doc, 1);
}
}
}
}

48
src/integration/nativeVoiceRecognition.ts Normal file
View File

@ -0,0 +1,48 @@
import {Observable} from "@babylonjs/core";
export class NativeVoiceRecognition {
public readonly onTextObservable: Observable<string> = new Observable<string>();
private recognition: SpeechRecognition;
constructor() {
console.log('speech created');
this.onTextObservable = new Observable<string>();
this.setup();
}
public stop() {
this.recognition.stop();
}
private setup() {
//const SpeechRecognition2 = SpeechRecognition || webkitSpeechRecognition
// const SpeechGrammarList = SpeechGrammarList || window.webkitSpeechGrammarList
//const SpeechRecognitionEvent = SpeechRecognitionEvent || webkitSpeechRecognitionEvent
try {
this.recognition = new webkitSpeechRecognition();
} catch (e) {
this.recognition = new SpeechRecognition();
}
this.recognition.continuous = false;
this.recognition.lang = 'en-US';
this.recognition.interimResults = true;
this.recognition.maxAlternatives = 1;
this.recognition.onresult = (event) => {
this.onTextObservable.notifyObservers(event.results[0][0].transcript);
console.log(event.results[0][0].transcript);
}
this.recognition.onend = () => {
console.log("recognition ended");
}
this.recognition.onstart = () => {
console.log("recognition started");
}
console.log("starting recognition");
this.recognition.start();
}
}

115
src/integration/pouchdbPersistenceManager.ts
View File

@ -6,7 +6,9 @@ import {v4 as uuidv4} from 'uuid';
import axios from "axios"; import axios from "axios";
import {DiagramManager} from "../diagram/diagramManager"; import {DiagramManager} from "../diagram/diagramManager";
import log, {Logger} from "loglevel"; import log, {Logger} from "loglevel";
import {syncDoc} from "./functions/syncDoc";
const logger: Logger = log.getLogger('PouchdbPersistenceManager');
export class PouchdbPersistenceManager { export class PouchdbPersistenceManager {
configObserver: Observable<AppConfigType> = new Observable<AppConfigType>(); configObserver: Observable<AppConfigType> = new Observable<AppConfigType>();
updateObserver: Observable<DiagramEntity> = new Observable<DiagramEntity>(); updateObserver: Observable<DiagramEntity> = new Observable<DiagramEntity>();
@ -16,8 +18,12 @@ export class PouchdbPersistenceManager {
private remote: PouchDB; private remote: PouchDB;
private diagramListings: PouchDB; private diagramListings: PouchDB;
private readonly logger: Logger = log.getLogger('PouchdbPersistenceManager');
private user: string;
constructor() { constructor() {
logger.setLevel('debug');
this.diagramListings = new PouchDB("diagramListings"); this.diagramListings = new PouchDB("diagramListings");
} }
@ -78,7 +84,7 @@ export class PouchdbPersistenceManager {
try { try {
this.db.put(newEntity); this.db.put(newEntity);
} catch (err) { } catch (err) {
this.logger.error(err); logger.error(err);
} }
} }
@ -90,7 +96,7 @@ export class PouchdbPersistenceManager {
const doc = await this.db.get(id); const doc = await this.db.get(id);
this.db.remove(doc); this.db.remove(doc);
} catch (err) { } catch (err) {
this.logger.error(err); logger.error(err);
} }
} }
@ -104,9 +110,10 @@ export class PouchdbPersistenceManager {
this.db.put(newDoc); this.db.put(newDoc);
} catch (err) { } catch (err) {
this.logger.error(err); logger.error(err);
} }
} }
public async getNewRelicData(): Promise<any[]> { public async getNewRelicData(): Promise<any[]> {
return []; return [];
} }
@ -161,7 +168,7 @@ export class PouchdbPersistenceManager {
try { try {
await this.setConfig(defaultConfig, true); await this.setConfig(defaultConfig, true);
} catch (err) { } catch (err) {
this.logger.error(err); logger.error(err);
} }
this.diagramListings.put({_id: defaultConfig.currentDiagramId, name: "New Diagram"}); this.diagramListings.put({_id: defaultConfig.currentDiagramId, name: "New Diagram"});
@ -171,11 +178,13 @@ export class PouchdbPersistenceManager {
} }
try { try {
const all = await this.db.allDocs({include_docs: true}); const all = await this.db.allDocs({include_docs: true});
for (const entity of all.rows) { for (const entity of all.rows) {
logger.debug(entity.doc);
this.updateObserver.notifyObservers(entity.doc, 1); this.updateObserver.notifyObservers(entity.doc, 1);
} }
} catch (err) { } catch (err) {
this.logger.error(err); logger.error(err);
} }
} }
@ -188,27 +197,12 @@ export class PouchdbPersistenceManager {
return null; return null;
} }
} }
syncDoc = function (info) {
this.logger.info(info);
if (info.direction == 'pull') {
const docs = info.change.docs;
for (const doc of docs) {
if (doc._deleted) {
this.removeObserver.notifyObservers({id: doc._id, template: doc.template}, 1);
} else {
this.updateObserver.notifyObservers(doc, 1);
}
}
}
}
async changeColor(oldColor: Color3, newColor: Color3) { async changeColor(oldColor: Color3, newColor: Color3) {
const all = await this.db.allDocs({include_docs: true}); const all = await this.db.allDocs({include_docs: true});
for (const entity of all.rows) { for (const entity of all.rows) {
this.logger.debug(`comparing ${entity.doc.color} to ${oldColor.toHexString()}`); logger.debug(`comparing ${entity.doc.color} to ${oldColor.toHexString()}`);
if (entity.doc.color == oldColor.toHexString()) { if (entity.doc.color == oldColor.toHexString()) {
entity.doc.color = newColor.toHexString(); entity.doc.color = newColor.toHexString();
this.db.put({...entity.doc, _rev: entity.doc._rev}); this.db.put({...entity.doc, _rev: entity.doc._rev});
@ -223,32 +217,71 @@ export class PouchdbPersistenceManager {
private async beginSync(remoteDbName: string) { private async beginSync(remoteDbName: string) {
try { try {
//const remoteDbName = "db1"; //const remoteDbName = "db1";
const remoteUserName = remoteDbName; const userHex = remoteDbName.split('-');
if (userHex.length < 2) {
return;
}
const username = hex_to_ascii(userHex[1]);
const remoteUserName = username;
const password = "password"; const password = "password";
const dbs = await axios.get(import.meta.env.VITE_SYNCDB_ENDPOINT + '_all_dbs'); const dbs = await axios.get(import.meta.env.VITE_SYNCDB_ENDPOINT + 'list');
logger.debug(dbs.data);
if (dbs.data.indexOf(remoteDbName) == -1) { if (dbs.data.indexOf(remoteDbName) == -1) {
this.logger.warn('sync target missing'); logger.warn('sync target missing');
const userEndpoint: string = import.meta.env.VITE_USER_ENDPOINT return;
console.log(userEndpoint); }
console.log(remoteDbName); const userEndpoint: string = import.meta.env.VITE_USER_ENDPOINT
const buildTarget = await axios.post(userEndpoint, logger.debug(userEndpoint);
{username: remoteUserName, password: password, db: remoteDbName}); logger.debug(remoteDbName);
if (buildTarget.status != 200) { const target = await axios.get(userEndpoint);
this.logger.info(buildTarget.statusText); if (target.status != 200) {
return; logger.info(target.statusText);
return;
}
if (target.data && target.data.userCtx) {
if (!target.data.userCtx.name) {
const buildTarget = await axios.post(userEndpoint,
{username: remoteUserName, password: password});
if (buildTarget.status != 200) {
logger.info(buildTarget.statusText);
return;
} else {
this.user = buildTarget.data.userCtx;
logger.debug(this.user);
}
} }
} }
this.logger.debug(dbs);
const remoteEndpoint: string = import.meta.env.VITE_SYNCDB_ENDPOINT;
console.log(remoteEndpoint);
this.remote = new PouchDB(remoteEndpoint + remoteDbName,
{auth: {username: remoteUserName, password: password}});
this.syncDoc = this.syncDoc.bind(this);
const remoteEndpoint: string = import.meta.env.VITE_SYNCDB_ENDPOINT;
console.log(remoteEndpoint + remoteDbName);
this.remote = new PouchDB(remoteEndpoint + remoteDbName,
{auth: {username: remoteUserName, password: password}, skip_setup: true});
const dbInfo = await this.remote.info();
console.log(dbInfo);
syncDoc.bind(this);
this.db.sync(this.remote, {live: true, retry: true}) this.db.sync(this.remote, {live: true, retry: true})
.on('change', this.syncDoc); .on('change', syncDoc)
.on('active', function (info) {
console.log('sync active', info)
})
.on('paused', function (info) {
console.log('sync paused', info)
})
.on('error', function (err) {
console.log('sync error', err)
});
} catch (err) { } catch (err) {
this.logger.error(err); logger.error(err);
} }
} }
} }
function hex_to_ascii(input) {
var hex = input.toString();
let output = '';
for (var n = 0; n < hex.length; n += 2) {
output += String.fromCharCode(parseInt(hex.substr(n, 2), 16));
}
return output;
}

9
src/menus/scaleMenu.ts
View File

@ -19,18 +19,13 @@ export class ScaleMenu extends AbstractMenu {
constructor(scene: Scene, xr: WebXRDefaultExperience, controllers: Controllers) { constructor(scene: Scene, xr: WebXRDefaultExperience, controllers: Controllers) {
super(scene, xr, controllers); super(scene, xr, controllers);
this.transformNode = new TransformNode("scaleMenu", scene); this.transformNode = new TransformNode("scaleMenu", scene);
this.xTransformNode = new TransformNode("xTransformNode", scene); this.xTransformNode = new TransformNode("xTransformNode", scene);
this.xTransformNode.parent = this.transformNode; this.xTransformNode.parent = this.transformNode;
this.yTransformNode = new TransformNode("yTransformNode", scene); this.yTransformNode = new TransformNode("yTransformNode", scene);
this.yTransformNode.parent = this.transformNode; this.yTransformNode.parent = this.transformNode;
this.zTransformNode = new TransformNode("zTransformNode", scene); this.zTransformNode = new TransformNode("zTransformNode", scene);
this.zTransformNode.parent = this.transformNode; this.zTransformNode.parent = this.transformNode;
//super.createHandle(this.transformNode); //super.createHandle(this.transformNode);
this.transformNode.position.y = 0; this.transformNode.position.y = 0;
this.transformNode.position.z = 0; this.transformNode.position.z = 0;
@ -59,8 +54,8 @@ export class ScaleMenu extends AbstractMenu {
//manager.rootContainer.position.y = 2; //manager.rootContainer.position.y = 2;
//manager.rootContainer.node.position.y = 2; //manager.rootContainer.node.position.y = 2;
this.xSlider = new Slider3D("xslider"); this.xSlider = new Slider3D("xslider");
this.ySlider = new Slider3D("xslider"); this.ySlider = new Slider3D("yslider");
this.zSlider = new Slider3D("xslider"); this.zSlider = new Slider3D("zslider");
manager.addControl(this.xSlider); manager.addControl(this.xSlider);
manager.addControl(this.ySlider); manager.addControl(this.ySlider);

10
src/util/functions/getFrontPosition.ts
View File

@ -1,7 +1,13 @@
import {Scene, Vector3} from "@babylonjs/core"; import {MeshBuilder, Scene, Vector3} from "@babylonjs/core";
const debug = false;
export function getFrontPosition(distance: number, scene: Scene): Vector3 { export function getFrontPosition(distance: number, scene: Scene): Vector3 {
const offset = new Vector3(0, 0, distance); const offset = new Vector3(0, 0, distance);
offset.applyRotationQuaternionInPlace(scene.activeCamera.absoluteRotation); offset.applyRotationQuaternionInPlace(scene.activeCamera.absoluteRotation);
return scene.activeCamera.globalPosition.add(offset); const newPos = scene.activeCamera.globalPosition.add(offset);
if (debug) {
const mesh = MeshBuilder.CreateIcoSphere("front", {radius: .1}, scene);
mesh.position = newPos;
}
return newPos;
} }

8
src/util/functions/groundMeshObserver.ts
View File

@ -5,9 +5,11 @@ import {EditMenu} from "../../menus/editMenu";
import {ControllerEventType} from "../../controllers/controllers"; import {ControllerEventType} from "../../controllers/controllers";
import {ConfigMenu} from "../../menus/configMenu"; import {ConfigMenu} from "../../menus/configMenu";
const logger = log.getLogger('groungMeshObserver');
export async function groundMeshObserver(ground, scene, diagramManager, controllers, spinner) { export async function groundMeshObserver(ground, scene, diagramManager, controllers, spinner) {
const xr = await WebXRDefaultExperience.CreateAsync(scene, { const xr = await WebXRDefaultExperience.CreateAsync(scene, {
floorMeshes: [ground], floorMeshes: [ground],
disableHandTracking: true,
disableTeleportation: true, disableTeleportation: true,
disableDefaultUI: true, disableDefaultUI: true,
outputCanvasOptions: { outputCanvasOptions: {
@ -32,6 +34,7 @@ export async function groundMeshObserver(ground, scene, diagramManager, controll
enterButton.style.display = "block"; enterButton.style.display = "block";
enterButton.addEventListener('click', (evt) => { enterButton.addEventListener('click', (evt) => {
evt.preventDefault(); evt.preventDefault();
//const voice = new VoiceRecognizer();
xr.baseExperience.enterXRAsync('immersive-vr', 'local-floor'); xr.baseExperience.enterXRAsync('immersive-vr', 'local-floor');
}); });
} }
@ -44,7 +47,8 @@ export async function groundMeshObserver(ground, scene, diagramManager, controll
xr.baseExperience.sessionManager.onXRSessionInit.add((session) => { xr.baseExperience.sessionManager.onXRSessionInit.add((session) => {
session.addEventListener('visibilitychange', (ev) => { session.addEventListener('visibilitychange', (ev) => {
this.logger.debug(ev); logger.debug(ev);
//this.logger.debug(ev);
}); });
}); });
@ -57,7 +61,7 @@ export async function groundMeshObserver(ground, scene, diagramManager, controll
//xr.baseExperience.camera.setTarget(new Vector3(0, 1.6, 3)); //xr.baseExperience.camera.setTarget(new Vector3(0, 1.6, 3));
window.addEventListener(('pa-button-state-change'), (event: any) => { window.addEventListener(('pa-button-state-change'), (event: any) => {
if (event.detail) { if (event.detail) {
log.debug('App', event.detail); logger.debug(event.detail);
} }
}); });
break; break;

13
src/util/functions/setMenuPosition.ts
View File

@ -47,19 +47,22 @@ export function setMenuPosition(node: TransformNode, scene: Scene, offset: Vecto
} }
const debug = true;
function setPosition(node: TransformNode, scene: Scene, offset: Vector3 = Vector3.Zero()) { function setPosition(node: TransformNode, scene: Scene, offset: Vector3 = Vector3.Zero()) {
const platform = scene.getMeshByName("platform"); const platform = scene.getNodeById("platform");
switch (scene.activeCamera.getClassName()) { switch (scene.activeCamera.getClassName()) {
case "WebXRCamera": case "WebXRCamera":
//const oldParent = node.parent; //const oldParent = node.parent;
//console.log(oldParent.name); //console.log(oldParent.name);
node.setParent(null); node.setParent(null);
const front = getFrontPosition(1, scene); const front = getFrontPosition(1, scene).clone();
const camPos = scene.activeCamera.globalPosition.clone(); const camPos = scene.activeCamera.globalPosition.clone();
node.position.x = front.x + offset.x; const newPos = new Vector3(front.x + offset.x, 1.2 + offset.y, front.z + offset.z);
node.position.z = front.z + offset.z; node.position = newPos;
node.position.y = 1.2 + offset.y;
node.lookAt(camPos); node.lookAt(camPos);
// const target = MeshBuilder.CreateIcoSphere("target", {radius: .1}, scene);
// target.position = newPos;
// target.setParent(platform);
node.setParent(platform); node.setParent(platform);
break; break;
case "FreeCamera": case "FreeCamera":

97
src/util/voiceRecognizer.ts Normal file
View File

@ -0,0 +1,97 @@
import {VuMeterEngine, WebVoiceProcessor} from "@picovoice/web-voice-processor";
import {EagleProfilerEnrollFeedback, EagleProfilerWorker, EagleWorker} from "@picovoice/eagle-web";
import {PvEngine} from "@picovoice/web-voice-processor/dist/types/types";
import {base64} from "rfc4648";
import {CobraWorker} from "@picovoice/cobra-web";
export class VoiceRecognizer {
private readonly profile = 'TT4GLAFik3sHYasWTbmNc7CX2xmJifp897E6xyJDwcD8rHA1ZgJTdLaLQ0Z1zJP3Lzv61JMJpuNsgSSxraWf62RECXJHA4SJ5RCSv3/MHXcnKlaGzcQ4gJD/vcl8tL45j02me6lNf1TtbI81qv/GBSH9u3p5V/p7fD1j/tZ8P3kzM40FG1QxP330Sp/dvSwU6oEc5iH7+D3Zsy4GEtD3iqGdAex4200elW753eXlKli92qLlaplDxCZZIwNG0/ER51NhVzg/D+ieZLE/rUjZAl+z4a9c7AurnQDTZ1fKyzwMzjiQsb8h6rIco+IlblAuE8dmVfFxZXrpfw8tViK5KwnpbHrnR8ebLA0Zd/G5l0yjXeCEh8Y26qkGZk43MolBgQ044KNNsJbDrm1o2sYGvo/BgEronFuBB+wNw0pVFqp1nrIqfT8IsH42q4rj7ByzxH+4QVr1uQVx7c89GTp1yYyqk+q9B+f0cG063KwnUofddkx0tmot3d8kPCZGL1H88qMm2NdNeCU6AFi1qhP1Ssd3jOsyzd4O3U9JMJIUruySzf6Qx3/JI4aSKJm0To/xEszrm028s1qD8I0+Kh3GxgtiW+toCzJrAIoRkxmIgV2HbOStjerTQDK2t1MMHpHEr7NfNaJxxripR2xDlw/r1Sh2sf58AhIYBk1b/U54PCsm7tWKc0YroWgO5qSpASdkIq2UX7AgQ6nWtcZPk5d5xdfRkhIy3Yf6yushwEz/+v3aa/fshkh9lbKnz2wt4MhmbMJv7WYenHQr0RvEDQFchNHUyx8D7fBLpKwrmHVDi0meG7pl0Q9DrFzykeBZi0m22T6xF5OUBqvUZWdcqa5ZoYvPZS5Uk0d3COhAGjgFEAL7IMK9rZei24mM98+vqyfD1RRK29rQFljHxO9lJ0N3NINW2PfZulPPqn0OtEhxU829W1k1JjOUgOucJUpI9D0H1vrcqTYq0F4mj3YTqJ0CNbqXEan+XlbjB6kGKDuO1V/YkLbShJRnmKVtaCS2m4zDCJ5Rc7x8J7E/Cx6AnVK7UWM9CLnnLoaJ6sib9UnBV1uSTK2BIp1mO8b+BjP2yJ2/l1xbYUPZQa/ECRarwDP9PY9lHym6WGf35BjuWBwxT6obifw9JyYaqFZxwNbKviqKshSW8wmh5euYu3s0hY/MhCD+ZCYZiJAXADArcv8z8Wlk1x/KmZwLOAKJfgDUcs+9Q8aKGq5/jTPcd6MuM5pAVwfoFMR9QD8uKmyrPuBdJMRZFULX0uqliyna2CTmsAJ+6B47AA61q1/50Vj8OTUBLi+fhIaw3Ch1ofMYopIqc+QT81ekH8b9/+pXaUi+moA6C53Mnch9hB/uKoJELPSNDpIy1fP08Ujv/K0Foft4X43w9dImveOEhw==';
public constructor() {
this.start();
}
public async start() {
let voicePresent = false;
const eagleProfiler = await EagleProfilerWorker.create(
'qQG03oXEGQRfPbX7H1VTZHLy/zelmMxcWXSy14/pskqri4LTJvBWmQ==',
{publicPath: '/voice/eagle_params.pv'}
)
const bytes = base64.parse(this.profile);
const eagle = await EagleWorker.create('qQG03oXEGQRfPbX7H1VTZHLy/zelmMxcWXSy14/pskqri4LTJvBWmQ=='
, {publicPath: '/voice/eagle_params.pv'},
[{bytes: bytes}]);
let data: Int16Array = null;
const engine: PvEngine = {
onmessage: async (e: MessageEvent) => {
if (!voicePresent) {
return;
}
switch (e.data.command) {
case 'process':
const inputData = e.data.inputFrame;
const validation = await eagle.process(inputData);
if (validation && validation[0] > .99) {
console.log('verified');
}
console.log(validation);
if (data == null) {
data = inputData;
} else {
data = new Int16Array(data.length + inputData.length);
data.set(data, 0);
data.set(inputData, data.length - inputData.length);
}
if (data.length > eagleProfiler.minEnrollSamples) {
console.log('enrolling');
console.log(data.length);
const result = await eagleProfiler.enroll(data);
if (result.percentage >= 100) {
console.log(data);
console.log(EagleProfilerEnrollFeedback[result.feedback]);
const profile = await eagleProfiler.export();
//console.log(profile.bytes.buffer);
console.log(base64.stringify(profile.bytes));
//console.log(profile.bytes);
//console.log(profile.bytes.buffer.toString('base64'));
//console.log(profile);
//WebVoiceProcessor.unsubscribe(engine);
//WebVoiceProcessor.unsubscribe(engine);
//eagleProfiler.release();
}
}
console.log(inputData.length);
break;
}
}
}
const devices = await navigator.mediaDevices.enumerateDevices();
devices.forEach((device) => {
if (device.kind === 'audioinput') {
console.log(device.label + ' ' + device.kind + ' ' + device.deviceId);
}
});
const vuCallback = (db) => {
console.log(db);
}
WebVoiceProcessor.setOptions({deviceId: 'default'});
const vuEngine = new VuMeterEngine(vuCallback);
//WebVoiceProcessor.subscribe(vuEngine);
const cobra = await CobraWorker.create('qQG03oXEGQRfPbX7H1VTZHLy/zelmMxcWXSy14/pskqri4LTJvBWmQ==',
(isVoice) => {
voicePresent = (isVoice && isVoice > .85);
});
WebVoiceProcessor.subscribe(cobra);
WebVoiceProcessor.subscribe(engine);
}
}

24
src/vrApp.ts
View File

@ -79,31 +79,7 @@ export class VrApp {
const gamepadManager = new GamepadManager(scene); const gamepadManager = new GamepadManager(scene);
/* /*
const voiceManager = new VoiceManager();
voiceManager.transcriptionObserver.add((text) => {
logger.info('Transcription', text);
switch (text.type) {
case TranscriptType.PartialTranscript:
if (text.words.length > 0 &&
text.words[0].text.toLowerCase() == 'meta') {
logger.info('Meta command', text.text);
}
break;
case TranscriptType.FinalTranscript:
logger.info('Final', text.words[0].text.toLowerCase().substring(0, 4));
if (text.words.length > 0 &&
text.words[0].text.toLowerCase().substring(0, 4) == 'meta' &&
text.words[0].confidence > .8) {
logger.info('Meta Final command',
text.words.map((e) => {
return e.text
}).slice(1).join(' '));
}
}
});
*/ */
addSceneInspector(scene); addSceneInspector(scene);

8
vite.config.ts
View File

@ -1,7 +1,9 @@
/// <reference types="vitest" />
import {defineConfig} from "vite"; import {defineConfig} from "vite";
/** @type {import('vite').UserConfig} */ /** @type {import('vite').UserConfig} */
export default defineConfig({ export default defineConfig({
test: {},
define: {}, define: {},
optimizeDeps: { optimizeDeps: {
esbuildOptions: { esbuildOptions: {
@ -12,10 +14,10 @@ export default defineConfig({
}, },
server: { server: {
port: 3001, port: 3001,
proxy: { proxy: {
'/.netlify': { '^/sync/.*': {
target: 'http://localhost:9999/', target: 'https://www.deepdiagram.com/',
changeOrigin: true,
} }
} }
}, },