Added voice manager, integrated cloud function capability.

functions/src/voiceServer.ts

@@ -0,0 +1,4 @@
+export const onRequest: PagesFunction = async (context) => {
+    const response = new Response('Hello World!');
+    return response;
+}

functions/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "compilerOptions": {
+    "target": "esnext",
+    "module": "esnext",
+    "lib": [
+      "esnext"
+    ],
+    "types": [
+      "@cloudflare/workers-types"
+    ]
+  }
+}

package-lock.json

@@ -12,7 +12,9 @@
         "@babylonjs/gui": "^6.15.0",
         "@babylonjs/havok": "1.1.1",
         "@babylonjs/inspector": "^6.15.0",
+        "@babylonjs/loaders": "^6.15.0",
         "@babylonjs/serializers": "^6.15.0",
+        "@cloudflare/workers-types": "^4.20230807.0",
         "@typed-mxgraph/typed-mxgraph": "^1.0.8",
         "dexie": "^3.2.4",
         "dexie-observable": "^4.0.1-beta.13",
@@ -21,6 +23,7 @@
         "mxgraph": "^4.2.2",
         "p2p-data-channel": "^1.10.7",
         "query-string": "^8.1.0",
+        "recordrtc": "^5.6.2",
         "ring-client-api": "11.7.7",
         "round": "^2.0.1",
         "uuid": "^9.0.0"
@@ -97,10 +100,9 @@
       }
     },
     "node_modules/@babylonjs/loaders": {
-      "version": "6.8.0",
+      "version": "6.15.0",
-      "resolved": "https://registry.npmjs.org/@babylonjs/loaders/-/loaders-6.8.0.tgz",
+      "resolved": "https://registry.npmjs.org/@babylonjs/loaders/-/loaders-6.15.0.tgz",
-      "integrity": "sha512-Ze3dsjwQ6ORYTkbacdlZVeN1PvC0KYcN2hzDNAqiQ0eeQkk96QDusgLkVHMM1UW55hS5PQefTua7iylOxWetDw==",
+      "integrity": "sha512-pE22KVLEWcsWPmiL+TFsNyqLTMGhnt/6oPDy3nWpMwHTfjeaoW9qTV7hFDwhXopzU1Ddp6c/DhzFZalgSvCvbg==",
-      "peer": true,
       "peerDependencies": {
         "@babylonjs/core": "^6.0.0",
         "babylonjs-gltf2interface": "^6.0.0"
@@ -124,6 +126,11 @@
         "babylonjs-gltf2interface": "^6.0.0"
       }
     },
+    "node_modules/@cloudflare/workers-types": {
+      "version": "4.20230807.0",
+      "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20230807.0.tgz",
+      "integrity": "sha512-gQczWuGE2rxmpzOCNn0zLbx8Xz0gqspdE9S7tu4Xax39q1csgO/E9flcS+KG3GHB522ugOh84inmABDhpeJnvQ=="
+    },
     "node_modules/@eneris/push-receiver": {
       "version": "3.1.4",
       "resolved": "https://registry.npmjs.org/@eneris/push-receiver/-/push-receiver-3.1.4.tgz",
@@ -3267,6 +3274,11 @@
         "node": ">=8.10.0"
       }
     },
+    "node_modules/recordrtc": {
+      "version": "5.6.2",
+      "resolved": "https://registry.npmjs.org/recordrtc/-/recordrtc-5.6.2.tgz",
+      "integrity": "sha512-1QNKKNtl7+KcwD1lyOgP3ZlbiJ1d0HtXnypUy7yq49xEERxk31PHvE9RCciDrulPCY7WJ+oz0R9hpNxgsIurGQ=="
+    },
     "node_modules/reflect-metadata": {
       "version": "0.1.13",
       "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz",

package.json

@@ -8,6 +8,7 @@
     "build": "vite build",
     "preview": "vite preview",
     "serve": "node server.js",
+    "serverBuild": "cd server && tsc",
     "havok": "cp ./node_modules/@babylonjs/havok/lib/esm/HavokPhysics.wasm ./node_modules/.vite/deps"
   },
   "dependencies": {
@@ -16,6 +17,8 @@
     "@babylonjs/havok": "1.1.1",
     "@babylonjs/inspector": "^6.15.0",
     "@babylonjs/serializers": "^6.15.0",
+    "@babylonjs/loaders": "^6.15.0",
+    "@cloudflare/workers-types": "^4.20230807.0",
     "ring-client-api": "11.7.7",
     "dexie": "^3.2.4",
     "dexie-observable": "^4.0.1-beta.13",
@@ -23,6 +26,7 @@
     "loglevel": "^1.8.1",
     "mxgraph": "^4.2.2",
     "@typed-mxgraph/typed-mxgraph": "^1.0.8",
+    "recordrtc": "^5.6.2",
     "round": "^2.0.1",
     "earcut": "^2.2.4",
     "uuid": "^9.0.0",

src/app.ts

@@ -19,6 +19,7 @@ import {GamepadManager} from "./controllers/gamepadManager";
 import {CustomEnvironment} from "./util/customEnvironment";
 import {DrawioManager} from "./integration/drawioManager";
 
+
 export class App {
     //preTasks = [havokModule];
     constructor() {
@@ -30,9 +31,10 @@ export class App {
         log.getLogger('App').setLevel('info');
         //log.getLogger('IndexdbPersistenceManager').setLevel('info');
         //log.getLogger('DiagramManager').setLevel('info');
-
         //log.getLogger('DiagramConnection').setLevel('debug');
         log.getLogger('DrawioManager').setLevel('debug');
+
+        log.getLogger('EntityTree').setLevel('debug');
         log.getLogger('EditMenu').setLevel('debug');
         const canvas = document.createElement("canvas");
         canvas.style.width = "100%";
@@ -106,7 +108,7 @@ export class App {
             const diagramManager = new DiagramManager(scene, xr.baseExperience);
             const rig = new Rigplatform(scene, xr, diagramManager);
             const toolbox = new Toolbox(scene, xr.baseExperience, diagramManager);
-            //const dioManager = new DrawioManager(scene, diagramManager);
+            const dioManager = new DrawioManager(scene, diagramManager);
             import ('./integration/indexdbPersistenceManager').then((module) => {
                 const persistenceManager = new module.IndexdbPersistenceManager("diagram");
                 diagramManager.setPersistenceManager(persistenceManager);
@@ -114,6 +116,8 @@ export class App {
                 persistenceManager.initialize();
             });
         });
+
+
         const gamepadManager = new GamepadManager(scene);
         window.addEventListener("keydown", (ev) => {
             // Shift+Ctrl+Alt+I
@@ -129,6 +133,7 @@ export class App {
                 });
             }
         });
+
         logger.info('keydown event listener added, use Ctrl+Shift+Alt+I to toggle debug layer');
         engine.runRenderLoop(() => {
             scene.render();

src/controllers/base.ts

@@ -223,7 +223,9 @@ export class Base {
         this.previousRotation = null;
         this.previousPosition = null;
         this.grabbedMesh = null;
-
+        if (mesh?.metadata?.template.indexOf('#') == -1) {
+            return;
+        }
         const entity = MeshConverter.toDiagramEntity(mesh);
         const event: DiagramEvent = {
             type: DiagramEventType.DROP,

src/controllers/rigplatform.ts

@@ -64,10 +64,12 @@ export class Rigplatform {
                 PhysicsShapeType.CYLINDER,
                 {friction: 1, center: Vector3.Zero(), radius: .5, mass: 10, restitution: .01},
                 scene);
+
         rigAggregate.body.setMotionType(PhysicsMotionType.DYNAMIC);
         rigAggregate.body.setGravityFactor(.001);
         this.fixRotation();
         this.body = rigAggregate.body;
+
         this.initializeControllers();
         scene.onActiveCameraChanged.add((s) => {
             this.camera = s.activeCamera;

src/diagram/diagramManager.ts

@@ -159,8 +159,11 @@ export class DiagramManager {
             case DiagramEventType.DROPPED:
                 break;
             case DiagramEventType.DROP:
-                this.getPersistenceManager()?.modify(mesh);
+                if (mesh.metadata.template.indexOf('#') > -1) {
-                TextLabel.updateTextNode(mesh, entity.text);
+                    this.getPersistenceManager()?.modify(mesh);
+                    TextLabel.updateTextNode(mesh, entity.text);
+                }
+
                 break;
             case DiagramEventType.ADD:
                 this.getPersistenceManager()?.add(mesh);
@@ -248,8 +251,8 @@ class DiagramShapePhysics {
         const aggregate = new PhysicsAggregate(mesh,
             shapeType, {mass: mass, restitution: .02, friction: .9}, scene);
         const body = aggregate.body;
-        body.setLinearDamping(.95);
+        body.setLinearDamping(1.95);
-        body.setAngularDamping(.99);
+        body.setAngularDamping(1.99);
 
         if (motionType) {
             body

src/integration/drawioManager.ts

@@ -1,147 +1,291 @@
 import log from "loglevel";
-import {Color3, Scene, TransformNode, Vector3} from "@babylonjs/core";
+import {Color3, Scene, Vector3} from "@babylonjs/core";
 import {DiagramManager} from "../diagram/diagramManager";
 import {DiagramEventType} from "../diagram/diagramEntity";
 
+type DrawIOEntity = {
+    text?: string,
+    id?: string,
+    parent?: string,
+    parentEntity?: DrawIOEntity,
+    geometry?: DrawIOGeometry,
+
+}
+type DrawIOGeometry = {
+    zIndex?: number,
+    x: number,
+    y: number,
+    width: number,
+    height: number
+
+}
+
+class EntityTree {
+    private readonly logger = log.getLogger('EntityTree');
+    private root: DrawIOEntity;
+    private readonly nodes: Map<string, DrawIOEntity> = new Map<string, DrawIOEntity>();
+    private readonly unparented: Array<DrawIOEntity> = new Array<DrawIOEntity>();
+
+    constructor() {
+        this.root = {};
+    }
+
+    public getNodes(): Array<DrawIOEntity> {
+        this.reparent();
+        const output: Array<DrawIOEntity> = new Array<DrawIOEntity>();
+        this.nodes.forEach((node) => {
+            if (node.parentEntity) {
+                const geometry = this.computeOffset(node);
+                node.geometry = geometry;
+            }
+            output.push(node);
+        });
+        return output;
+    }
+
+    public reparent() {
+        this.unparented.forEach((node) => {
+            if (this.nodes.has(node.parent)) {
+                this.logger.debug('reparenting node: ' + node.id + ' to parent: ' + node.parent);
+                node.parentEntity = this.nodes.get(node.parent);
+            } else {
+                this.logger.warn('parent node does not exist for id: ' + node.id +
+                    ' parent id: ' + node.parent);
+            }
+        });
+    }
+
+    public addNode(node: DrawIOEntity) {
+        if (this.nodes.has(node.id)) {
+            this.logger.warn('node already exists for id: ' + node.id);
+        } else {
+            if (node.parent) {
+                if (this.nodes.has(node.parent)) {
+                    node.parentEntity = this.nodes.get(node.parent);
+                    this.nodes.set(node.id, node);
+                } else {
+                    this.logger.warn('parent node does not exist for id: ' + node.id +
+                        ' parent id: ' + node.parent);
+                    this.unparented.push(node);
+                }
+            } else {
+                this.logger.warn('no parent for node id: ' + node.id + 'setting as root');
+                this.nodes.set(node.id, node);
+                this.root = node;
+            }
+        }
+    }
+
+    private computeOffset(node: DrawIOEntity): DrawIOGeometry {
+        if (node.parentEntity) {
+            const parentgeo = this.computeOffset(node.parentEntity);
+            if (parentgeo) {
+                const parentzIndex = 1 + parentgeo.zIndex ? parentgeo.zIndex : 0;
+                const geo = {
+                    x: node.geometry.x,
+                    y: node.geometry.y,
+                    width: node.geometry.width,
+                    height: node.geometry.height,
+                    zIndex: node.geometry.zIndex ? node.geometry.zIndex + parentzIndex : parentzIndex + 1
+                };
+                return geo;
+            } else {
+                const geo = {
+                    x: node.geometry.x,
+                    y: node.geometry.y,
+                    width: node.geometry.width,
+                    height: node.geometry.height,
+                    zIndex: node.geometry.zIndex ? node.geometry.zIndex : 0
+                };
+                return geo;
+            }
+
+        } else {
+            if (node.geometry) {
+                if (node.geometry.zIndex === undefined) {
+                    node.geometry.zIndex = 0;
+                }
+                return node.geometry;
+            } else {
+                return {
+                    x: 0,
+                    y: 0,
+                    width: 0,
+                    height: 0,
+                    zIndex: 0
+                }
+            }
+
+        }
+    }
+}
+
+type DrawIOConnector = {
+    id: string,
+    source: string,
+    target: string,
+    text: string
+}
+
 export class DrawioManager {
     private diagramManager: DiagramManager;
-    private readonly zdepth: Map<string, number> = new Map<string, number>();
+    private connectors: Array<DrawIOConnector> = [];
     private readonly scene: Scene;
     private readonly logger = log.getLogger('DrawioManager');
     private minY = 0;
     private minX = 0;
     private maxX = 0;
     private maxY = 0;
+    private maxZ = 0;
 
     constructor(scene: Scene, diagramManager: DiagramManager) {
         this.scene = scene;
         this.diagramManager = diagramManager;
-        this.getGraph();
+        this.buildGraph();
     }
 
 
-
+    private async fetchData(url: string): Promise<Document> {
-    private async getGraph() {
         this.logger.debug("starting to get graph");
-        const entities: Array<{ text: string, id: string, geometry: { x: number, y: number, width: number, height: number } }>
+        const graph = await fetch(url);
-            = new Array<{ text: string; id: string, geometry: { x: number; y: number; width: number; height: number } }>();
-
-        const graph = await fetch('/arch_demo.xml');
         this.logger.debug('got graph');
         const graphXml = await graph.text();
         const doc = new DOMParser().parseFromString(graphXml, 'text/html');
-        //this.logger.debug(doc);
+        return doc;
-        const firstDiagram = doc.querySelectorAll('diagram')[0];
+    }
-        const mxDiagram = firstDiagram.querySelector('mxGraphModel');
-        const width = mxDiagram.getAttribute('pageWidth');
-        const height = mxDiagram.getAttribute('pageHeight');
-        this.logger.debug('begin parse');
-        mxDiagram.querySelectorAll('mxCell').forEach((cell) => {
 
+    private getDiagram(doc: Document, index: number): Element {
+        const firstDiagram = doc.querySelectorAll('diagram')[index];
+        const mxDiagram = firstDiagram.querySelector('mxGraphModel');
+        return mxDiagram;
+    }
+
+    private parseDiagram(mxDiagram: Element): EntityTree {
+        const entityTree = new EntityTree();
+
+        mxDiagram.querySelectorAll('mxCell').forEach((cell) => {
             const value = cell.getAttribute('value');
-            if (!value) {
+            let ent = null;
-                //this.logger.warn('no value for :' , cell);
+            if (value) {
-            } else {
+                ent = new DOMParser().parseFromString(value, 'text/html');
-                const ent = new DOMParser().parseFromString(value, 'text/html');
                 const errorNode = ent.querySelector("parsererror");
-                this.logger.debug(value);
                 if (errorNode) {
-                    //this.logger.debug(value);
+                    this.logger.error(value);
+                }
+
+            }
+
+            const text = ent ? this.getText(ent, '') : '';
+            const id = cell.getAttribute('id');
+            const parent = cell.getAttribute('parent');
+            const source = cell.getAttribute('source');
+            const target = cell.getAttribute('target');
+            const edge = cell.getAttribute('target');
+            if (source && target && edge) {
+                this.connectors.push({id: id, source: source, target: target, text: text});
+            } else {
+
+                const geo = cell.querySelector('[id="' + id + '"] > mxGeometry');
+                let geometry = null;
+                if (geo) {
+                    geometry = {
+                        x: Number.parseFloat(geo.getAttribute('x')),
+                        y: Number.parseFloat(geo.getAttribute('y')),
+                        width: Number.parseFloat(geo.getAttribute('width')),
+                        height: Number.parseFloat(geo.getAttribute('height')),
+                    }
                 } else {
-                    const text = this.getText(ent, '');
+                    geometry = {
-                    const id = cell.getAttribute('id');
+                        x: 0,
-                    const parent = cell.getAttribute('parent');
+                        y: 0,
-                    if (this.zdepth.has(parent)) {
+                        width: 0,
-                        this.zdepth.set(id, this.zdepth.get(parent) + .2);
+                        height: 0
+                    }
+                }
+
+
+                //entities.push({text: text, id: id, parent: parent, geometry: this.fixMinMax(geometry)});
+                if (text) {
+                    this.logger.debug('Text' + text);
+                    this.logger.debug('Geometry' + JSON.stringify(geometry));
+                }
+                if (geometry) {
+                    if (Number.isNaN(geometry.x) || Number.isNaN(geometry.y)
+                        || Number.isNaN(geometry.width) ||
+                        Number.isNaN(geometry.height)) {
+                        this.logger.warn('invalid geometry for node: ' + id, geometry);
                     } else {
-                        this.zdepth.set(cell.getAttribute('id'), 0);
+                        entityTree.addNode({text: text, id: id, parent: parent, geometry: geometry});
-                    }
+
-                    const geo = cell.querySelector('mxGeometry');
-                    const geometry = {
-                        x: geo.getAttribute('x'),
-                        y: geo.getAttribute('y'),
-                        width: geo.getAttribute('width'),
-                        height: geo.getAttribute('height'),
-                    }
-                    entities.push({text: text, id: id, geometry: this.fixMinMax(geometry)});
-                    if (text) {
-                        this.logger.debug('Text' + text);
-                        this.logger.debug('Geometry' + JSON.stringify(geometry));
                     }
                 }
             }
         });
+        return entityTree
+    }
+
+    private async buildGraph() {
+
+        const doc = await this.fetchData('/arch_demo.xml');
+        const mxDiagram = this.getDiagram(doc, 0);
+        this.logger.debug('begin parse');
+        const entities: EntityTree = this.parseDiagram(mxDiagram);
+
+        entities.getNodes().forEach((node) => {
+            if (node.geometry.x < this.minX) {
+                this.minX = node.geometry.x;
+                this.logger.debug('minX: ' + this.minX);
+            }
+            if (node.geometry.y < this.minY) {
+                this.minY = node.geometry.y;
+                this.logger.debug('minY: ' + this.minY);
+            }
+            if (node.geometry.x + node.geometry.width > this.maxX) {
+                this.maxX = node.geometry.x + node.geometry.width;
+                this.logger.debug('maxX: ' + this.maxX);
+            }
+            if (node.geometry.y + node.geometry.height > this.maxY) {
+                this.maxY = node.geometry.y + node.geometry.height;
+                this.logger.debug('maxY: ' + this.maxY);
+            }
+            if (node.geometry.zIndex > this.maxZ) {
+                this.maxZ = node.geometry.zIndex;
+                this.logger.debug('maxZ: ' + this.maxZ);
+            }
+
+        });
+        this.logger.info('minX: ' + this.minX + ' minY: ' + this.minY + ' maxX: ' + this.maxX + ' maxY: ' + this.maxY);
+
+
         this.logger.debug('done parsing');
-
+        this.logger.debug(this.connectors);
-        this.logger.debug('MinX' + this.minX);
+        this.createSceneData(entities.getNodes());
-        this.logger.debug('MinY' + this.minY);
-        this.logger.debug('MaxX' + this.maxX);
-        this.logger.debug('MaxY' + this.maxY);
-        const diagramWidth = this.maxX - this.minX;
-        const diagramHeight = this.maxY - this.minY;
-        let scale = 1;
-        if (diagramHeight > diagramWidth) {
-            scale = 20 / diagramHeight;
-        } else {
-            scale = 20 / diagramWidth;
-        }
-        const anchor = new TransformNode('anchor', this.scene);
-
-        if (entities.length > 0) {
-            entities.forEach((entity) => {
-                this.diagramManager.onDiagramEventObservable.notifyObservers(
-                    {
-                        type: DiagramEventType.ADD,
-                        entity: {
-                            text: entity.text,
-                            id: entity.id,
-                            position: new Vector3((entity.geometry.x - this.minX) * scale + (entity.geometry.width * scale / 2),
-                                (entity.geometry.y - this.minY) * scale + (entity.geometry.height * scale / 2),
-                                2 + this.zdepth.get(entity.id)),
-                            scale: new Vector3(entity.geometry.width * scale, entity.geometry.height * scale, .1),
-                            color: Color3.Blue().toHexString(),
-                            template: '#box-template'
-                        }
-                    }
-                );
-
-                //box.metadata = {text: entity.text};
-                //box.setParent(anchor);
-                //DrawioManager.updateTextNode(box, entity.text);
-            });
-            anchor.position.y = 20;
-            anchor.rotation.x = Math.PI;
-
-        }
-
-        this.logger.debug('Scale' + scale);
-
 
     }
 
-    private fixMinMax(geometry: { x: string; y: string; width: string; height: string; }):
+    private createSceneData(nodes) {
-        { x: number, y: number, width: number, height: number } {
+        const yOffset = 20;
-        let x = 0;
+        const scale = .001;
-        if (geometry.x) {
+        nodes.forEach((entity) => {
-            x = parseFloat(geometry.x);
+            this.diagramManager.onDiagramEventObservable.notifyObservers(
-            if (x < this.minX) {
+                {
-                this.minX = x;
+                    type: DiagramEventType.ADD,
-            }
+                    entity: {
-            if (x > this.maxX) {
+                        text: entity.text,
-                this.maxX = x;
+                        id: entity.id,
-            }
+                        position: new Vector3(
-        }
+                            (entity.geometry.x * scale) - (entity.geometry.width * scale / 2),
-        let y = 0;
+                            yOffset - (entity.geometry.y * scale) + (entity.geometry.height * scale / 2),
-        if (geometry.y) {
+                            entity.geometry.zIndex * .1),
-            y = parseFloat(geometry.y);
+                        scale: new Vector3(entity.geometry.width * scale, entity.geometry.height * scale, .05),
-            if (y < this.minY) {
+                        color: Color3.Blue().toHexString(),
-                this.minY = y;
+                        template: '#box-template'
-            }
+                    }
-            if (y > this.maxY) {
+                }
-                this.maxY = y;
+            );
-            }
+        });
-        }
+
-        return ({x: x, y: y, width: parseFloat(geometry.width), height: parseFloat(geometry.height)});
+
     }
 
     private getText(obj: Node, text: string): string {

src/integration/voiceManager.ts

@@ -0,0 +1,73 @@
+import RecordRTC from 'recordrtc';
+
+export class VoiceManager {
+    private socket: WebSocket;
+    private token: string;
+    private recorder: RecordRTC;
+    private data: any[] = [];
+
+    constructor() {
+
+    }
+
+    public async setupConnection() {
+        const response = await fetch('/api/voice/token');
+        const data = await response.json();
+        this.token = data.token;
+        if (!this.socket) {
+            this.socket = new WebSocket(`wss://api.assemblyai.com/v2/realtime/ws?sample_rate=16000&token=${this.token}`);
+            this.socket.onmessage = (message) => {
+                const res = JSON.parse(message.data);
+                if (this.data) {
+                    this.data.push(res);
+                    //this.target.emit('transcriptiondata', {data: res});
+                }
+            }
+            this.socket.onopen = this.socketOpen;
+        } else {
+            switch (this.socket.readyState) {
+                case 0:
+                    console.log('socket opening');
+                    break;
+                case 1:
+                    console.log('socket already open');
+                    break;
+                case 2:
+                    console.log('dang, socket is closing');
+                    this.socket = null;
+                    break;
+                case 3:
+                    console.log('Socket is closed');
+                    this.socket = null;
+                    break
+                default:
+                    console.log(`socket state is unknown: ${this.socket.readyState}`);
+            }
+
+        }
+    }
+
+    private async socketOpen() {
+        if (!this.recorder) {
+            const stream = await navigator.mediaDevices.getUserMedia({audio: true});
+            this.recorder = new RecordRTC(stream, {
+                type: 'audio', mimeType: 'audio/webm;codecs=pcm', // endpoint requires 16bit PCM audio
+                recorderType: RecordRTC.StereoAudioRecorder, timeSlice: 300, // set 250 ms intervals of data that sends to AAI
+                desiredSampRate: 16000, numberOfAudioChannels: 1, // real-time requires only one channel
+                bufferSize: 4096, audioBitsPerSecond: 128000, ondataavailable: (blob) => {
+                    const reader = new FileReader();
+                    reader.onload = () => {
+                        const base64data: string = (reader.result as string);
+                        // audio data must be sent as a base64 encoded string
+                        if (this.socket && (this.socket.readyState === 1)) {
+                            this.socket.send(JSON.stringify({audio_data: base64data.split('base64,')[1]}));
+                        } else {
+                            console.log('no socket available');
+                        }
+                    };
+                    reader.readAsDataURL(blob);
+                },
+            });
+        }
+    }
+}

src/util/appConfig.ts

@@ -16,7 +16,7 @@ export class AppConfig {
     private _turnSnap = 0;
     private rotateSnap = 0;
     private createSnap = 0;
-    _physicsEnabled = true;
+    _physicsEnabled = false;
     private readonly defaultGridSnapIndex = 1;
     private persistenceManager: IPersistenceManager = null;
     private gridSnapArray: SnapValue[] =

src/util/customPhysics.ts

@@ -21,21 +21,29 @@ export class CustomPhysics {
                     const linearVelocity = new Vector3();
                     body.getLinearVelocityToRef(linearVelocity);
                     if (linearVelocity.length() < .1) {
-                        body.disablePreStep = false;
+                        if (true) {
-                        const bodyId = body._pluginData.hpBodyId[0];
+                            body.disablePreStep = false;
-                        // const position = body._pluginData._hknp.HP_Body_GetPosition(bodyId);
+                            const pos: Vector3 = body.getObjectCenterWorld();
-                        const pos: Vector3 = body.getObjectCenterWorld();
+                            const val: Vector3 = AppConfig.config.snapGridVal(pos);
-                        const val: Vector3 = AppConfig.config.snapGridVal(pos);
+                            body.transformNode.position.set(val.x, val.y, val.z);
-                        //body.setTargetTransform(val, body.transformNode.rotationQuaternion);
+                            const rot: Quaternion =
-                        body.transformNode.position.set(val.x, val.y, val.z);
+                                Quaternion.FromEulerVector(AppConfig.config.snapRotateVal(body.transformNode.rotationQuaternion.toEulerAngles()))
-                        const rot: Quaternion =
-                            Quaternion.FromEulerVector(AppConfig.config.snapRotateVal(body.transformNode.rotationQuaternion.toEulerAngles()))
 
-                        body.transformNode.rotationQuaternion.set(
+                            body.transformNode.rotationQuaternion.set(
-                            rot.x, rot.y, rot.z, rot.w
+                                rot.x, rot.y, rot.z, rot.w
-                        );
+                            );
 
-                        body.disablePreStep = true;
+                            //mesh.metadata.snapped=true;
+                            //(this.scene.getPhysicsEngine().getPhysicsPlugin() as IPhysicsEnginePluginV2).syncTransform(body, body.transformNode);
+                            this.scene.onAfterRenderObservable.addOnce(() => {
+                                body.disablePreStep = true;
+                            });
+
+                        } else {
+
+                        }
+                    } else {
+                        //mesh.metadata.snapped = false;
                     }
                     //mesh.position = mesh.physicsImpostor.physicsBody.position;
                     //mesh.rotationQuaternion = mesh.physicsImpostor.physicsBody.quaternion;

tsconfig.json

@@ -8,15 +8,30 @@
     "useDefineForClassFields": true, // enable latest ECMA runtime behavior with older ECMA/JavaScript versions (delete this line if target: "ESNext" or "ES2022"+)
     "module": "ESNext", // use the latest ECMA/JavaScript syntax for our import statements and such
     "moduleResolution": "node", // ensures we are using CommonJS for our npm packages
-    "noResolve": false, // disable TypeScript from automatically detecting/adding files based on import statements and etc (it's less helpful than you think)
+    "noResolve": false,
-    "isolatedModules": true, // allows our code to be processed by other transpilers, such as preventing non-module TS files (you could delete this since we're only using base TypeScript)
+    // disable TypeScript from automatically detecting/adding files based on import statements and etc (it's less helpful than you think)
-    "removeComments": true, // remove comments from our outputted code to save on space (look into terser if you want to protect the outputted JS even more)
+    "isolatedModules": true,
-    "esModuleInterop": true, // treats non-ES6 modules separately from ES6 modules (helpful if module: "ESNext")
+    // allows our code to be processed by other transpilers, such as preventing non-module TS files (you could delete this since we're only using base TypeScript)
-    "noImplicitAny": false, // usually prevents code from using "any" type fallbacks to prevent untraceable JS errors, but we'll need this disabled for our example code
+    "removeComments": true,
-    "noUnusedLocals": false, // usually raises an error for any unused local variables, but we'll need this disabled for our example code
+    // remove comments from our outputted code to save on space (look into terser if you want to protect the outputted JS even more)
-    "noUnusedParameters": true, // raises an error for unused parameters
+    "esModuleInterop": true,
-    "noImplicitReturns": true, // raises an error for functions that return nothing
+    // treats non-ES6 modules separately from ES6 modules (helpful if module: "ESNext")
-    "skipLibCheck": true // skip type-checking of .d.ts files (it speeds up transpiling)
+    "noImplicitAny": false,
+    // usually prevents code from using "any" type fallbacks to prevent untraceable JS errors, but we'll need this disabled for our example code
+    "noUnusedLocals": false,
+    // usually raises an error for any unused local variables, but we'll need this disabled for our example code
+    "noUnusedParameters": true,
+    // raises an error for unused parameters
+    "noImplicitReturns": true,
+    // raises an error for functions that return nothing
+    "skipLibCheck": true
+    // skip type-checking of .d.ts files (it speeds up transpiling)
   },
-  "include": ["src"] // specify location(s) of .ts files
+  "include": [
+    "src"
+  ],
+  // specify location(s) of .ts files
+  "exclude": [
+    "functions"
+  ]
 }