Added voice manager, integrated cloud function capability.
This commit is contained in:
parent
81627853e4
commit
00dac7f4ec
4
functions/src/voiceServer.ts
Normal file
4
functions/src/voiceServer.ts
Normal file
@ -0,0 +1,4 @@
|
||||
export const onRequest: PagesFunction = async (context) => {
|
||||
const response = new Response('Hello World!');
|
||||
return response;
|
||||
}
|
||||
12
functions/tsconfig.json
Normal file
12
functions/tsconfig.json
Normal file
@ -0,0 +1,12 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "esnext",
|
||||
"module": "esnext",
|
||||
"lib": [
|
||||
"esnext"
|
||||
],
|
||||
"types": [
|
||||
"@cloudflare/workers-types"
|
||||
]
|
||||
}
|
||||
}
|
||||
20
package-lock.json
generated
20
package-lock.json
generated
@ -12,7 +12,9 @@
|
||||
"@babylonjs/gui": "^6.15.0",
|
||||
"@babylonjs/havok": "1.1.1",
|
||||
"@babylonjs/inspector": "^6.15.0",
|
||||
"@babylonjs/loaders": "^6.15.0",
|
||||
"@babylonjs/serializers": "^6.15.0",
|
||||
"@cloudflare/workers-types": "^4.20230807.0",
|
||||
"@typed-mxgraph/typed-mxgraph": "^1.0.8",
|
||||
"dexie": "^3.2.4",
|
||||
"dexie-observable": "^4.0.1-beta.13",
|
||||
@ -21,6 +23,7 @@
|
||||
"mxgraph": "^4.2.2",
|
||||
"p2p-data-channel": "^1.10.7",
|
||||
"query-string": "^8.1.0",
|
||||
"recordrtc": "^5.6.2",
|
||||
"ring-client-api": "11.7.7",
|
||||
"round": "^2.0.1",
|
||||
"uuid": "^9.0.0"
|
||||
@ -97,10 +100,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babylonjs/loaders": {
|
||||
"version": "6.8.0",
|
||||
"resolved": "https://registry.npmjs.org/@babylonjs/loaders/-/loaders-6.8.0.tgz",
|
||||
"integrity": "sha512-Ze3dsjwQ6ORYTkbacdlZVeN1PvC0KYcN2hzDNAqiQ0eeQkk96QDusgLkVHMM1UW55hS5PQefTua7iylOxWetDw==",
|
||||
"peer": true,
|
||||
"version": "6.15.0",
|
||||
"resolved": "https://registry.npmjs.org/@babylonjs/loaders/-/loaders-6.15.0.tgz",
|
||||
"integrity": "sha512-pE22KVLEWcsWPmiL+TFsNyqLTMGhnt/6oPDy3nWpMwHTfjeaoW9qTV7hFDwhXopzU1Ddp6c/DhzFZalgSvCvbg==",
|
||||
"peerDependencies": {
|
||||
"@babylonjs/core": "^6.0.0",
|
||||
"babylonjs-gltf2interface": "^6.0.0"
|
||||
@ -124,6 +126,11 @@
|
||||
"babylonjs-gltf2interface": "^6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@cloudflare/workers-types": {
|
||||
"version": "4.20230807.0",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20230807.0.tgz",
|
||||
"integrity": "sha512-gQczWuGE2rxmpzOCNn0zLbx8Xz0gqspdE9S7tu4Xax39q1csgO/E9flcS+KG3GHB522ugOh84inmABDhpeJnvQ=="
|
||||
},
|
||||
"node_modules/@eneris/push-receiver": {
|
||||
"version": "3.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@eneris/push-receiver/-/push-receiver-3.1.4.tgz",
|
||||
@ -3267,6 +3274,11 @@
|
||||
"node": ">=8.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/recordrtc": {
|
||||
"version": "5.6.2",
|
||||
"resolved": "https://registry.npmjs.org/recordrtc/-/recordrtc-5.6.2.tgz",
|
||||
"integrity": "sha512-1QNKKNtl7+KcwD1lyOgP3ZlbiJ1d0HtXnypUy7yq49xEERxk31PHvE9RCciDrulPCY7WJ+oz0R9hpNxgsIurGQ=="
|
||||
},
|
||||
"node_modules/reflect-metadata": {
|
||||
"version": "0.1.13",
|
||||
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz",
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
"build": "vite build",
|
||||
"preview": "vite preview",
|
||||
"serve": "node server.js",
|
||||
"serverBuild": "cd server && tsc",
|
||||
"havok": "cp ./node_modules/@babylonjs/havok/lib/esm/HavokPhysics.wasm ./node_modules/.vite/deps"
|
||||
},
|
||||
"dependencies": {
|
||||
@ -16,6 +17,8 @@
|
||||
"@babylonjs/havok": "1.1.1",
|
||||
"@babylonjs/inspector": "^6.15.0",
|
||||
"@babylonjs/serializers": "^6.15.0",
|
||||
"@babylonjs/loaders": "^6.15.0",
|
||||
"@cloudflare/workers-types": "^4.20230807.0",
|
||||
"ring-client-api": "11.7.7",
|
||||
"dexie": "^3.2.4",
|
||||
"dexie-observable": "^4.0.1-beta.13",
|
||||
@ -23,6 +26,7 @@
|
||||
"loglevel": "^1.8.1",
|
||||
"mxgraph": "^4.2.2",
|
||||
"@typed-mxgraph/typed-mxgraph": "^1.0.8",
|
||||
"recordrtc": "^5.6.2",
|
||||
"round": "^2.0.1",
|
||||
"earcut": "^2.2.4",
|
||||
"uuid": "^9.0.0",
|
||||
|
||||
@ -19,6 +19,7 @@ import {GamepadManager} from "./controllers/gamepadManager";
|
||||
import {CustomEnvironment} from "./util/customEnvironment";
|
||||
import {DrawioManager} from "./integration/drawioManager";
|
||||
|
||||
|
||||
export class App {
|
||||
//preTasks = [havokModule];
|
||||
constructor() {
|
||||
@ -30,9 +31,10 @@ export class App {
|
||||
log.getLogger('App').setLevel('info');
|
||||
//log.getLogger('IndexdbPersistenceManager').setLevel('info');
|
||||
//log.getLogger('DiagramManager').setLevel('info');
|
||||
|
||||
//log.getLogger('DiagramConnection').setLevel('debug');
|
||||
log.getLogger('DrawioManager').setLevel('debug');
|
||||
|
||||
log.getLogger('EntityTree').setLevel('debug');
|
||||
log.getLogger('EditMenu').setLevel('debug');
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.style.width = "100%";
|
||||
@ -106,7 +108,7 @@ export class App {
|
||||
const diagramManager = new DiagramManager(scene, xr.baseExperience);
|
||||
const rig = new Rigplatform(scene, xr, diagramManager);
|
||||
const toolbox = new Toolbox(scene, xr.baseExperience, diagramManager);
|
||||
//const dioManager = new DrawioManager(scene, diagramManager);
|
||||
const dioManager = new DrawioManager(scene, diagramManager);
|
||||
import ('./integration/indexdbPersistenceManager').then((module) => {
|
||||
const persistenceManager = new module.IndexdbPersistenceManager("diagram");
|
||||
diagramManager.setPersistenceManager(persistenceManager);
|
||||
@ -114,6 +116,8 @@ export class App {
|
||||
persistenceManager.initialize();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
const gamepadManager = new GamepadManager(scene);
|
||||
window.addEventListener("keydown", (ev) => {
|
||||
// Shift+Ctrl+Alt+I
|
||||
@ -129,6 +133,7 @@ export class App {
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
logger.info('keydown event listener added, use Ctrl+Shift+Alt+I to toggle debug layer');
|
||||
engine.runRenderLoop(() => {
|
||||
scene.render();
|
||||
|
||||
@ -223,7 +223,9 @@ export class Base {
|
||||
this.previousRotation = null;
|
||||
this.previousPosition = null;
|
||||
this.grabbedMesh = null;
|
||||
|
||||
if (mesh?.metadata?.template.indexOf('#') == -1) {
|
||||
return;
|
||||
}
|
||||
const entity = MeshConverter.toDiagramEntity(mesh);
|
||||
const event: DiagramEvent = {
|
||||
type: DiagramEventType.DROP,
|
||||
|
||||
@ -64,10 +64,12 @@ export class Rigplatform {
|
||||
PhysicsShapeType.CYLINDER,
|
||||
{friction: 1, center: Vector3.Zero(), radius: .5, mass: 10, restitution: .01},
|
||||
scene);
|
||||
|
||||
rigAggregate.body.setMotionType(PhysicsMotionType.DYNAMIC);
|
||||
rigAggregate.body.setGravityFactor(.001);
|
||||
this.fixRotation();
|
||||
this.body = rigAggregate.body;
|
||||
|
||||
this.initializeControllers();
|
||||
scene.onActiveCameraChanged.add((s) => {
|
||||
this.camera = s.activeCamera;
|
||||
|
||||
@ -159,8 +159,11 @@ export class DiagramManager {
|
||||
case DiagramEventType.DROPPED:
|
||||
break;
|
||||
case DiagramEventType.DROP:
|
||||
this.getPersistenceManager()?.modify(mesh);
|
||||
TextLabel.updateTextNode(mesh, entity.text);
|
||||
if (mesh.metadata.template.indexOf('#') > -1) {
|
||||
this.getPersistenceManager()?.modify(mesh);
|
||||
TextLabel.updateTextNode(mesh, entity.text);
|
||||
}
|
||||
|
||||
break;
|
||||
case DiagramEventType.ADD:
|
||||
this.getPersistenceManager()?.add(mesh);
|
||||
@ -248,8 +251,8 @@ class DiagramShapePhysics {
|
||||
const aggregate = new PhysicsAggregate(mesh,
|
||||
shapeType, {mass: mass, restitution: .02, friction: .9}, scene);
|
||||
const body = aggregate.body;
|
||||
body.setLinearDamping(.95);
|
||||
body.setAngularDamping(.99);
|
||||
body.setLinearDamping(1.95);
|
||||
body.setAngularDamping(1.99);
|
||||
|
||||
if (motionType) {
|
||||
body
|
||||
|
||||
@ -1,147 +1,291 @@
|
||||
import log from "loglevel";
|
||||
import {Color3, Scene, TransformNode, Vector3} from "@babylonjs/core";
|
||||
import {Color3, Scene, Vector3} from "@babylonjs/core";
|
||||
import {DiagramManager} from "../diagram/diagramManager";
|
||||
import {DiagramEventType} from "../diagram/diagramEntity";
|
||||
|
||||
type DrawIOEntity = {
|
||||
text?: string,
|
||||
id?: string,
|
||||
parent?: string,
|
||||
parentEntity?: DrawIOEntity,
|
||||
geometry?: DrawIOGeometry,
|
||||
|
||||
}
|
||||
type DrawIOGeometry = {
|
||||
zIndex?: number,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number
|
||||
|
||||
}
|
||||
|
||||
class EntityTree {
|
||||
private readonly logger = log.getLogger('EntityTree');
|
||||
private root: DrawIOEntity;
|
||||
private readonly nodes: Map<string, DrawIOEntity> = new Map<string, DrawIOEntity>();
|
||||
private readonly unparented: Array<DrawIOEntity> = new Array<DrawIOEntity>();
|
||||
|
||||
constructor() {
|
||||
this.root = {};
|
||||
}
|
||||
|
||||
public getNodes(): Array<DrawIOEntity> {
|
||||
this.reparent();
|
||||
const output: Array<DrawIOEntity> = new Array<DrawIOEntity>();
|
||||
this.nodes.forEach((node) => {
|
||||
if (node.parentEntity) {
|
||||
const geometry = this.computeOffset(node);
|
||||
node.geometry = geometry;
|
||||
}
|
||||
output.push(node);
|
||||
});
|
||||
return output;
|
||||
}
|
||||
|
||||
public reparent() {
|
||||
this.unparented.forEach((node) => {
|
||||
if (this.nodes.has(node.parent)) {
|
||||
this.logger.debug('reparenting node: ' + node.id + ' to parent: ' + node.parent);
|
||||
node.parentEntity = this.nodes.get(node.parent);
|
||||
} else {
|
||||
this.logger.warn('parent node does not exist for id: ' + node.id +
|
||||
' parent id: ' + node.parent);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public addNode(node: DrawIOEntity) {
|
||||
if (this.nodes.has(node.id)) {
|
||||
this.logger.warn('node already exists for id: ' + node.id);
|
||||
} else {
|
||||
if (node.parent) {
|
||||
if (this.nodes.has(node.parent)) {
|
||||
node.parentEntity = this.nodes.get(node.parent);
|
||||
this.nodes.set(node.id, node);
|
||||
} else {
|
||||
this.logger.warn('parent node does not exist for id: ' + node.id +
|
||||
' parent id: ' + node.parent);
|
||||
this.unparented.push(node);
|
||||
}
|
||||
} else {
|
||||
this.logger.warn('no parent for node id: ' + node.id + 'setting as root');
|
||||
this.nodes.set(node.id, node);
|
||||
this.root = node;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private computeOffset(node: DrawIOEntity): DrawIOGeometry {
|
||||
if (node.parentEntity) {
|
||||
const parentgeo = this.computeOffset(node.parentEntity);
|
||||
if (parentgeo) {
|
||||
const parentzIndex = 1 + parentgeo.zIndex ? parentgeo.zIndex : 0;
|
||||
const geo = {
|
||||
x: node.geometry.x,
|
||||
y: node.geometry.y,
|
||||
width: node.geometry.width,
|
||||
height: node.geometry.height,
|
||||
zIndex: node.geometry.zIndex ? node.geometry.zIndex + parentzIndex : parentzIndex + 1
|
||||
};
|
||||
return geo;
|
||||
} else {
|
||||
const geo = {
|
||||
x: node.geometry.x,
|
||||
y: node.geometry.y,
|
||||
width: node.geometry.width,
|
||||
height: node.geometry.height,
|
||||
zIndex: node.geometry.zIndex ? node.geometry.zIndex : 0
|
||||
};
|
||||
return geo;
|
||||
}
|
||||
|
||||
} else {
|
||||
if (node.geometry) {
|
||||
if (node.geometry.zIndex === undefined) {
|
||||
node.geometry.zIndex = 0;
|
||||
}
|
||||
return node.geometry;
|
||||
} else {
|
||||
return {
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 0,
|
||||
height: 0,
|
||||
zIndex: 0
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type DrawIOConnector = {
|
||||
id: string,
|
||||
source: string,
|
||||
target: string,
|
||||
text: string
|
||||
}
|
||||
|
||||
export class DrawioManager {
|
||||
private diagramManager: DiagramManager;
|
||||
private readonly zdepth: Map<string, number> = new Map<string, number>();
|
||||
private connectors: Array<DrawIOConnector> = [];
|
||||
private readonly scene: Scene;
|
||||
private readonly logger = log.getLogger('DrawioManager');
|
||||
private minY = 0;
|
||||
private minX = 0;
|
||||
private maxX = 0;
|
||||
private maxY = 0;
|
||||
private maxZ = 0;
|
||||
|
||||
constructor(scene: Scene, diagramManager: DiagramManager) {
|
||||
this.scene = scene;
|
||||
this.diagramManager = diagramManager;
|
||||
this.getGraph();
|
||||
this.buildGraph();
|
||||
}
|
||||
|
||||
|
||||
|
||||
private async getGraph() {
|
||||
private async fetchData(url: string): Promise<Document> {
|
||||
this.logger.debug("starting to get graph");
|
||||
const entities: Array<{ text: string, id: string, geometry: { x: number, y: number, width: number, height: number } }>
|
||||
= new Array<{ text: string; id: string, geometry: { x: number; y: number; width: number; height: number } }>();
|
||||
|
||||
const graph = await fetch('/arch_demo.xml');
|
||||
const graph = await fetch(url);
|
||||
this.logger.debug('got graph');
|
||||
const graphXml = await graph.text();
|
||||
const doc = new DOMParser().parseFromString(graphXml, 'text/html');
|
||||
//this.logger.debug(doc);
|
||||
const firstDiagram = doc.querySelectorAll('diagram')[0];
|
||||
const mxDiagram = firstDiagram.querySelector('mxGraphModel');
|
||||
const width = mxDiagram.getAttribute('pageWidth');
|
||||
const height = mxDiagram.getAttribute('pageHeight');
|
||||
this.logger.debug('begin parse');
|
||||
mxDiagram.querySelectorAll('mxCell').forEach((cell) => {
|
||||
return doc;
|
||||
}
|
||||
|
||||
private getDiagram(doc: Document, index: number): Element {
|
||||
const firstDiagram = doc.querySelectorAll('diagram')[index];
|
||||
const mxDiagram = firstDiagram.querySelector('mxGraphModel');
|
||||
return mxDiagram;
|
||||
}
|
||||
|
||||
private parseDiagram(mxDiagram: Element): EntityTree {
|
||||
const entityTree = new EntityTree();
|
||||
|
||||
mxDiagram.querySelectorAll('mxCell').forEach((cell) => {
|
||||
const value = cell.getAttribute('value');
|
||||
if (!value) {
|
||||
//this.logger.warn('no value for :' , cell);
|
||||
} else {
|
||||
const ent = new DOMParser().parseFromString(value, 'text/html');
|
||||
let ent = null;
|
||||
if (value) {
|
||||
ent = new DOMParser().parseFromString(value, 'text/html');
|
||||
const errorNode = ent.querySelector("parsererror");
|
||||
this.logger.debug(value);
|
||||
if (errorNode) {
|
||||
//this.logger.debug(value);
|
||||
this.logger.error(value);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
const text = ent ? this.getText(ent, '') : '';
|
||||
const id = cell.getAttribute('id');
|
||||
const parent = cell.getAttribute('parent');
|
||||
const source = cell.getAttribute('source');
|
||||
const target = cell.getAttribute('target');
|
||||
const edge = cell.getAttribute('target');
|
||||
if (source && target && edge) {
|
||||
this.connectors.push({id: id, source: source, target: target, text: text});
|
||||
} else {
|
||||
|
||||
const geo = cell.querySelector('[id="' + id + '"] > mxGeometry');
|
||||
let geometry = null;
|
||||
if (geo) {
|
||||
geometry = {
|
||||
x: Number.parseFloat(geo.getAttribute('x')),
|
||||
y: Number.parseFloat(geo.getAttribute('y')),
|
||||
width: Number.parseFloat(geo.getAttribute('width')),
|
||||
height: Number.parseFloat(geo.getAttribute('height')),
|
||||
}
|
||||
} else {
|
||||
const text = this.getText(ent, '');
|
||||
const id = cell.getAttribute('id');
|
||||
const parent = cell.getAttribute('parent');
|
||||
if (this.zdepth.has(parent)) {
|
||||
this.zdepth.set(id, this.zdepth.get(parent) + .2);
|
||||
geometry = {
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 0,
|
||||
height: 0
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//entities.push({text: text, id: id, parent: parent, geometry: this.fixMinMax(geometry)});
|
||||
if (text) {
|
||||
this.logger.debug('Text' + text);
|
||||
this.logger.debug('Geometry' + JSON.stringify(geometry));
|
||||
}
|
||||
if (geometry) {
|
||||
if (Number.isNaN(geometry.x) || Number.isNaN(geometry.y)
|
||||
|| Number.isNaN(geometry.width) ||
|
||||
Number.isNaN(geometry.height)) {
|
||||
this.logger.warn('invalid geometry for node: ' + id, geometry);
|
||||
} else {
|
||||
this.zdepth.set(cell.getAttribute('id'), 0);
|
||||
}
|
||||
const geo = cell.querySelector('mxGeometry');
|
||||
const geometry = {
|
||||
x: geo.getAttribute('x'),
|
||||
y: geo.getAttribute('y'),
|
||||
width: geo.getAttribute('width'),
|
||||
height: geo.getAttribute('height'),
|
||||
}
|
||||
entities.push({text: text, id: id, geometry: this.fixMinMax(geometry)});
|
||||
if (text) {
|
||||
this.logger.debug('Text' + text);
|
||||
this.logger.debug('Geometry' + JSON.stringify(geometry));
|
||||
entityTree.addNode({text: text, id: id, parent: parent, geometry: geometry});
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
return entityTree
|
||||
}
|
||||
|
||||
private async buildGraph() {
|
||||
|
||||
const doc = await this.fetchData('/arch_demo.xml');
|
||||
const mxDiagram = this.getDiagram(doc, 0);
|
||||
this.logger.debug('begin parse');
|
||||
const entities: EntityTree = this.parseDiagram(mxDiagram);
|
||||
|
||||
entities.getNodes().forEach((node) => {
|
||||
if (node.geometry.x < this.minX) {
|
||||
this.minX = node.geometry.x;
|
||||
this.logger.debug('minX: ' + this.minX);
|
||||
}
|
||||
if (node.geometry.y < this.minY) {
|
||||
this.minY = node.geometry.y;
|
||||
this.logger.debug('minY: ' + this.minY);
|
||||
}
|
||||
if (node.geometry.x + node.geometry.width > this.maxX) {
|
||||
this.maxX = node.geometry.x + node.geometry.width;
|
||||
this.logger.debug('maxX: ' + this.maxX);
|
||||
}
|
||||
if (node.geometry.y + node.geometry.height > this.maxY) {
|
||||
this.maxY = node.geometry.y + node.geometry.height;
|
||||
this.logger.debug('maxY: ' + this.maxY);
|
||||
}
|
||||
if (node.geometry.zIndex > this.maxZ) {
|
||||
this.maxZ = node.geometry.zIndex;
|
||||
this.logger.debug('maxZ: ' + this.maxZ);
|
||||
}
|
||||
|
||||
});
|
||||
this.logger.info('minX: ' + this.minX + ' minY: ' + this.minY + ' maxX: ' + this.maxX + ' maxY: ' + this.maxY);
|
||||
|
||||
|
||||
this.logger.debug('done parsing');
|
||||
|
||||
this.logger.debug('MinX' + this.minX);
|
||||
this.logger.debug('MinY' + this.minY);
|
||||
this.logger.debug('MaxX' + this.maxX);
|
||||
this.logger.debug('MaxY' + this.maxY);
|
||||
const diagramWidth = this.maxX - this.minX;
|
||||
const diagramHeight = this.maxY - this.minY;
|
||||
let scale = 1;
|
||||
if (diagramHeight > diagramWidth) {
|
||||
scale = 20 / diagramHeight;
|
||||
} else {
|
||||
scale = 20 / diagramWidth;
|
||||
}
|
||||
const anchor = new TransformNode('anchor', this.scene);
|
||||
|
||||
if (entities.length > 0) {
|
||||
entities.forEach((entity) => {
|
||||
this.diagramManager.onDiagramEventObservable.notifyObservers(
|
||||
{
|
||||
type: DiagramEventType.ADD,
|
||||
entity: {
|
||||
text: entity.text,
|
||||
id: entity.id,
|
||||
position: new Vector3((entity.geometry.x - this.minX) * scale + (entity.geometry.width * scale / 2),
|
||||
(entity.geometry.y - this.minY) * scale + (entity.geometry.height * scale / 2),
|
||||
2 + this.zdepth.get(entity.id)),
|
||||
scale: new Vector3(entity.geometry.width * scale, entity.geometry.height * scale, .1),
|
||||
color: Color3.Blue().toHexString(),
|
||||
template: '#box-template'
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
//box.metadata = {text: entity.text};
|
||||
//box.setParent(anchor);
|
||||
//DrawioManager.updateTextNode(box, entity.text);
|
||||
});
|
||||
anchor.position.y = 20;
|
||||
anchor.rotation.x = Math.PI;
|
||||
|
||||
}
|
||||
|
||||
this.logger.debug('Scale' + scale);
|
||||
|
||||
this.logger.debug(this.connectors);
|
||||
this.createSceneData(entities.getNodes());
|
||||
|
||||
}
|
||||
|
||||
private fixMinMax(geometry: { x: string; y: string; width: string; height: string; }):
|
||||
{ x: number, y: number, width: number, height: number } {
|
||||
let x = 0;
|
||||
if (geometry.x) {
|
||||
x = parseFloat(geometry.x);
|
||||
if (x < this.minX) {
|
||||
this.minX = x;
|
||||
}
|
||||
if (x > this.maxX) {
|
||||
this.maxX = x;
|
||||
}
|
||||
}
|
||||
let y = 0;
|
||||
if (geometry.y) {
|
||||
y = parseFloat(geometry.y);
|
||||
if (y < this.minY) {
|
||||
this.minY = y;
|
||||
}
|
||||
if (y > this.maxY) {
|
||||
this.maxY = y;
|
||||
}
|
||||
}
|
||||
return ({x: x, y: y, width: parseFloat(geometry.width), height: parseFloat(geometry.height)});
|
||||
private createSceneData(nodes) {
|
||||
const yOffset = 20;
|
||||
const scale = .001;
|
||||
nodes.forEach((entity) => {
|
||||
this.diagramManager.onDiagramEventObservable.notifyObservers(
|
||||
{
|
||||
type: DiagramEventType.ADD,
|
||||
entity: {
|
||||
text: entity.text,
|
||||
id: entity.id,
|
||||
position: new Vector3(
|
||||
(entity.geometry.x * scale) - (entity.geometry.width * scale / 2),
|
||||
yOffset - (entity.geometry.y * scale) + (entity.geometry.height * scale / 2),
|
||||
entity.geometry.zIndex * .1),
|
||||
scale: new Vector3(entity.geometry.width * scale, entity.geometry.height * scale, .05),
|
||||
color: Color3.Blue().toHexString(),
|
||||
template: '#box-template'
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
}
|
||||
|
||||
private getText(obj: Node, text: string): string {
|
||||
|
||||
73
src/integration/voiceManager.ts
Normal file
73
src/integration/voiceManager.ts
Normal file
@ -0,0 +1,73 @@
|
||||
import RecordRTC from 'recordrtc';
|
||||
|
||||
export class VoiceManager {
|
||||
private socket: WebSocket;
|
||||
private token: string;
|
||||
private recorder: RecordRTC;
|
||||
private data: any[] = [];
|
||||
|
||||
constructor() {
|
||||
|
||||
}
|
||||
|
||||
public async setupConnection() {
|
||||
const response = await fetch('/api/voice/token');
|
||||
const data = await response.json();
|
||||
this.token = data.token;
|
||||
if (!this.socket) {
|
||||
this.socket = new WebSocket(`wss://api.assemblyai.com/v2/realtime/ws?sample_rate=16000&token=${this.token}`);
|
||||
this.socket.onmessage = (message) => {
|
||||
const res = JSON.parse(message.data);
|
||||
if (this.data) {
|
||||
this.data.push(res);
|
||||
//this.target.emit('transcriptiondata', {data: res});
|
||||
}
|
||||
}
|
||||
this.socket.onopen = this.socketOpen;
|
||||
} else {
|
||||
switch (this.socket.readyState) {
|
||||
case 0:
|
||||
console.log('socket opening');
|
||||
break;
|
||||
case 1:
|
||||
console.log('socket already open');
|
||||
break;
|
||||
case 2:
|
||||
console.log('dang, socket is closing');
|
||||
this.socket = null;
|
||||
break;
|
||||
case 3:
|
||||
console.log('Socket is closed');
|
||||
this.socket = null;
|
||||
break
|
||||
default:
|
||||
console.log(`socket state is unknown: ${this.socket.readyState}`);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private async socketOpen() {
|
||||
if (!this.recorder) {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({audio: true});
|
||||
this.recorder = new RecordRTC(stream, {
|
||||
type: 'audio', mimeType: 'audio/webm;codecs=pcm', // endpoint requires 16bit PCM audio
|
||||
recorderType: RecordRTC.StereoAudioRecorder, timeSlice: 300, // set 250 ms intervals of data that sends to AAI
|
||||
desiredSampRate: 16000, numberOfAudioChannels: 1, // real-time requires only one channel
|
||||
bufferSize: 4096, audioBitsPerSecond: 128000, ondataavailable: (blob) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => {
|
||||
const base64data: string = (reader.result as string);
|
||||
// audio data must be sent as a base64 encoded string
|
||||
if (this.socket && (this.socket.readyState === 1)) {
|
||||
this.socket.send(JSON.stringify({audio_data: base64data.split('base64,')[1]}));
|
||||
} else {
|
||||
console.log('no socket available');
|
||||
}
|
||||
};
|
||||
reader.readAsDataURL(blob);
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -16,7 +16,7 @@ export class AppConfig {
|
||||
private _turnSnap = 0;
|
||||
private rotateSnap = 0;
|
||||
private createSnap = 0;
|
||||
_physicsEnabled = true;
|
||||
_physicsEnabled = false;
|
||||
private readonly defaultGridSnapIndex = 1;
|
||||
private persistenceManager: IPersistenceManager = null;
|
||||
private gridSnapArray: SnapValue[] =
|
||||
|
||||
@ -21,21 +21,29 @@ export class CustomPhysics {
|
||||
const linearVelocity = new Vector3();
|
||||
body.getLinearVelocityToRef(linearVelocity);
|
||||
if (linearVelocity.length() < .1) {
|
||||
body.disablePreStep = false;
|
||||
const bodyId = body._pluginData.hpBodyId[0];
|
||||
// const position = body._pluginData._hknp.HP_Body_GetPosition(bodyId);
|
||||
const pos: Vector3 = body.getObjectCenterWorld();
|
||||
const val: Vector3 = AppConfig.config.snapGridVal(pos);
|
||||
//body.setTargetTransform(val, body.transformNode.rotationQuaternion);
|
||||
body.transformNode.position.set(val.x, val.y, val.z);
|
||||
const rot: Quaternion =
|
||||
Quaternion.FromEulerVector(AppConfig.config.snapRotateVal(body.transformNode.rotationQuaternion.toEulerAngles()))
|
||||
if (true) {
|
||||
body.disablePreStep = false;
|
||||
const pos: Vector3 = body.getObjectCenterWorld();
|
||||
const val: Vector3 = AppConfig.config.snapGridVal(pos);
|
||||
body.transformNode.position.set(val.x, val.y, val.z);
|
||||
const rot: Quaternion =
|
||||
Quaternion.FromEulerVector(AppConfig.config.snapRotateVal(body.transformNode.rotationQuaternion.toEulerAngles()))
|
||||
|
||||
body.transformNode.rotationQuaternion.set(
|
||||
rot.x, rot.y, rot.z, rot.w
|
||||
);
|
||||
body.transformNode.rotationQuaternion.set(
|
||||
rot.x, rot.y, rot.z, rot.w
|
||||
);
|
||||
|
||||
body.disablePreStep = true;
|
||||
//mesh.metadata.snapped=true;
|
||||
//(this.scene.getPhysicsEngine().getPhysicsPlugin() as IPhysicsEnginePluginV2).syncTransform(body, body.transformNode);
|
||||
this.scene.onAfterRenderObservable.addOnce(() => {
|
||||
body.disablePreStep = true;
|
||||
});
|
||||
|
||||
} else {
|
||||
|
||||
}
|
||||
} else {
|
||||
//mesh.metadata.snapped = false;
|
||||
}
|
||||
//mesh.position = mesh.physicsImpostor.physicsBody.position;
|
||||
//mesh.rotationQuaternion = mesh.physicsImpostor.physicsBody.quaternion;
|
||||
|
||||
@ -8,15 +8,30 @@
|
||||
"useDefineForClassFields": true, // enable latest ECMA runtime behavior with older ECMA/JavaScript versions (delete this line if target: "ESNext" or "ES2022"+)
|
||||
"module": "ESNext", // use the latest ECMA/JavaScript syntax for our import statements and such
|
||||
"moduleResolution": "node", // ensures we are using CommonJS for our npm packages
|
||||
"noResolve": false, // disable TypeScript from automatically detecting/adding files based on import statements and etc (it's less helpful than you think)
|
||||
"isolatedModules": true, // allows our code to be processed by other transpilers, such as preventing non-module TS files (you could delete this since we're only using base TypeScript)
|
||||
"removeComments": true, // remove comments from our outputted code to save on space (look into terser if you want to protect the outputted JS even more)
|
||||
"esModuleInterop": true, // treats non-ES6 modules separately from ES6 modules (helpful if module: "ESNext")
|
||||
"noImplicitAny": false, // usually prevents code from using "any" type fallbacks to prevent untraceable JS errors, but we'll need this disabled for our example code
|
||||
"noUnusedLocals": false, // usually raises an error for any unused local variables, but we'll need this disabled for our example code
|
||||
"noUnusedParameters": true, // raises an error for unused parameters
|
||||
"noImplicitReturns": true, // raises an error for functions that return nothing
|
||||
"skipLibCheck": true // skip type-checking of .d.ts files (it speeds up transpiling)
|
||||
"noResolve": false,
|
||||
// disable TypeScript from automatically detecting/adding files based on import statements and etc (it's less helpful than you think)
|
||||
"isolatedModules": true,
|
||||
// allows our code to be processed by other transpilers, such as preventing non-module TS files (you could delete this since we're only using base TypeScript)
|
||||
"removeComments": true,
|
||||
// remove comments from our outputted code to save on space (look into terser if you want to protect the outputted JS even more)
|
||||
"esModuleInterop": true,
|
||||
// treats non-ES6 modules separately from ES6 modules (helpful if module: "ESNext")
|
||||
"noImplicitAny": false,
|
||||
// usually prevents code from using "any" type fallbacks to prevent untraceable JS errors, but we'll need this disabled for our example code
|
||||
"noUnusedLocals": false,
|
||||
// usually raises an error for any unused local variables, but we'll need this disabled for our example code
|
||||
"noUnusedParameters": true,
|
||||
// raises an error for unused parameters
|
||||
"noImplicitReturns": true,
|
||||
// raises an error for functions that return nothing
|
||||
"skipLibCheck": true
|
||||
// skip type-checking of .d.ts files (it speeds up transpiling)
|
||||
},
|
||||
"include": ["src"] // specify location(s) of .ts files
|
||||
"include": [
|
||||
"src"
|
||||
],
|
||||
// specify location(s) of .ts files
|
||||
"exclude": [
|
||||
"functions"
|
||||
]
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user