Added Toolbox with colors.

src/app.ts

@@ -14,18 +14,20 @@ import {
     Scene,
     Texture,
     Vector3,
-    WebXRDefaultExperience
+    WebXRDefaultExperience,
+    WebXRState
 } from "@babylonjs/core";
 ///import {havokModule} from "./util/havok";
 import HavokPhysics from "@babylonjs/havok";
 import {Rigplatform} from "./controllers/rigplatform";
 import {DiagramManager} from "./diagram/diagramManager";
+import {Toolbox} from "./toolbox/toolbox";
+
 
 
 export class App {
     //preTasks = [havokModule];
 
-    private token: string;
     public static scene: Scene;
     public static xr: WebXRDefaultExperience;
     public static rig: Rigplatform;
@@ -66,21 +68,42 @@ export class App {
         scene.enablePhysics(new Vector3(0, -9.8, 0), havokPlugin);
         const camera: ArcRotateCamera = new ArcRotateCamera("Camera", Math.PI / 2, Math.PI / 2, 2,
             new Vector3(0, 1.6, 0), scene);
+        camera.radius = 0;
         camera.attachControl(canvas, true);
+
+
         new HemisphericLight("light1", new Vector3(1, 1, 0), scene);
         const photoDome = new PhotoDome('sky',
             './outdoor_field.jpeg', {},
             scene);
-
+        const ground = this.createGround();
         App.xr = await WebXRDefaultExperience.CreateAsync(scene, {
-            floorMeshes: [this.createGround()],
+            floorMeshes: [ground],
             disableTeleportation: true,
-            optionalFeatures: true
+            outputCanvasOptions: {
+                canvasOptions: {
+                    framebufferScaleFactor: 1
+                }
+            },
+            optionalFeatures: true,
+            pointerSelectionOptions: {
+                enablePointerSelectionOnAllControllers: true
+            }
 
         });
+        App.xr.baseExperience.onStateChangedObservable.add((state) => {
+            if (state == WebXRState.IN_XR)  {
+                App.xr.baseExperience.camera.position = new Vector3(0, 1.6, 0);
+            }
+        });
+
+
+
         const diagramManager = new DiagramManager(App.scene, App.xr.baseExperience);
         App.rig = new Rigplatform(App.scene, App.xr);
+        const toolbox = new Toolbox(scene, App.xr.baseExperience);
 
+        //camera.parent = App.rig.rigMesh;
         window.addEventListener("keydown", (ev) => {
             // Shift+Ctrl+Alt+I
             if (ev.shiftKey && ev.ctrlKey && ev.altKey && ev.keyCode === 73) {

src/controllers/base.ts

@@ -1,44 +1,137 @@
-import {AbstractMesh, Scene, Vector3, WebXRControllerComponent, WebXRInputSource} from "@babylonjs/core";
+import {
+    AbstractMesh,
+    InstancedMesh,
+    Mesh,
+    Scene,
+    Vector3,
+    WebXRControllerComponent,
+    WebXRDefaultExperience,
+    WebXRInputSource
+} from "@babylonjs/core";
+import {MeshConverter} from "../diagram/meshConverter";
+import {DiagramManager} from "../diagram/diagramManager";
+import {DiagramEvent, DiagramEventType} from "../diagram/diagramEntity";
 
 export class Base {
     static stickVector = Vector3.Zero();
     protected controller: WebXRInputSource;
     protected speedFactor = 4;
     protected readonly scene: Scene;
-    protected currentMesh: AbstractMesh = null;
+    protected grabbedMesh: AbstractMesh = null;
-    constructor(controller:
+    protected previousParent: string = null;
-                    WebXRInputSource, scene: Scene) {
+    protected previousRotation: Vector3 = null;
+    protected previousScaling: Vector3 = null;
+    protected previousPosition: Vector3 = null;
+    protected readonly xr: WebXRDefaultExperience;
+
+    constructor(controller: WebXRInputSource, scene: Scene, xr: WebXRDefaultExperience) {
         this.controller = controller;
         this.scene = scene;
-        this.scene.registerAfterRender(() => {
+        this.xr = xr;
-            this.currentMesh= this.scene.getPointerOverMesh();
+
-        });
         this.controller.onMotionControllerInitObservable.add((init) => {
             if (init.components['xr-standard-trigger']) {
                 init.components['xr-standard-trigger']
                     .onButtonStateChangedObservable
-                    .add((value) => {
+                    .add(() => {
-                        if (value.value == 1) {
+
-                            console.log(value);
-                        }
                     });
             }
             this.initGrip(init.components['xr-standard-squeeze']);
         });
     }
+    private createCopy(mesh: AbstractMesh) {
+        if (!mesh.isAnInstance) {
+            return  new InstancedMesh("new", (mesh as Mesh));
+        } else {
+            return  new InstancedMesh("new", (mesh as InstancedMesh).sourceMesh);
+        }
 
+    }
     private initGrip(grip: WebXRControllerComponent) {
-        grip.onButtonStateChangedObservable.add((value) => {
+        grip.onButtonStateChangedObservable.add(() => {
-            if (value.value > .5) {
+            if (grip.changes.pressed) {
-                if (this.currentMesh) {
+                if (grip.pressed){
-                    this.currentMesh.setParent(this.controller.pointer);
+                    let mesh = this.scene.meshUnderPointer;
+                    if (this.xr.pointerSelection.getMeshUnderPointer) {
+                        mesh = this.xr.pointerSelection.getMeshUnderPointer(this.controller.uniqueId);
+                    }
+                    if (!mesh) {
+                        return;
+                    }
+                    if (!mesh?.metadata?.template) {
+                        if (mesh.id == "handle") {
+                            mesh && mesh.setParent(this.controller.motionController.rootMesh);
+                            this.grabbedMesh = mesh;
+                        } else {
+                            return;
+                        }
+
+                    }
+                    this.previousParent = mesh?.parent?.id;
+                    this.previousRotation = mesh?.rotation.clone();
+                    this.previousScaling = mesh?.scaling.clone();
+                    this.previousPosition = mesh?.position.clone();
+
+                    if ("toolbox" != mesh?.parent?.parent?.id) {
+                        mesh && mesh.setParent(this.controller.motionController.rootMesh);
+                        this.grabbedMesh = mesh;
+                    } else {
+                        const newMesh = this.createCopy(mesh);
+                        newMesh.position = mesh.absolutePosition.clone();
+                        newMesh.rotation = mesh.absoluteRotationQuaternion.toEulerAngles().clone();
+                        newMesh.scaling = mesh.absoluteScaling.clone();
+                        newMesh.material = mesh.material;
+                        newMesh.metadata = mesh.metadata;
+                        newMesh && newMesh.setParent(this.controller.motionController.rootMesh);
+
+                        this.grabbedMesh = newMesh;
+                        this.previousParent = null;
                     }
                 } else {
-                if (this.currentMesh) {
+                    let mesh = this.scene.meshUnderPointer;
-                    this.currentMesh.setParent(null);
+                    if (this.xr.pointerSelection.getMeshUnderPointer) {
+                        mesh = this.xr.pointerSelection.getMeshUnderPointer(this.controller.uniqueId);
+                    }
+                    if (!this.grabbedMesh) {
+                        console.log("no grabbed mesh");
+                        return;
+                    }
+                    if (mesh?.id != this?.grabbedMesh?.id) {
+                        console.log("not the same mesh");
+                    }
+                    mesh = this.grabbedMesh;
+                    if (!mesh?.metadata?.template) {
+                        if (mesh.id == "handle") {
+                            mesh && mesh.setParent(null);
+                            this.grabbedMesh = null;
+                            this.previousParent = null;
+                            return;
                         }
                     }
 
+                    if (this.previousParent) {
+                        const p = this.scene.getMeshById(this.previousParent);
+                        if (p) {
+                            mesh && mesh.setParent(this.scene.getMeshById(this.previousParent));
+                        } else {
+                            mesh && mesh.setParent(null);
+                        }
+                    } else {
+                        mesh && mesh.setParent(null)
+                    }
+                    const entity = MeshConverter.toDiagramEntity(mesh);
+                    const event: DiagramEvent = {
+                        type: DiagramEventType.DROP,
+                        entity: entity
+                    }
+                    this.previousParent = null;
+                    this.previousScaling = null;
+                    this.previousRotation = null;
+                    this.previousPosition = null;
+                    DiagramManager.onDiagramEventObservable.notifyObservers(event);
+                }
+            }
         });
     }
 }

src/controllers/controllers.ts

@@ -5,6 +5,18 @@ export enum ControllerMovementMode {
     ROTATE,
     TRANSLATE
 }
+export class MeshHoverEvent {
+    public readonly mesh: AbstractMesh;
+    public readonly pointerId: string;
+    public readonly pointerMeshId: string;
+    public readonly isHovered: boolean;
+    constructor(mesh: AbstractMesh, isHovered: boolean, pointerId: string, pointerMeshId: string) {
+        this.mesh = mesh;
+        this.isHovered = isHovered;
+        this.pointerId = pointerId;
+        this.pointerMeshId = pointerMeshId;
+    }
+}
 export class Controllers {
     public static movable: TransformNode | AbstractMesh;
     public static controllerObserver =  new Observable();

src/controllers/left.ts

@@ -1,4 +1,4 @@
-import {Scene, Vector3, WebXRInputSource} from "@babylonjs/core";
+import {Scene, Vector3, WebXRDefaultExperience, WebXRInputSource} from "@babylonjs/core";
 import {Base} from "./base";
 import {Controllers} from "./controllers";
 
@@ -7,9 +7,9 @@ export class Left extends Base {
     public static instance: Left;
 
     constructor(controller:
-                    WebXRInputSource, scene: Scene) {
+                    WebXRInputSource, scene: Scene, xr: WebXRDefaultExperience) {
 
-        super(controller, scene);
+        super(controller, scene, xr);
 
         Left.instance = this;
         this.controller.onMotionControllerInitObservable.add((init) => {
@@ -21,8 +21,6 @@ export class Left extends Base {
                     } else {
                         this.moveMovable(value);
                     }
-
-
                 });
             }
         });

src/controllers/right.ts

@@ -1,35 +1,35 @@
 import {Base} from "./base";
-import {Angle, Scene, Vector3, WebXRControllerComponent, WebXRInputSource} from "@babylonjs/core";
+import {
-import {Bmenu} from "../menus/bmenu";
+    Angle,
-import {DiagramManager} from "../diagram/diagramManager";
+    Scene,
+    Vector3,
+    WebXRControllerComponent,
+    WebXRDefaultExperience,
+    WebXRInputSource
+} from "@babylonjs/core";
 import {ControllerMovementMode, Controllers} from "./controllers";
-import {BmenuState} from "../menus/MenuState";
-import {DiagramEvent, DiagramEventType} from "../diagram/diagramEntity";
 
 export class Right extends Base {
-    private bmenu: Bmenu;
     public static instance: Right;
 
-    private down: boolean = false;
-
     constructor(controller:
-                    WebXRInputSource, scene: Scene) {
+                    WebXRInputSource, scene: Scene, xr: WebXRDefaultExperience) {
-        super(controller, scene);
+        super(controller, scene, xr);
         Right.instance = this;
         this.controller.onMotionControllerInitObservable.add((init) => {
             this.initTrigger(init.components['xr-standard-trigger']);
             this.initBButton(init.components['b-button']);
             this.initAButton(init.components['a-button']);
             this.initThumbstick(init.components['xr-standard-thumbstick']);
-
-
         });
+
+
     }
     private initBButton(bbutton: WebXRControllerComponent) {
         if (bbutton) {
-            bbutton.onButtonStateChangedObservable.add((value) => {
+            bbutton.onButtonStateChangedObservable.add((button) => {
-                if (value.pressed) {
+                if (button.pressed) {
-                    this.bmenu.toggle(this.controller.grip);
+                    Controllers.controllerObserver.notifyObservers({type: 'b-button', value: button.value});
                 }
             });
         }
@@ -39,20 +39,9 @@ export class Right extends Base {
         if (trigger) {
             trigger
                 .onButtonStateChangedObservable
-                .add((value) => {
+                .add((button) => {
-                    if (value.value > .4 && !this.down) {
+                    if (button.pressed) {
-                        this.down = true;
+                        Controllers.controllerObserver.notifyObservers({type: 'trigger', value: button.value});
-                        if (this.bmenu.getState() == BmenuState.ADDING) {
-                            this.bmenu.setState(BmenuState.DROPPING);
-                            const event: DiagramEvent = {
-                                type: DiagramEventType.DROP,
-                                entity: null
-                            }
-                            DiagramManager.onDiagramEventObservable.notifyObservers(event);
-                        }
-                    }
-                    if (value.value < .05) {
-                        this.down = false;
                     }
                 });
         }
@@ -62,14 +51,7 @@ export class Right extends Base {
         if (abutton) {
             abutton.onButtonStateChangedObservable.add((value) => {
                 if (value.pressed) {
-                    if (DiagramManager.currentMesh) {
+                    Controllers.controllerObserver.notifyObservers({type: 'menu'});
-                        if (Controllers.movable) {
-                            Controllers.movable = null;
-                        } else {
-                            Controllers.movable = DiagramManager.currentMesh;
-                        }
-
-                    }
                 }
             });
         }
@@ -114,13 +96,6 @@ export class Right extends Base {
         }
     }
 
-
-
-    public setBMenu(menu: Bmenu) {
-        this.bmenu = menu;
-        this.bmenu.setController(this.controller);
-    }
-
     private rotateMovable(value: { x: number; y: number }) {
         if (Math.abs(value.y) > .1) {
             Controllers.movable.rotation.x +=

src/controllers/rigplatform.ts

@@ -17,8 +17,8 @@ import {
 import {Right} from "./right";
 import {Left} from "./left";
 import {Bmenu} from "../menus/bmenu";
-import {Hud} from "../information/hud";
 import {Controllers} from "./controllers";
+import {BmenuState} from "../menus/MenuState";
 
 
 export class Rigplatform {
@@ -36,6 +36,7 @@ export class Rigplatform {
     private turning: boolean = false;
 
     constructor(scene: Scene, xr: WebXRDefaultExperience) {
+
         this.scene = scene;
         Rigplatform.xr = xr;
         Rigplatform.instance = this;
@@ -43,12 +44,15 @@ export class Rigplatform {
         this.bMenu = new Bmenu(scene, xr.baseExperience);
         this.camera = scene.activeCamera;
         this.rigMesh = MeshBuilder.CreateBox("platform", {width: 2, height: .02, depth: 2}, scene);
-        new Hud(this.rigMesh, scene);
+        //new Hud(this.rigMesh, scene);
+
         for (const cam of scene.cameras) {
             cam.parent = this.rigMesh;
-            cam.position = new Vector3(0, 1.6, 0);
+
+            //cam.position = new Vector3(0, 1.6, 0);
         }
 
+
         const myMaterial = new StandardMaterial("myMaterial", scene);
         myMaterial.diffuseColor = Color3.Blue();
         this.rigMesh.material = myMaterial;
@@ -84,7 +88,7 @@ export class Rigplatform {
         const ray = this.camera.getForwardRay();
         const direction = ray.direction.applyRotationQuaternion(Rigplatform.x90).scale(val);
         this.body.setLinearVelocity(direction);
-        console.log(val);
+        //console.log(val);
     }
 
     public stop() {
@@ -122,7 +126,6 @@ export class Rigplatform {
                 this.body.setAngularVelocity(Vector3.Zero());
             }
         }
-
     }
 
     #initializeControllers() {
@@ -130,8 +133,7 @@ export class Rigplatform {
             let controller;
             switch (source.inputSource.handedness) {
                 case "right":
-                    Right.instance = new Right(source, this.scene);
+                    Right.instance = new Right(source, this.scene, Rigplatform.xr);
-                    Right.instance.setBMenu(this.bMenu);
                     Controllers.controllerObserver.add((event: { type: string, value: number }) => {
                         switch (event.type) {
                             case "turn":
@@ -149,13 +151,14 @@ export class Rigplatform {
                             case "stop":
                                 this.stop();
                                 break;
-
+                            case "menu":
+                                this.bMenu.toggle();
+                                break;
                         }
-
                     });
                     break;
                 case "left":
-                    Left.instance = new Left(source, this.scene);
+                    Left.instance = new Left(source, this.scene, Rigplatform.xr);
                     break;
 
             }
@@ -174,41 +177,43 @@ export class Rigplatform {
         ///simplify this with a map
 
         window.addEventListener("keydown", (ev) => {
+            if (this.bMenu.getState() !== BmenuState.MODIFYING) {
                 switch (ev.key) {
                     case "w":
-                    this.forwardback(Rigplatform.LINEAR_VELOCITY);
+                        this.forwardback(-.1);
                         break;
                     case "s":
-                    this.forwardback(-1 * Rigplatform.LINEAR_VELOCITY);
+                        this.forwardback(.1);
                         break;
                     case "a":
-                    this.leftright(Rigplatform.LINEAR_VELOCITY);
+                        this.leftright(-.2);
                         break;
                     case "d":
-                    this.leftright(-1 * Rigplatform.LINEAR_VELOCITY);
+                        this.leftright(.2);
                         break;
                     case "q":
-                    this.turn(-1 * Rigplatform.ANGULAR_VELOCITY);
+                        this.turn(-1);
                         break;
                     case "e":
-                    this.turn(Rigplatform.ANGULAR_VELOCITY);
+                        this.turn(1);
                         break;
                     case "W":
-                    this.updown(-1 * Rigplatform.LINEAR_VELOCITY);
+                        this.updown(-.1);
                         break;
                     case "S":
-                    this.updown(Rigplatform.LINEAR_VELOCITY);
+                        this.updown(.1);
                         break;
                     case " ":
-                    this.bMenu.toggle(this.rigMesh)
+
+                }
             }
 
         });
         window.addEventListener("keyup", (ev) => {
             const keys = "wsadqeWS";
-
             if (keys.indexOf(ev.key) > -1) {
                 this.stop();
+                this.turn(0);
             }
         });
     }
@@ -217,9 +222,13 @@ export class Rigplatform {
         this.scene.registerBeforeRender(() => {
             const q = this.rigMesh.rotationQuaternion;
             this.body.setAngularVelocity(Vector3.Zero());
+            if (q) {
                 const e = q.toEulerAngles();
                 e.y += this.yRotation;
                 q.copyFrom(Quaternion.FromEulerAngles(0, e.y, 0));
+            }
+
+
         });
     }
 }

src/diagram/diagramManager.ts

@@ -1,17 +1,17 @@
 import {
     AbstractMesh,
-    Color3,
+    InputBlock, Material, NodeMaterial,
     Observable,
     Scene,
-    StandardMaterial,
     WebXRExperienceHelper
 } from "@babylonjs/core";
-import {v4 as uuidv4} from 'uuid';
+
 import {DiagramEntity, DiagramEvent, DiagramEventType} from "./diagramEntity";
 import {IPersistenceManager} from "./persistenceManager";
 import {IndexdbPersistenceManager} from "./indexdbPersistenceManager";
 import {MeshConverter} from "./meshConverter";
 
+
 export class DiagramManager {
     private persistenceManager: IPersistenceManager = new IndexdbPersistenceManager("diagram");
     static onDiagramEventObservable = new Observable();
@@ -19,6 +19,7 @@ export class DiagramManager {
     private xr: WebXRExperienceHelper;
     static currentMesh: AbstractMesh;
 
+    private materialMap: Map<string, Material> = new Map<string, Material>();
     constructor(scene: Scene, xr: WebXRExperienceHelper) {
         this.scene = scene;
         this.xr = xr;
@@ -36,66 +37,41 @@ export class DiagramManager {
 
 
     #onRemoteEvent(event: DiagramEntity) {
-        const mesh = this.#createMesh(event);
+        //const mesh = Toolbox.instance.newMesh(ToolType[Object.entries(ToolType).find(e => e[1] == event.template)[0]], event.id);
-        if (!mesh.material) {
+        const mesh = MeshConverter.fromDiagramEntity(event, this.scene);
-            const material = new StandardMaterial("material-" + event.id, this.scene);
+        if (event.parent) {
-            material.diffuseColor = Color3.FromHexString(event.color);
+            mesh.parent = this.scene.getMeshById(event.parent);
-            mesh.material = material;
         }
+    }
+    private buildNodeMaterial() {
+        const nodeMaterial = new NodeMaterial("nodeMaterial", this.scene, { emitComments: true });
+        const positionInput = new InputBlock("position");
+        positionInput.setAsAttribute("position");
 
     }
-
     #onDiagramEvent(event: DiagramEvent) {
         const entity = event.entity;
         let mesh;
         if (entity) {
-            mesh = this.scene.getMeshByName(entity.id);
+            mesh = this.scene.getMeshById(entity.id);
         }
-
         switch (event.type) {
             case DiagramEventType.CLEAR:
-                if (DiagramManager.currentMesh) {
-                    DiagramManager.currentMesh.dispose();
-                    DiagramManager.currentMesh = null;
-                }
                 break;
             case DiagramEventType.DROPPED:
                 break;
             case DiagramEventType.DROP:
-                if (DiagramManager.currentMesh) {
+                this.persistenceManager.add(mesh);
-                    const newName = uuidv4();
-                    const newMesh = DiagramManager.currentMesh.clone("id" + newName, DiagramManager.currentMesh.parent);
-                    newMesh.id = "id" + newName;
-                    newMesh.material = DiagramManager.currentMesh.material.clone("material" + newName);
-                    DiagramManager.currentMesh.setParent(null);
-                    this.persistenceManager.add(DiagramManager.currentMesh);
-                    DiagramManager.onDiagramEventObservable.notifyObservers({
-                        type: DiagramEventType.DROPPED,
-                        entity: MeshConverter.toDiagramEntity(DiagramManager.currentMesh)
-                    });
-                    DiagramManager.currentMesh = newMesh;
-                }
                 break;
             case DiagramEventType.ADD:
-                if (DiagramManager.currentMesh) {
-                    DiagramManager.currentMesh.dispose();
-                }
-                if (mesh) {
-                    return;
-                } else {
-                    mesh = this.#createMesh(entity);
-                }
-                DiagramManager.currentMesh = mesh;
                 break;
             case DiagramEventType.MODIFY:
-                if (!mesh) {
-
-                } else {
-
-                }
-                DiagramManager.currentMesh = mesh;
                 break;
             case DiagramEventType.REMOVE:
+                if (mesh) {
+                    this.persistenceManager.remove(mesh);
+                    mesh.dispose();
+                }
                 break;
         }
     }

src/diagram/indexdbPersistenceManager.ts

@@ -12,6 +12,10 @@ export class IndexdbPersistenceManager implements  IPersistenceManager {
         this.db.version(1).stores({entities: "id,position,rotation,last_seen,template,text,scale,color"});
     }
     public  add(mesh: AbstractMesh) {
+        if (!mesh) {
+            console.log("Adding null mesh");
+            return;
+        }
         const entity = <any>MeshConverter.toDiagramEntity(mesh);
         entity.position = this.vectoxys(mesh.position);
         entity.rotation = this.vectoxys(mesh.rotation);
@@ -26,11 +30,15 @@ export class IndexdbPersistenceManager implements  IPersistenceManager {
         return new Vector3(xyz.x, xyz.y, xyz.z);
     }
 
-    public  remove() {
+    public  remove(mesh: AbstractMesh)  {
-
+        this.db["entities"].delete(mesh.id);
     }
-    public  modify() {
+    public  modify(mesh) {
-
+        const entity = <any>MeshConverter.toDiagramEntity(mesh);
+        entity.position = this.vectoxys(mesh.position);
+        entity.rotation = this.vectoxys(mesh.rotation);
+        entity.scale = this.vectoxys(mesh.scaling);
+        this.db["entities"].update(mesh.id, entity);
     }
     public initialize() {
         this.db['entities'].each((e) => {

src/diagram/meshConverter.ts

@@ -1,18 +1,15 @@
 import {DiagramEntity} from "./diagramEntity";
-import {
+import {AbstractMesh, Color3, InstancedMesh, Mesh, Scene, StandardMaterial} from "@babylonjs/core";
-    AbstractMesh,
-    Color3,
-    DynamicTexture,
-    Mesh,
-    MeshBuilder,
-    Scene,
-    StandardMaterial
-} from "@babylonjs/core";
 import {v4 as uuidv4} from 'uuid';
+import {Toolbox} from "../toolbox/toolbox";
+
 
 export class MeshConverter {
     public static toDiagramEntity(mesh: AbstractMesh): DiagramEntity {
         const entity = <DiagramEntity>{};
+        if ("new" == mesh?.id) {
+            mesh.id = "id" + uuidv4();
+        }
         entity.id = mesh.id;
         entity.position = mesh.position;
         entity.rotation = mesh.rotation;
@@ -25,70 +22,36 @@ export class MeshConverter {
         }
         return entity;
     }
-    public static fromDiagramEntity(entity: DiagramEntity, scene: Scene): AbstractMesh {
 
+    public static fromDiagramEntity(entity: DiagramEntity, scene: Scene): AbstractMesh {
         if (!entity.id) {
             entity.id = "id" + uuidv4();
         }
-            let mesh: Mesh;
+        let mesh = scene.getMeshById(entity.id);
-            switch (entity.template) {
+        if (mesh) {
-                case "#plane-template":
+            console.log('mesh already exists');
-
+        } else {
-                case "#text-template":
+            mesh = scene.getMeshById("tool-" + entity.template + "-" + entity.color);
-
+            if (mesh) {
-                    const material = new StandardMaterial("material-" + entity.id, scene);
+                if (mesh.isAnInstance) {
-                    material.backFaceCulling = false;
+                    console.log('error: mesh is an instance');
-                    const font_size = 48;
+                } else {
-                    const font = "bold 48px roboto";
+                    mesh = new InstancedMesh(entity.id, (mesh as Mesh));
-                    const planeHeight=1;
-                    const DTHeight = 1.5*font_size;
-                    const ratio = planeHeight / DTHeight;
-                    const text = 'This is some text to put on a plane';
-                    const tempText = new DynamicTexture("dynamic texture", 64, scene);
-                    const tempContext = tempText.getContext();
-                    tempContext.font = font;
-                    const DTWidth = tempContext.measureText(text).width;
-                    const planeWidth = DTWidth * ratio;
-
-
-                    const myDynamicTexture = new DynamicTexture("dynamic texture",
-                        {width: DTWidth, height: DTHeight},
-                        scene, false);
-                    mesh= MeshBuilder.CreatePlane(entity.id, {
-                        width: planeWidth,
-                        height: planeHeight
-                    }, scene);
-
-                    myDynamicTexture.drawText('This is some short text',
-                        null, null,
-
-                        font, "#000000", "#FFFFFF",
-                        true, true);
-                    material.diffuseTexture = myDynamicTexture;
-                    mesh.material = material;
-
-                    break;
-                case "#box-template":
-                    mesh = MeshBuilder.CreateBox(entity.id,
-                        {
-                            width: 1,
-                            height: 1,
-                            depth: 1
-                        }, scene);
-
-                    break;
-                case "#sphere-template":
-                    mesh = MeshBuilder.CreateSphere(entity.id, {diameter: 1}, scene);
-                    break
-                case "#cylinder-template":
-                    mesh = MeshBuilder.CreateCylinder(entity.id, {
-                        diameter: 1,
-                        height: 1
-                    }, scene);
-                    break;
-                default:
-                    mesh = null;
                 }
+            } else {
+                console.log('no mesh found for ' + entity.template + "-" + entity.color);
+                Toolbox.instance.updateToolbox(entity.color);
+                mesh = scene.getMeshById("tool-" + entity.template + "-" + entity.color);
+                if (!mesh) {
+                    console.log('no mesh found for ' + entity.template + "-" + entity.color);
+                } else {
+                    mesh = new InstancedMesh(entity.id, (mesh as Mesh));
+                }
+                //Toolbox.instance.buildTool(Toolbox.getToolTypeFromString(entity.template), entity.color);
+            }
+        }
+
+
         if (mesh) {
             mesh.metadata = {template: entity.template};
             if (entity.text) {
@@ -101,7 +64,7 @@ export class MeshConverter {
                 mesh.rotation = entity.rotation;
             }
             if (entity.parent) {
-                    mesh.parent = scene.getMeshByName(entity.parent);
+                mesh.parent = scene.getNodeById(entity.parent);
             }
             if (entity.scale) {
                 mesh.scaling = entity.scale;
@@ -111,6 +74,7 @@ export class MeshConverter {
                 material.diffuseColor = Color3.FromHexString(entity.color);
                 mesh.material = material;
             }
+
         }
 
         return mesh;

src/integration/ring/cameras.ts

@@ -16,7 +16,7 @@ export class Cameras {
 
         const cameras = await axios.get('https://local.immersiveidea.com/api/cameras');
         this.cameras = cameras;
-        console.log(cameras);
+        //console.log(cameras);
     }
 
     public createCameras() {

src/menus/MenuState.ts

@@ -2,6 +2,7 @@ export enum BmenuState {
     NONE,
     ADDING, // Adding a new entity
     DROPPING, // Dropping an entity
+    MODIFYING, // Editing an entity
     REMOVING, // Removing an entity
 
 }

src/menus/bmenu.ts

@@ -1,38 +1,62 @@
-import {AbstractMesh, Scene, Vector3, WebXRExperienceHelper, WebXRInputSource} from "@babylonjs/core";
+import {
-import {GUI3DManager, NearMenu, TouchHolographicButton} from "@babylonjs/gui";
+    GizmoManager,
+    PointerEventTypes,
+    Scene,
+    Vector3,
+    WebXRExperienceHelper
+} from "@babylonjs/core";
+import {Button3D, GUI3DManager, InputText, StackPanel3D, TextBlock} from "@babylonjs/gui";
 import {DiagramManager} from "../diagram/diagramManager";
 import {BmenuState} from "./MenuState";
-import {DiagramEntity, DiagramEvent, DiagramEventType} from "../diagram/diagramEntity";
+import {DiagramEvent, DiagramEventType} from "../diagram/diagramEntity";
-
+import {MeshConverter} from "../diagram/meshConverter";
 
 export class Bmenu {
     private state: BmenuState = BmenuState.NONE;
     private manager: GUI3DManager;
     private readonly scene: Scene;
-
+    private gizmoManager: GizmoManager;
-    private rightController: AbstractMesh;
     private xr: WebXRExperienceHelper;
+    private textInput: any;
 
     constructor(scene: Scene, xr: WebXRExperienceHelper) {
+        // this.textInput = document.createElement("input");
+        //this.textInput.type = "text";
+        // document.body.appendChild(this.textInput);
         this.scene = scene;
         this.xr = xr;
+        this.gizmoManager = new GizmoManager(scene);
 
-        DiagramManager.onDiagramEventObservable.add((event: DiagramEvent) => {
+        this.scene.onPointerObservable.add((pointerInfo) => {
-            if (event.type === DiagramEventType.DROPPED) {
+            switch (pointerInfo.type) {
-                this.state = BmenuState.ADDING;
+                case PointerEventTypes.POINTERPICK:
+                    if (pointerInfo.pickInfo?.pickedMesh?.metadata?.template &&
+                        pointerInfo.pickInfo?.pickedMesh?.parent?.parent?.id != "toolbox") {
+                        switch (this.state) {
+                            case BmenuState.REMOVING:
+                                console.log("removing " + pointerInfo.pickInfo.pickedMesh.id);
+                                const event: DiagramEvent = {
+                                    type: DiagramEventType.REMOVE,
+                                    entity:
+                                        MeshConverter.toDiagramEntity(pointerInfo.pickInfo.pickedMesh)
+                                }
+                                DiagramManager.onDiagramEventObservable.notifyObservers(event);
+                                break;
+                        }
+                        break;
+                    }
             }
         });
-
-    }
-
-    setController(controller: WebXRInputSource) {
-        this.rightController = controller.grip;
     }
 
     makeButton(name: string, id: string) {
-        const button = new TouchHolographicButton(name);
+        const button = new Button3D(name);
-        button.text = name;
+        button.scaling = new Vector3(.1, .1, .1);
         button.name = id;
+        const text = new TextBlock(name, name);
+        text.fontSize = "24px";
+        text.color = "white";
+        button.content = text;
         button.onPointerClickObservable.add(this.#clickhandler, -1, false, this);
         return button;
     }
@@ -45,75 +69,36 @@ export class Bmenu {
         this.state = state;
     }
 
-    toggle(mesh: AbstractMesh) {
+    toggle() {
-        console.log(mesh.name);
+        //console.log(mesh.name);
         if (this.manager) {
             this.manager.dispose();
             this.manager = null;
         } else {
             this.manager = new GUI3DManager(this.scene);
-            const panel = new NearMenu();
+            const panel = new StackPanel3D();
             this.manager.addControl(panel);
-            const follower = panel.defaultBehavior.followBehavior;
+            panel.addControl(this.makeButton("Modify", "modify"));
-            follower.maxViewHorizontalDegrees = 45;
+            panel.addControl(this.makeButton("Remove", "remove"));
-            follower.useFixedVerticalOffset = true;
-            follower.fixedVerticalOffset = 1;
-            follower.defaultDistance = 2;
-            follower.maximumDistance = 3;
-            follower.minimumDistance = 1;
-
-            panel.backPlateMargin = .01;
-            panel.scaling = new Vector3(.5, .5, .1);
-            panel.margin = .01;
-            //panel.scaling.x = .5;
-            //panel.scaling.y = .5;
-            //const camdir = panel.mesh.getDirection(this.xr.camera.globalPosition);
-            //panel.mesh.lookAt(this.xr.camera.globalPosition);
-            panel.addButton(this.makeButton("Add Box", "addBox"));
-            panel.addButton(this.makeButton("Add Sphere", "addSphere"));
-            panel.addButton(this.makeButton("Add Cylinder", "addCylinder"));
-            panel.addButton(this.makeButton("Add Text", "addText"));
-            panel.addButton(this.makeButton("Remove", "remove"));
-            panel.addButton(this.makeButton("Done Adding", "doneAdding"));
             this.manager.controlScaling = .5;
-
+            const offset = new Vector3(0, -.2, 3);
+            offset.applyRotationQuaternionInPlace(this.scene.activeCamera.absoluteRotation);
+            panel.node.position =
+                this.scene.activeCamera.globalPosition.add(offset);
+            panel.node.lookAt(this.scene.activeCamera.globalPosition);
+            panel.node.rotation.y = panel.node.rotation.y + Math.PI;
         }
     }
 
     #clickhandler(_info, state) {
-        console.log(state.currentTarget.name);
-
-        const id = this?.rightController?.id || null;
-        let entity: DiagramEntity = {
-            template: null,
-            position: new Vector3(-0.02, -.090, .13),
-            rotation: new Vector3(76.04, 0, 0),
-            scale: new Vector3(.1, .1, .1),
-            color: "#CC0000",
-            text: "text",
-            last_seen: new Date(),
-            parent: id
-        };
-
         switch (state.currentTarget.name) {
-            case "addBox":
+            case "modify":
-                entity.template = "#box-template";
+                this.state = BmenuState.MODIFYING;
-                this.state = BmenuState.ADDING;
+                this.gizmoManager.boundingBoxGizmoEnabled = true;
-                break;
+                this.gizmoManager.gizmos.boundingBoxGizmo.scaleBoxSize = .01;
-            case "addSphere":
+                this.gizmoManager.gizmos.boundingBoxGizmo.rotationSphereSize = .01;
-                entity.template = "#sphere-template";
+                this.gizmoManager.gizmos.boundingBoxGizmo.scaleDragSpeed = 1;
-                this.state = BmenuState.ADDING;
+                this.gizmoManager.usePointerToAttachGizmos = false;
-                break;
-            case "addCylinder":
-                entity.template = "#cylinder-template";
-                this.state = BmenuState.ADDING;
-                break;
-            case "addText":
-                entity.template = "#text-template";
-                this.state = BmenuState.ADDING;
-                break;
-            case "doneAdding":
-                this.state = BmenuState.NONE;
                 break;
             case "remove":
                 this.state = BmenuState.REMOVING;
@@ -122,17 +107,7 @@ export class Bmenu {
                 console.log("Unknown button");
                 return;
         }
-        if (this.state === BmenuState.ADDING) {
+        this.manager.dispose();
-            const event: DiagramEvent = {
+        this.manager = null;
-                type: DiagramEventType.ADD,
-                entity: entity
-            }
-            DiagramManager.onDiagramEventObservable.notifyObservers(event);
-        } else {
-            const event: DiagramEvent = {
-                type: DiagramEventType.CLEAR
-            }
-            DiagramManager.onDiagramEventObservable.notifyObservers(event);
-        }
     }
 }

src/menus/keyboard.ts

@@ -0,0 +1,35 @@
+import {AbstractMesh, MeshBuilder, Scene, Vector3, WebXRExperienceHelper} from "@babylonjs/core";
+import {
+    AdvancedDynamicTexture,
+    Button3D,
+    GUI3DManager, InputText, PlanePanel,
+    StackPanel, StackPanel3D,
+    TextBlock,
+    TouchHolographicButton
+} from "@babylonjs/gui";
+import {MyMenu} from "../util/myMenu";
+
+export class Keyboard {
+    private manager: GUI3DManager;
+    private readonly scene: Scene;
+    private mesh: AbstractMesh;
+    private panel: AbstractMesh;
+    private xr: WebXRExperienceHelper;
+    constructor(scene: Scene, xr: WebXRExperienceHelper, mesh: AbstractMesh ) {
+        this.scene = scene;
+        this.xr = xr;
+        this.mesh = mesh;
+    }
+    public async show() {
+        this.panel = MeshBuilder.CreatePlane("hudPlane", {width: 1, height: 1}, this.scene);
+        const inputTexture = AdvancedDynamicTexture.CreateForMesh(this.panel, 1024, 1024);
+        await inputTexture.parseFromURLAsync("./textInputTexture.json", false);
+
+        this.panel.position = this.xr.camera.getFrontPosition(3);
+        this.panel.position.y = this.panel.position.y + 2;
+        this.panel.lookAt(this.xr.camera.getFrontPosition(-1));
+        this.panel.rotation.y = this.panel.rotation.y + Math.PI;
+
+    }
+
+}

src/server/ring/ringCamera.ts

@@ -14,10 +14,10 @@ export class RingCamera {
 
     public async getCameras() {
         const cams = await this.ringApi.getCameras();
-        console.log(cams[0]);
+        //console.log(cams[0]);
 
         const camid = cams.map((value) => value.id);
-        console.log(camid);
+        //console.log(camid);
         return cams;
     }
 }

src/toolbox/toolbox.ts

@@ -0,0 +1,196 @@
+import {
+    AbstractMesh, Angle,
+    Color3, InstancedMesh, Mesh,
+    MeshBuilder,
+    Scene,
+    StandardMaterial, TransformNode,
+    Vector3,
+    WebXRExperienceHelper
+} from "@babylonjs/core";
+
+import {CameraHelper} from "../util/cameraHelper";
+import {AdvancedDynamicTexture, Button3D, ColorPicker, GUI3DManager, StackPanel3D, TextBlock} from "@babylonjs/gui";
+
+export enum ToolType {
+    BOX ="#box-template",
+    Sphere="#sphere-template",
+    Cylinder="#cylinder-template",
+    Cone  ="#cone-template",
+    PLANE   ="#plane-template",
+    OBJECT  ="#object-template",
+}
+
+export class Toolbox {
+    public static getToolTypeFromString(type: string): ToolType {
+        return ToolType[Object.keys(ToolType).find(() => type)]
+    }
+
+    private index = 0;
+    public static instance: Toolbox;
+    private readonly scene: Scene;
+    private readonly xr: WebXRExperienceHelper;
+    public readonly node : TransformNode;
+    private readonly manager: GUI3DManager;
+    private readonly gridsize = 5;
+    private addPanel: StackPanel3D;
+    constructor (scene:Scene, xr: WebXRExperienceHelper) {
+        this.scene = scene;
+        this.addPanel = new StackPanel3D();
+        this.manager  = new GUI3DManager(scene);
+        this.manager.addControl(this.addPanel);
+        this.node = new TransformNode("toolbox", this.scene);
+        const handle = MeshBuilder.CreateCapsule("handle", { radius: .01 , orientation: Vector3.Right(), height: .3}, this.scene);
+        handle.id = "handle";
+        const handleMaterial =  new StandardMaterial("handle-material", this.scene);
+        handleMaterial.diffuseColor = Color3.FromHexString("#EEEEFF");
+        handle.material = handleMaterial;
+        handle.position = CameraHelper.getFrontPosition(2, this.scene);
+        handle.position.y = 1.6;
+        this.node.parent = handle;
+        this.xr = xr;
+        if (!this.scene.activeCamera) {
+            return;
+        } else {
+            this.buildToolbox();
+        }
+        Toolbox.instance = this;
+    }
+    private buildToolbox() {
+        this.node.position.y = -.2;
+        this.node.scaling= new Vector3(0.5, 0.5, 0.5);
+        const color = "#7777FF";
+        this.buildColor(Color3.FromHexString(color));
+
+        const addButton= new Button3D("add-button");
+        const text = new TextBlock("add-button-text", "Add Color");
+        text.color="white";
+        text.fontSize = "48px";
+        text.text = "Add Color";
+        addButton.content = text;
+        this.addPanel.node.parent = this.node;
+        this.addPanel.addControl(addButton);
+        this.addPanel.node.rotation =
+            new Vector3(
+                Angle.FromDegrees(0).radians(),
+                Angle.FromDegrees(180).radians(),
+                Angle.FromDegrees(0).radians());
+        this.addPanel.node.scaling = new Vector3(.1, .1,.1);
+        this.addPanel.position = new Vector3(0, 0, .5);
+        addButton.onPointerClickObservable.add(() => {
+           this.buildColor(Color3.Random());
+
+
+        });
+
+    }
+    private calculatePosition(i: number) {
+        return (i/this.gridsize)-.5-(1/this.gridsize/2);
+    }
+    private buildColor(color: Color3) {
+        const width = 1;
+        const depth = .2;
+        const material = new StandardMaterial("material-" + color.toHexString(), this.scene);
+        material.diffuseColor = color;
+        const mesh = MeshBuilder.CreateBox("toolbox-color-" + color.toHexString(), {width: width, height: .01, depth: depth}, this.scene);
+        mesh.material = material;
+        mesh.position.z = this.index++/4;
+        mesh.parent = this.node;
+        let i = 0;
+        for (const tool of enumKeys(ToolType)) {
+            const newItem = this.buildTool(ToolType[tool], mesh);
+            if (newItem) {
+                newItem.position = new Vector3(this.calculatePosition(++i), .1, 0);
+            }
+        }
+        const myPlane = MeshBuilder.CreatePlane("myPlane", {width: .1, height: .1}, this.scene);
+        myPlane.parent=mesh;
+        myPlane.position= new Vector3(this.calculatePosition(++i), .1, 0);
+
+        const advancedTexture2 = AdvancedDynamicTexture.CreateForMesh(myPlane, 1024, 1024);
+        const colorPicker = new ColorPicker("color-picker");
+        colorPicker.scaleY = 5;
+        colorPicker.scaleX = 5;
+        colorPicker.value = color;
+        colorPicker.onValueChangedObservable.add((value) => {
+            material.diffuseColor = value;
+            material.id = "material-" + value.toHexString();
+            material.name = "material-" + value.toHexString();
+            mesh.id = "toolbox-color-" + value.toHexString();
+            mesh.name = "toolbox-color-" + value.toHexString();
+        });
+
+        advancedTexture2.addControl(colorPicker);
+        this.addPanel.position.z += .25;
+        this.node.position.z -= .125;
+    }
+    public updateToolbox(color: string) {
+        if (this.scene.getMeshById("toolbox-color-" + color)) {
+            return;
+        } else {
+            this.buildColor(Color3.FromHexString(color));
+        }
+    }
+    private nextPosition() {
+
+    }
+
+    public buildTool(tool: ToolType, parent: AbstractMesh) {
+        let newItem: Mesh;
+        const id = tool + "-" + (parent.material as StandardMaterial).diffuseColor.toHexString();
+        const material = parent.material;
+        const toolname = "tool-" + id;
+        switch (tool) {
+            case ToolType.BOX:
+                newItem = MeshBuilder.CreateBox(toolname, {width: 1, height: 1, depth: 1}, this.scene);
+                break;
+            case ToolType.Sphere:
+                newItem = MeshBuilder.CreateSphere(toolname, {diameter: 1}, this.scene);
+                break;
+            case ToolType.Cylinder:
+                newItem = MeshBuilder.CreateCylinder(toolname, {height: 1, diameter: 1}, this.scene);
+                break;
+            case ToolType.Cone:
+                newItem = MeshBuilder.CreateCylinder(toolname, {diameterTop: 0, height: 1, diameterBottom: 1}, this.scene);
+                break;
+            case ToolType.PLANE:
+                newItem = MeshBuilder.CreatePlane(toolname, {width: 1, height: 1}, this.scene);
+                break;
+            case ToolType.OBJECT:
+                break;
+        }
+        if (newItem)  {
+            newItem.material = material;
+            newItem.id = "tool-" + id;
+            if (tool === ToolType.PLANE) {
+                newItem.material.backFaceCulling = false;
+            }
+            newItem.scaling = new Vector3(0.1, 0.1, 0.1);
+            newItem.parent = parent;
+            if (!newItem.material) {
+                newItem.material = parent.material;
+            }
+            if (newItem.metadata) {
+                newItem.metadata.template = tool;
+            } else {
+                newItem.metadata = {template: tool};
+            }
+            const instance = new InstancedMesh("instance-"+id, newItem);
+            if (instance.metadata) {
+                instance.metadata.template = tool;
+            } else {
+                instance.metadata = {template: tool};
+            }
+            instance.parent= parent;
+            newItem.setEnabled(false)
+            return instance;
+        } else {
+            return null;
+        }
+    }
+    public show() {
+        this.buildToolbox();
+    }
+}
+function enumKeys<O extends object, K extends keyof O = keyof O>(obj: O): K[] {
+    return Object.keys(obj).filter(k => Number.isNaN(+k)) as K[];
+}

src/util/cameraHelper.ts

@@ -0,0 +1,9 @@
+import {Scene, Vector3} from "@babylonjs/core";
+
+export class CameraHelper {
+    public static getFrontPosition(distance: number, scene: Scene): Vector3 {
+        const offset = new Vector3(0, 0, distance);
+        offset.applyRotationQuaternionInPlace(scene.activeCamera.absoluteRotation);
+        return scene.activeCamera.globalPosition.add(offset);
+    }
+}

src/util/gmap.ts

@@ -19,8 +19,8 @@ export class Gmap {
             maptype: 'satellite'
         })
             .on('progress', function (info) {
-                console.log(info.count);
+                //console.log(info.count);
-                console.log(info.total);
+                //console.log(info.total);
                 const image = info.image;
 
                 image.style.position = 'absolute';

src/util/myMenu.ts

@@ -0,0 +1,10 @@
+import {PlanePanel, TouchHolographicMenu} from "@babylonjs/gui";
+
+export class MyMenu extends PlanePanel {
+    public arrangeChildren: boolean = true;
+    protected _arrangeChildren() {
+        if (this.arrangeChildren) {
+            super._arrangeChildren();
+        }
+    }
+}