From 1e174e81d3dd171c50bd181bda63329e86b654e2 Mon Sep 17 00:00:00 2001 From: Michael Mainguy Date: Mon, 29 Dec 2025 18:13:43 -0600 Subject: [PATCH] Add local database mode for browser-only diagrams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add /db/local/:db path type that stores diagrams locally without syncing - New diagrams now default to local storage (browser-only) - Share button creates public copy when sharing local diagrams - Add storage type badges (Local/Public/Private) in diagram manager - Add GitHub Actions workflow for automated builds - Block local- database requests at server with 404 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .github/workflows/build.yml | 31 ++++ SHARING_PLAN.md | 150 ++++++++++++++++++++ SYNC_PLAN.md | 179 ++++++++++++++++++++++++ package.json | 2 +- server/middleware/dbAuth.js | 10 ++ src/integration/database/pouchData.ts | 69 ++++++++- src/react/pages/createDiagramModal.tsx | 13 +- src/react/pages/manageDiagramsModal.tsx | 32 ++++- src/react/pages/vrExperience.tsx | 73 +++++++++- src/react/webRouter.tsx | 5 + src/util/functions/getPath.ts | 38 ++++- 11 files changed, 586 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 SHARING_PLAN.md create mode 100644 SYNC_PLAN.md diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..2edbb62 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,31 @@ +name: Build + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + runs-on: linux_amd64 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: npm ci + + - name: Build Front End + run: npm run build + env: + NODE_OPTIONS: '--max-old-space-size=4096' + + - name: Restart Server + run: | + # Kill existing process on port 3001 + lsof -ti:3001 | xargs kill -9 2>/dev/null || true + + # Start server in background with nohup + nohup npm run start > /tmp/immersive.log 2>&1 & \ No newline at end of file diff --git a/SHARING_PLAN.md b/SHARING_PLAN.md new file mode 100644 index 0000000..9e6014c --- /dev/null +++ b/SHARING_PLAN.md @@ -0,0 +1,150 @@ +# Self-Hosted Diagram Sharing with Express-PouchDB + +## Requirements (Confirmed) +- **Storage**: In-memory (ephemeral) - lost on server restart +- **Content**: Copy current diagram entities when creating share +- **Expiration**: No expiration - links work until server restart +- **Encryption**: None - keep it simple, anyone with link can access + +## Architecture + +``` +┌─────────────────────────────────────────────────┐ +│ Express Server (port 3001) │ +├─────────────────────────────────────────────────┤ +│ /api/share/* → Share management API │ +│ /pouchdb/* → express-pouchdb (sync) │ +│ /share/:uuid → Client share route │ +│ /api/* → Existing API routes │ +│ /* → Vite static files │ +└─────────────────────────────────────────────────┘ + │ + └── In-Memory PouchDB (per share UUID) +``` + +## Implementation Steps + +### Phase 1: Server-Side Setup + +#### 1.1 Add Dependencies +```bash +npm install express-pouchdb pouchdb-adapter-memory +``` + +#### 1.2 Create PouchDB Server Service +**New file: `server/services/pouchdbServer.js`** +- Initialize PouchDB with memory adapter +- Track active share databases in a Map +- Export `getShareDB(shareId)`, `shareExists(shareId)`, `createPouchDBMiddleware()` + +#### 1.3 Create Share API +**New file: `server/api/share.js`** +- `POST /api/share/create` - Generate UUID, create in-memory DB, copy entities +- `GET /api/share/:id/exists` - Check if share exists +- `GET /api/share/stats` - Debug endpoint for active shares + +#### 1.4 Update API Router +**Edit: `server/api/index.js`** +- Add `import shareRouter from "./share.js"` +- Mount at `router.use("/share", shareRouter)` + +#### 1.5 Mount Express-PouchDB +**Edit: `server.js`** +- Import `createPouchDBMiddleware` from pouchdbServer.js +- Mount at `app.use("/pouchdb", createPouchDBMiddleware())` + +--- + +### Phase 2: Client-Side Integration + +#### 2.1 Update URL Parsing +**Edit: `src/util/functions/getPath.ts`** +- Add `getPathInfo()` function returning `{ dbName, isShare, shareId }` +- Detect `/share/:uuid` pattern + +#### 2.2 Update PouchDB Persistence Manager +**Edit: `src/integration/database/pouchdbPersistenceManager.ts`** + +In `initLocal()`: +- Call `getPathInfo()` to detect share URLs +- If share: use `share-{uuid}` as local DB name, call `beginShareSync()` + +Add new method `beginShareSync(shareId)`: +- Check share exists via `/api/share/:id/exists` +- Connect to `${origin}/pouchdb/share-${shareId}` +- Set up presence with `share-${shareId}` as DB name +- Begin live sync (no encryption) + +#### 2.3 Add React Route +**Edit: `src/react/webRouter.tsx`** +- Add route `{ path: "/share/:uuid", element: }` +- No ProtectedRoute wrapper (public access) + +#### 2.4 Add Share Button Handler +**Edit: `src/react/pages/vrExperience.tsx`** +- Add `isShare` prop +- Add `handleShare()` function: + 1. Get all entities from local PouchDB + 2. POST to `/api/share/create` with entities + 3. Copy resulting URL to clipboard + 4. Show confirmation + +--- + +### Phase 3: Presence Integration + +The WebSocket presence system already routes by database name. Since shares use `share-{uuid}` as the database name, presence works automatically. + +**Edit: `server/server.js`** (WebSocket server) +- Update `originIsAllowed()` to allow localhost for development + +--- + +## Files to Modify + +| File | Action | Purpose | +|------|--------|---------| +| `package.json` | Edit | Add express-pouchdb, pouchdb-adapter-memory | +| `server.js` | Edit | Mount /pouchdb middleware | +| `server/api/index.js` | Edit | Add share router | +| `server/services/pouchdbServer.js` | Create | PouchDB memory initialization | +| `server/api/share.js` | Create | Share API endpoints | +| `server/server.js` | Edit | Allow localhost origins | +| `src/util/functions/getPath.ts` | Edit | Add getPathInfo() | +| `src/integration/database/pouchdbPersistenceManager.ts` | Edit | Add share sync logic | +| `src/react/webRouter.tsx` | Edit | Add /share/:uuid route | +| `src/react/pages/vrExperience.tsx` | Edit | Add share button handler | +| `src/util/featureConfig.ts` | Edit | Enable shareCollaborate feature | + +--- + +## User Flow + +### Creating a Share +1. User has a diagram open at `/db/public/mydiagram` +2. Clicks "Share" button +3. Client fetches all entities from local PouchDB +4. POSTs to `/api/share/create` with entities +5. Server creates in-memory DB, copies entities, returns UUID +6. Client copies `https://server.com/share/{uuid}` to clipboard +7. User shares link with collaborators + +### Joining a Share +1. User navigates to `https://server.com/share/{uuid}` +2. React Router renders VrExperience with `isShare=true` +3. PouchdbPersistenceManager detects share URL +4. Checks `/api/share/:uuid/exists` - returns true +5. Creates local PouchDB `share-{uuid}` +6. Connects to `/pouchdb/share-{uuid}` for sync +7. Entities replicate to local, render in scene +8. Presence WebSocket connects with `share-{uuid}` as room + +--- + +## Future Authentication (Not Implemented Now) + +Structure allows easy addition later: +- express-pouchdb middleware can be wrapped with auth middleware +- Share API can require JWT/session tokens +- Could add password-protected shares +- Could add read-only vs read-write permissions diff --git a/SYNC_PLAN.md b/SYNC_PLAN.md new file mode 100644 index 0000000..abe8003 --- /dev/null +++ b/SYNC_PLAN.md @@ -0,0 +1,179 @@ +# Future Sync Strategy: Keeping Local and Public Clones in Sync + +## Current State (v1) + +- Sharing creates a **ONE-TIME COPY** from local to public +- Copies diverge independently after sharing +- No automatic sync between local and public versions +- Local diagrams are browser-only (IndexedDB via PouchDB) +- Public diagrams sync with server via express-pouchdb + +### URL Scheme + +| Route | Sync | Access | Status | +|-------|------|--------|--------| +| `/db/local/:id` | None | Browser-only | Implemented | +| `/db/public/:id` | Yes | Anyone | Implemented | +| `/db/private/:id` | Yes | Authorized users | Route only (no auth) | + +## Future Options + +### Option 1: Manual Push/Pull (Recommended for v2) + +Add explicit user-triggered sync between local and public copies. + +**Features:** +- "Push to Public" button - sends local changes to public copy +- "Pull from Public" button - gets public changes into local +- Track `lastSyncedAt` timestamp +- Show indicator when copies have diverged +- Conflict resolution: Last write wins (simple) or user choice (advanced) + +**Pros:** +- User stays in control +- Clear mental model +- Simple to implement incrementally + +**Cons:** +- Manual effort required +- Risk of forgetting to sync + +### Option 2: Automatic Background Sync + +Continuous bidirectional sync between local and public copies. + +**Features:** +- Real-time sync like Google Docs +- Works across devices +- Offline-first with automatic merge + +**Pros:** +- Seamless experience +- Always up to date + +**Cons:** +- Complex conflict resolution (may need CRDTs) +- Higher performance overhead +- Harder to reason about state + +### Option 3: Fork/Branch Model + +One-way relationship: local is "draft", public is "published". + +**Features:** +- Push only (local → public) +- No pull mechanism +- Public is the "source of truth" once published + +**Pros:** +- Clear mental model +- No merge conflicts +- Simple implementation + +**Cons:** +- Cannot incorporate public changes back to local +- Multiple people can't collaborate on draft + +## Recommended Implementation (v2) + +Implement **Option 1 (Manual Push/Pull)** as it provides the best balance of user control and simplicity. + +### Data Model Changes + +Add to diagram directory entry: +```typescript +interface DiagramEntry { + _id: string; + name: string; + description: string; + storageType: 'local' | 'public' | 'private'; + createdAt: string; + + // New fields for sync tracking + publicCopyId?: string; // ID of the public clone (if shared) + lastPushedAt?: string; // When changes were last pushed to public + lastPulledAt?: string; // When public changes were last pulled + publicVersion?: number; // Version number of public copy at last sync +} +``` + +### API Endpoints + +```typescript +// Push local changes to public +POST /api/sync/push +Body: { localDbName: string, publicDbName: string } +Response: { success: boolean, documentsUpdated: number } + +// Pull public changes to local +POST /api/sync/pull +Body: { localDbName: string, publicDbName: string } +Response: { success: boolean, documentsUpdated: number } + +// Check if copies have diverged +GET /api/sync/status?local={localDbName}&public={publicDbName} +Response: { + diverged: boolean, + localChanges: number, + publicChanges: number, + lastSyncedAt: string +} +``` + +### UI Components + +1. **Sync Status Indicator** + - Shows in header when viewing a local diagram that has a public copy + - Green check: In sync + - Orange dot: Changes pending + - Red warning: Conflicts detected + +2. **Push/Pull Buttons** + - In hamburger menu under "Share" section + - "Push to Public" - shows confirmation with change count + - "Pull from Public" - shows confirmation with change count + +3. **Divergence Warning Badge** + - Shows on diagram card in Manage Diagrams modal + - Indicates when local and public have diverged + +4. **Conflict Resolution Dialog** + - Shows when both local and public have changes to same entity + - Options: Keep Local, Keep Public, Keep Both (creates duplicate) + +### Implementation Phases + +**Phase 1: Tracking** +- Add `publicCopyId` when sharing local → public +- Track sharing relationship in directory + +**Phase 2: Push** +- Implement push from local to public +- Overwrite public with local changes +- Update `lastPushedAt` timestamp + +**Phase 3: Pull** +- Implement pull from public to local +- Merge public changes into local +- Update `lastPulledAt` timestamp + +**Phase 4: Status** +- Implement divergence detection +- Add UI indicators +- Show sync status in Manage Diagrams + +**Phase 5: Conflict Resolution** +- Detect entity-level conflicts +- Show resolution dialog +- Allow user to choose resolution strategy + +## Migration Notes + +Existing diagrams without `storageType` are treated as `public` for backwards compatibility. When such diagrams are loaded, the UI should work correctly but sync tracking features won't be available until the diagram metadata is updated. + +## Security Considerations + +- Push/pull operations should validate that the user has access to both databases +- Public databases remain world-readable/writable +- Private database sync will require authentication tokens +- Rate limiting should be applied to sync operations diff --git a/package.json b/package.json index 9132b07..6dbf8e7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "immersive", "private": true, - "version": "0.0.8-42", + "version": "0.0.8-43", "type": "module", "license": "MIT", "engines": { diff --git a/server/middleware/dbAuth.js b/server/middleware/dbAuth.js index 4390863..76361b5 100644 --- a/server/middleware/dbAuth.js +++ b/server/middleware/dbAuth.js @@ -9,6 +9,7 @@ * * Database naming patterns: * / - Root endpoint, always allowed (server info) + * /local-{dbname} - Should never reach server (client-only), return 404 * /public-{dbname} - No auth required, anyone can read/write * /private-{dbname} - Auth required * /{dbname} - Treated as private by default @@ -24,6 +25,15 @@ export function dbAuthMiddleware(req, res, next) { return next(); } + // Local databases should never reach the server (they're browser-only) + if (dbName.startsWith('local-')) { + console.log(`[DB Auth] Local database access rejected: ${req.method} ${req.path}`); + return res.status(404).json({ + error: 'not_found', + reason: 'Local databases are browser-only and do not sync to server' + }); + } + // Check if this is a public database (name starts with 'public-') const isPublic = dbName.startsWith('public-'); diff --git a/src/integration/database/pouchData.ts b/src/integration/database/pouchData.ts index f5922aa..511976f 100644 --- a/src/integration/database/pouchData.ts +++ b/src/integration/database/pouchData.ts @@ -5,7 +5,7 @@ import {DiagramEventObserverMask} from "../../diagram/types/diagramEventObserver import log, {Logger} from "loglevel"; import PouchDB from 'pouchdb'; import {importDiagramFromJSON, DiagramExport} from "../../util/functions/exportDiagramAsJSON"; -import {isPublicPath, getRemoteDbPath} from "../../util/functions/getPath"; +import {getDbType, getRemoteDbPath, shouldSync} from "../../util/functions/getPath"; export class PouchData { public readonly onDBEntityUpdateObservable: Observable = new Observable(); @@ -26,14 +26,25 @@ export class PouchData { } /** - * Initialize sync with remote express-pouchdb for public databases + * Initialize sync with remote express-pouchdb for public/private databases + * Local databases do not sync */ private initSync() { + const dbType = getDbType(); const remoteDbPath = getRemoteDbPath(); - const isPublic = isPublicPath(); - if (!remoteDbPath || !isPublic) { - this._logger.debug('[Sync] Not a public path, skipping remote sync'); + if (dbType === 'local') { + this._logger.debug('[Sync] Local database - no remote sync'); + return; + } + + if (dbType === 'private') { + this._logger.info('[Sync] Private database sync not yet implemented'); + return; + } + + if (!remoteDbPath || !shouldSync()) { + this._logger.debug('[Sync] No remote path or sync disabled, skipping'); return; } @@ -186,4 +197,52 @@ export class PouchData { this._logger.warn('CONFLICTS!', doc._conflicts); } } + + /** + * Copy all documents from this database to a new public database. + * Used when sharing a local diagram to make it publicly accessible. + * @param newDbName - The name for the new public database + * @returns The URL path to the new public diagram + */ + public async copyToPublic(newDbName: string): Promise { + this._logger.info(`[Copy] Starting copy to public-${newDbName}`); + + // Create the remote public database + const remoteUrl = `${window.location.origin}/pouchdb/public-${newDbName}`; + const remoteDb = new PouchDB(remoteUrl); + + try { + // Get all docs from local database + const allDocs = await this._db.allDocs({ include_docs: true }); + this._logger.debug(`[Copy] Found ${allDocs.rows.length} documents to copy`); + + // Copy each document to the remote database + for (const row of allDocs.rows) { + if (row.doc) { + // Remove PouchDB internal fields for clean insert + const { _rev, ...docWithoutRev } = row.doc; + try { + await remoteDb.put(docWithoutRev); + } catch (err) { + // Document might already exist if this is a retry + this._logger.warn(`[Copy] Failed to copy doc ${row.id}:`, err); + } + } + } + + this._logger.info(`[Copy] Successfully copied ${allDocs.rows.length} documents`); + return `/db/public/${newDbName}`; + } catch (err) { + this._logger.error('[Copy] Error copying to public:', err); + throw err; + } + } + + /** + * Get all documents in the database (for export/copy operations) + */ + public async getAllDocs(): Promise { + const result = await this._db.allDocs({ include_docs: true }); + return result.rows.map(row => row.doc).filter(doc => doc && doc.id !== 'metadata'); + } } \ No newline at end of file diff --git a/src/react/pages/createDiagramModal.tsx b/src/react/pages/createDiagramModal.tsx index 4a699cf..11853f9 100644 --- a/src/react/pages/createDiagramModal.tsx +++ b/src/react/pages/createDiagramModal.tsx @@ -7,11 +7,13 @@ import {useFeatureState} from "../hooks/useFeatures"; import ComingSoonBadge from "../components/ComingSoonBadge"; import UpgradeBadge from "../components/UpgradeBadge"; import {useAuth0} from "@auth0/auth0-react"; +import {useNavigate} from "react-router-dom"; export default function CreateDiagramModal({createOpened, closeCreate}) { const logger = log.getLogger('createDiagramModal'); const db = usePouch(); const { loginWithRedirect } = useAuth0(); + const navigate = useNavigate(); // Feature flags const privateDesignsState = useFeatureState('privateDesigns'); @@ -43,7 +45,14 @@ export default function CreateDiagramModal({createOpened, closeCreate}) { logger.warn('cannot find directory', err); } const id = 'diagram-' + v4(); - const newDiagram = {...diagram, _id: id, type: 'diagram'}; + // All new diagrams start as local (browser-only) + const newDiagram = { + ...diagram, + _id: id, + type: 'diagram', + storageType: 'local', + createdAt: new Date().toISOString() + }; if (!doc) { await db.put({_id: 'directory', diagrams: [newDiagram], type: 'directory'}); } else { @@ -56,6 +65,8 @@ export default function CreateDiagramModal({createOpened, closeCreate}) { await db.put(doc); } closeCreate(); + // Navigate to the local diagram + navigate(`/db/local/${id}`); } return ( diff --git a/src/react/pages/manageDiagramsModal.tsx b/src/react/pages/manageDiagramsModal.tsx index 6d0d5c1..d1e55f9 100644 --- a/src/react/pages/manageDiagramsModal.tsx +++ b/src/react/pages/manageDiagramsModal.tsx @@ -1,4 +1,4 @@ -import {Button, Card, Container, Group, Modal, Paper, SimpleGrid, Stack} from "@mantine/core"; +import {Badge, Button, Card, Container, Group, Modal, Paper, SimpleGrid, Stack} from "@mantine/core"; import React from "react"; import {useDoc, usePouch} from "use-pouchdb"; import {IconTrash, IconDownload} from "@tabler/icons-react"; @@ -7,6 +7,26 @@ import log from "loglevel"; import {useFeatureLimit} from "../hooks/useFeatures"; import {exportDiagramAsJSON} from "../../util/functions/exportDiagramAsJSON"; +// Helper to get the correct URL path based on storage type +function getDiagramPath(diagram: any): string { + const storageType = diagram.storageType || 'public'; // Default to public for backwards compat + return `/db/${storageType}/${diagram._id}`; +} + +// Helper to get badge color and text for storage type +function getStorageTypeBadge(storageType: string): { color: string; label: string } { + switch (storageType) { + case 'local': + return { color: 'gray', label: 'Local' }; + case 'public': + return { color: 'green', label: 'Public' }; + case 'private': + return { color: 'blue', label: 'Private' }; + default: + return { color: 'green', label: 'Public' }; // Default for backwards compat + } +} + export default function ManageDiagramsModal({openCreate, manageOpened, closeManage}) { const logger = log.getLogger('manageDiagramsModal'); const {doc: diagram, error} = useDoc('directory', {}, {_id: 'directory', diagrams: []}); @@ -34,10 +54,16 @@ export default function ManageDiagramsModal({openCreate, manageOpened, closeMana }; const cards = diagrams.map((diagram) => { + const badgeInfo = getStorageTypeBadge(diagram.storageType); + const diagramPath = getDiagramPath(diagram); + return ( - {diagram.name} + + {diagram.name} + {badgeInfo.label} + @@ -46,7 +72,7 @@ export default function ManageDiagramsModal({openCreate, manageOpened, closeMana -