From e6a3329c9bb4d3418e32e7996c43eb43e4f65c0b Mon Sep 17 00:00:00 2001 From: Michael Mainguy Date: Tue, 3 Mar 2026 09:02:20 -0600 Subject: [PATCH] Add Editor source code tools and expand indexed Babylon.js packages Adds search_babylon_editor_source and get_babylon_editor_source MCP tools for searching and retrieving Editor source code. Expands source indexing to include inspector, viewer, addons, accessibility, node-editor, and procedural-textures packages. Improves pathToDocId to handle Editor paths and adds Editor URL construction fallback in getDocumentByPath. Co-Authored-By: Claude Opus 4.6 --- package.json | 3 +- scripts/index-editor-source.ts | 229 ++++++++++++++++++ scripts/index-source.ts | 9 + src/mcp/handlers.test.ts | 2 +- .../editor/get-editor-source.handler.test.ts | 138 +++++++++++ .../editor/get-editor-source.handler.ts | 61 +++++ .../search-editor-source.handler.test.ts | 191 +++++++++++++++ .../editor/search-editor-source.handler.ts | 72 ++++++ src/mcp/handlers/index.ts | 8 +- src/mcp/repository-config.ts | 1 + src/mcp/server.ts | 2 +- src/search/lancedb-search.test.ts | 61 +++++ src/search/lancedb-search.ts | 50 +++- 13 files changed, 816 insertions(+), 11 deletions(-) create mode 100644 scripts/index-editor-source.ts create mode 100644 src/mcp/handlers/editor/get-editor-source.handler.test.ts create mode 100644 src/mcp/handlers/editor/get-editor-source.handler.ts create mode 100644 src/mcp/handlers/editor/search-editor-source.handler.test.ts create mode 100644 src/mcp/handlers/editor/search-editor-source.handler.ts diff --git a/package.json b/package.json index a572f43..f621767 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "index:docs": "tsx scripts/index-docs.ts", "index:api": "tsx scripts/index-api.ts", "index:source": "tsx scripts/index-source.ts", - "index:all": "npm run index:docs && npm run index:api && npm run index:source", + "index:editor-source": "tsx scripts/index-editor-source.ts", + "index:all": "npm run index:docs && npm run index:api && npm run index:source && npm run index:editor-source", "index-docs": "npm run index:docs", "index-api": "npm run index:api", "index-source": "npm run index:source" diff --git a/scripts/index-editor-source.ts b/scripts/index-editor-source.ts new file mode 100644 index 0000000..8a6d507 --- /dev/null +++ b/scripts/index-editor-source.ts @@ -0,0 +1,229 @@ +// MUST set environment variable before any imports that use @xenova/transformers +// This prevents onnxruntime-node from being loaded on Alpine Linux (musl libc) +if (process.env.TRANSFORMERS_BACKEND === 'wasm' || process.env.TRANSFORMERS_BACKEND === 'onnxruntime-web') { + process.env.ONNXRUNTIME_BACKEND = 'wasm'; +} + +import { connect } from '@lancedb/lancedb'; +import { pipeline } from '@xenova/transformers'; +import fs from 'fs/promises'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +interface SourceCodeChunk { + id: string; + filePath: string; + package: string; + content: string; + startLine: number; + endLine: number; + language: string; + imports: string; + exports: string; + url: string; + vector: number[]; +} + +const CHUNK_SIZE = 200; +const CHUNK_OVERLAP = 20; + +async function getAllSourceFiles(dir: string): Promise { + const files: string[] = []; + + try { + const entries = await fs.readdir(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + + if (entry.isDirectory()) { + if (!['node_modules', 'dist', 'build', 'lib', '.git', 'declaration'].includes(entry.name)) { + const subFiles = await getAllSourceFiles(fullPath); + files.push(...subFiles); + } + } else if (entry.isFile()) { + if (/\.(ts|tsx|js|jsx)$/.test(entry.name)) { + files.push(fullPath); + } + } + } + } catch { + return []; + } + + return files; +} + +function extractImports(content: string): string { + const imports: string[] = []; + const importRegex = /import\s+(?:{[^}]+}|[^;]+)\s+from\s+['"]([^'"]+)['"]/g; + let match; + + while ((match = importRegex.exec(content)) !== null) { + if (match[1]) { + imports.push(match[1]); + } + } + + return imports.slice(0, 20).join(', '); +} + +function extractExports(content: string): string { + const exports: string[] = []; + const exportRegex = /export\s+(?:class|function|interface|type|const|let|var|enum|default)\s+([A-Za-z_$][A-Za-z0-9_$]*)/g; + let match; + + while ((match = exportRegex.exec(content)) !== null) { + if (match[1]) { + exports.push(match[1]); + } + } + + return exports.slice(0, 20).join(', '); +} + +function extractComments(code: string): string { + const comments: string[] = []; + + const singleLineRegex = /\/\/\s*(.+)$/gm; + let match; + while ((match = singleLineRegex.exec(code)) !== null) { + if (match[1]) { + comments.push(match[1].trim()); + } + } + + const multiLineRegex = /\/\*\*?([\s\S]*?)\*\//g; + while ((match = multiLineRegex.exec(code)) !== null) { + if (match[1]) { + comments.push(match[1].trim()); + } + } + + return comments.slice(0, 5).join(' '); +} + +async function main() { + const projectRoot = path.join(__dirname, '..'); + const dbPath = path.join(projectRoot, 'data', 'lancedb'); + const repositoryPath = path.join(projectRoot, 'data', 'repositories', 'Editor'); + const tableName = 'babylon_editor_source'; + + // Editor packages with their source paths (relative to repo root) + const packages = [ + { name: 'editor', srcPath: 'editor/src' }, + { name: 'tools', srcPath: 'tools/src' }, + { name: 'website', srcPath: 'website/src' }, + ]; + + console.log('Starting Editor source code indexing...'); + console.log(`Database path: ${dbPath}`); + console.log(`Repository path: ${repositoryPath}`); + console.log(`Packages: ${packages.map(p => p.name).join(', ')}`); + console.log(); + + // Initialize LanceDB + console.log('Initializing LanceDB connection...'); + const db = await connect(dbPath); + + // Load embedding model + console.log('Loading embedding model...'); + const embedder = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2'); + console.log('Embedding model loaded'); + + const chunks: SourceCodeChunk[] = []; + let totalFiles = 0; + + for (const pkg of packages) { + console.log(`\nIndexing package: ${pkg.name}...`); + const packagePath = path.join(repositoryPath, pkg.srcPath); + + try { + const files = await getAllSourceFiles(packagePath); + console.log(`Found ${files.length} source files in ${pkg.name}`); + + for (let i = 0; i < files.length; i++) { + const file = files[i]!; + try { + const content = await fs.readFile(file, 'utf-8'); + const lines = content.split('\n'); + + const imports = extractImports(content); + const exports = extractExports(content); + const language = file.endsWith('.ts') || file.endsWith('.tsx') ? 'typescript' : 'javascript'; + const relativePath = path.relative(repositoryPath, file); + + // Chunk the file + for (let j = 0; j < lines.length; j += CHUNK_SIZE - CHUNK_OVERLAP) { + const startLine = j + 1; + const endLine = Math.min(j + CHUNK_SIZE, lines.length); + const chunkLines = lines.slice(j, endLine); + const chunkContent = chunkLines.join('\n'); + + if (chunkContent.trim().length === 0) { + continue; + } + + // Create embedding text + const fileName = path.basename(file); + const dirName = path.dirname(relativePath).split('/').pop() || ''; + const comments = extractComments(chunkContent); + const embeddingText = `${fileName} ${dirName} ${comments} ${chunkContent.substring(0, 1000)}`; + + // Generate embedding + const result = await embedder(embeddingText, { + pooling: 'mean', + normalize: true, + }); + const vector = Array.from(result.data) as number[]; + + // Generate GitHub URL for Editor repo + const url = `https://github.com/BabylonJS/Editor/blob/master/${relativePath}#L${startLine}-L${endLine}`; + + chunks.push({ + id: `${relativePath}:${startLine}-${endLine}`, + filePath: relativePath, + package: pkg.name, + content: chunkContent, + startLine, + endLine, + language, + imports, + exports, + url, + vector, + }); + } + + totalFiles++; + if (totalFiles % 50 === 0) { + console.log(`Processed ${totalFiles} files, ${chunks.length} chunks...`); + } + } catch (error) { + console.error(`Error processing ${file}:`, error); + } + } + } catch (error) { + console.error(`Error indexing package ${pkg.name}:`, error); + } + } + + console.log(`\nTotal files processed: ${totalFiles}`); + console.log(`Total source code chunks: ${chunks.length}`); + console.log('Creating LanceDB table...'); + + // Drop existing table if it exists + const tableNames = await db.tableNames(); + if (tableNames.includes(tableName)) { + await db.dropTable(tableName); + } + + // Create new table + await db.createTable(tableName, chunks); + console.log('\n✓ Editor source code indexing completed successfully!'); +} + +main().catch(console.error); diff --git a/scripts/index-source.ts b/scripts/index-source.ts index 4540fae..33d4e9b 100644 --- a/scripts/index-source.ts +++ b/scripts/index-source.ts @@ -14,6 +14,15 @@ async function main() { 'materials', 'loaders', 'serializers', + 'inspector', + 'inspector-v2', + 'ui-controls', + 'viewer', + 'addons', + 'accessibility', + 'node-editor', + 'node-particle-editor', + 'procedural-textures' ]; console.log('Starting source code indexing for Babylon.js packages...'); diff --git a/src/mcp/handlers.test.ts b/src/mcp/handlers.test.ts index d94a51f..dce5f9d 100644 --- a/src/mcp/handlers.test.ts +++ b/src/mcp/handlers.test.ts @@ -17,7 +17,7 @@ describe('MCP Handlers', () => { it('should register all required tools', () => { setupHandlers(mockServer); - expect(registerToolSpy).toHaveBeenCalledTimes(6); + expect(registerToolSpy).toHaveBeenCalledTimes(8); }); it('should register search_babylon_docs tool', () => { diff --git a/src/mcp/handlers/editor/get-editor-source.handler.test.ts b/src/mcp/handlers/editor/get-editor-source.handler.test.ts new file mode 100644 index 0000000..6996ffb --- /dev/null +++ b/src/mcp/handlers/editor/get-editor-source.handler.test.ts @@ -0,0 +1,138 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import * as getEditorSourceHandler from './get-editor-source.handler.js'; +import * as searchInstance from '../shared/search-instance.js'; + +vi.mock('../shared/search-instance.js', () => ({ + getSearchInstance: vi.fn(), +})); + +describe('get-editor-source.handler', () => { + let mockServer: McpServer; + let mockSearch: any; + + beforeEach(() => { + mockServer = { + registerTool: vi.fn(), + } as any; + + mockSearch = { + getEditorSourceFile: vi.fn(), + }; + + vi.mocked(searchInstance.getSearchInstance).mockResolvedValue(mockSearch); + }); + + describe('register', () => { + it('should register get_babylon_editor_source tool with correct metadata', () => { + getEditorSourceHandler.register(mockServer); + + expect(mockServer.registerTool).toHaveBeenCalledWith( + 'get_babylon_editor_source', + expect.objectContaining({ + description: + 'Retrieve full Babylon.js Editor source code file or specific line range', + inputSchema: expect.any(Object), + }), + expect.any(Function) + ); + }); + + it('should define input schema with filePath, startLine, and endLine', () => { + getEditorSourceHandler.register(mockServer); + + const callArgs = vi.mocked(mockServer.registerTool).mock.calls[0]; + const schema = callArgs![1]; + + expect(schema.inputSchema).toHaveProperty('filePath'); + expect(schema.inputSchema).toHaveProperty('startLine'); + expect(schema.inputSchema).toHaveProperty('endLine'); + }); + }); + + describe('handler execution', () => { + let handler: Function; + + beforeEach(() => { + getEditorSourceHandler.register(mockServer); + const callArgs = vi.mocked(mockServer.registerTool).mock.calls[0]; + handler = callArgs![2]; + }); + + it('should retrieve full source file', async () => { + const sourceCode = 'export class Editor {\n constructor() {}\n}'; + mockSearch.getEditorSourceFile.mockResolvedValue(sourceCode); + + const result = await handler({ filePath: 'editor/src/editor.ts' }); + + expect(mockSearch.getEditorSourceFile).toHaveBeenCalledWith( + 'editor/src/editor.ts', + undefined, + undefined + ); + expect(result.content[0].text).toContain('export class Editor'); + expect(result.content[0].text).toContain('"totalLines": 3'); + }); + + it('should retrieve specific line range', async () => { + const sourceCode = 'line 10\nline 11\nline 12'; + mockSearch.getEditorSourceFile.mockResolvedValue(sourceCode); + + const result = await handler({ + filePath: 'editor/src/main.tsx', + startLine: 10, + endLine: 12, + }); + + expect(mockSearch.getEditorSourceFile).toHaveBeenCalledWith( + 'editor/src/main.tsx', + 10, + 12 + ); + expect(result.content[0].text).toContain('"startLine": 10'); + expect(result.content[0].text).toContain('"endLine": 12'); + }); + + it('should return not found for non-existent file', async () => { + mockSearch.getEditorSourceFile.mockResolvedValue(null); + + const result = await handler({ filePath: 'editor/src/nonexistent.ts' }); + + expect(result.content[0].text).toContain('Editor source file not found'); + expect(result.content[0].text).toContain('editor/src/nonexistent.ts'); + }); + + it('should detect typescript language from .ts extension', async () => { + mockSearch.getEditorSourceFile.mockResolvedValue('const x = 1;'); + + const result = await handler({ filePath: 'editor/src/file.ts' }); + + expect(result.content[0].text).toContain('"language": "typescript"'); + }); + + it('should detect typescript language from .tsx extension', async () => { + mockSearch.getEditorSourceFile.mockResolvedValue('const App = () =>
;'); + + const result = await handler({ filePath: 'editor/src/app.tsx' }); + + expect(result.content[0].text).toContain('"language": "typescript"'); + }); + + it('should detect javascript language from .js extension', async () => { + mockSearch.getEditorSourceFile.mockResolvedValue('var x = 1;'); + + const result = await handler({ filePath: 'editor/src/file.js' }); + + expect(result.content[0].text).toContain('"language": "javascript"'); + }); + + it('should handle errors gracefully', async () => { + mockSearch.getEditorSourceFile.mockRejectedValue(new Error('File read error')); + + const result = await handler({ filePath: 'editor/src/error.ts' }); + + expect(result.content[0].text).toContain('Error'); + expect(result.content[0].text).toContain('File read error'); + }); + }); +}); diff --git a/src/mcp/handlers/editor/get-editor-source.handler.ts b/src/mcp/handlers/editor/get-editor-source.handler.ts new file mode 100644 index 0000000..de85048 --- /dev/null +++ b/src/mcp/handlers/editor/get-editor-source.handler.ts @@ -0,0 +1,61 @@ +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { getSearchInstance } from '../shared/search-instance.js'; +import { + formatJsonResponse, + formatNotFoundResponse, +} from '../shared/response-formatters.js'; +import { withErrorHandling } from '../shared/error-handlers.js'; + +export function register(server: McpServer): void { + server.registerTool( + 'get_babylon_editor_source', + { + description: + 'Retrieve full Babylon.js Editor source code file or specific line range', + inputSchema: { + filePath: z + .string() + .describe( + 'Relative file path from Editor repository root (e.g., "editor/src/editor/layout/main.tsx")' + ), + startLine: z + .number() + .optional() + .describe('Optional start line number (1-indexed)'), + endLine: z + .number() + .optional() + .describe('Optional end line number (1-indexed)'), + }, + }, + withErrorHandling( + async ({ filePath, startLine, endLine }) => { + const search = await getSearchInstance(); + const sourceCode = await search.getEditorSourceFile(filePath, startLine, endLine); + + if (!sourceCode) { + return formatNotFoundResponse( + filePath, + 'Editor source file', + 'The path may be incorrect or the file does not exist in the Editor repository.' + ); + } + + return formatJsonResponse({ + filePath, + startLine: startLine || 1, + endLine: endLine || sourceCode.split('\n').length, + totalLines: sourceCode.split('\n').length, + /* c8 ignore next 3 */ + language: + filePath.endsWith('.ts') || filePath.endsWith('.tsx') + ? 'typescript' + : 'javascript', + content: sourceCode, + }); + }, + 'retrieving Editor source file' + ) + ); +} diff --git a/src/mcp/handlers/editor/search-editor-source.handler.test.ts b/src/mcp/handlers/editor/search-editor-source.handler.test.ts new file mode 100644 index 0000000..8307d14 --- /dev/null +++ b/src/mcp/handlers/editor/search-editor-source.handler.test.ts @@ -0,0 +1,191 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import * as searchEditorSourceHandler from './search-editor-source.handler.js'; +import * as searchInstance from '../shared/search-instance.js'; + +vi.mock('../shared/search-instance.js', () => ({ + getSearchInstance: vi.fn(), +})); + +describe('search-editor-source.handler', () => { + let mockServer: McpServer; + let mockSearch: any; + + beforeEach(() => { + mockServer = { + registerTool: vi.fn(), + } as any; + + mockSearch = { + searchSourceCode: vi.fn(), + }; + + vi.mocked(searchInstance.getSearchInstance).mockResolvedValue(mockSearch); + }); + + describe('register', () => { + it('should register search_babylon_editor_source tool with correct metadata', () => { + searchEditorSourceHandler.register(mockServer); + + expect(mockServer.registerTool).toHaveBeenCalledWith( + 'search_babylon_editor_source', + expect.objectContaining({ + description: 'Search Babylon.js Editor source code files', + inputSchema: expect.any(Object), + }), + expect.any(Function) + ); + }); + + it('should define input schema with query, package, and limit', () => { + searchEditorSourceHandler.register(mockServer); + + const callArgs = vi.mocked(mockServer.registerTool).mock.calls[0]; + const schema = callArgs![1]; + + expect(schema.inputSchema).toHaveProperty('query'); + expect(schema.inputSchema).toHaveProperty('package'); + expect(schema.inputSchema).toHaveProperty('limit'); + }); + }); + + describe('handler execution', () => { + let handler: Function; + + beforeEach(() => { + searchEditorSourceHandler.register(mockServer); + const callArgs = vi.mocked(mockServer.registerTool).mock.calls[0]; + handler = callArgs![2]; + }); + + it('should search with query using babylon_editor_source table', async () => { + mockSearch.searchSourceCode.mockResolvedValue([ + { + filePath: 'editor/src/editor/layout/main.tsx', + package: 'editor', + content: 'export function MainLayout() {', + startLine: 1, + endLine: 200, + language: 'typescript', + imports: 'react, @babylonjs/core', + exports: 'MainLayout', + url: 'https://github.com/BabylonJS/Editor/blob/master/editor/src/editor/layout/main.tsx#L1-L200', + score: 0.85, + }, + ]); + + const result = await handler({ query: 'main layout' }); + + expect(mockSearch.searchSourceCode).toHaveBeenCalledWith('main layout', { + package: undefined, + limit: 5, + tableName: 'babylon_editor_source', + }); + expect(result.content[0].text).toContain('MainLayout'); + expect(result.content[0].text).toContain('editor/src/editor/layout/main.tsx'); + }); + + it('should apply package filter', async () => { + mockSearch.searchSourceCode.mockResolvedValue([ + { + filePath: 'tools/src/tools/mesh-tool.ts', + package: 'tools', + content: 'export class MeshTool {', + startLine: 1, + endLine: 100, + language: 'typescript', + imports: '@babylonjs/core', + exports: 'MeshTool', + url: 'https://github.com/BabylonJS/Editor/blob/master/tools/src/tools/mesh-tool.ts#L1-L100', + score: 0.9, + }, + ]); + + await handler({ query: 'mesh', package: 'tools' }); + + expect(mockSearch.searchSourceCode).toHaveBeenCalledWith('mesh', { + package: 'tools', + limit: 5, + tableName: 'babylon_editor_source', + }); + }); + + it('should respect limit parameter', async () => { + mockSearch.searchSourceCode.mockResolvedValue([]); + + await handler({ query: 'test', limit: 10 }); + + expect(mockSearch.searchSourceCode).toHaveBeenCalledWith('test', { + package: undefined, + limit: 10, + tableName: 'babylon_editor_source', + }); + }); + + it('should return no results message when empty', async () => { + mockSearch.searchSourceCode.mockResolvedValue([]); + + const result = await handler({ query: 'nonexistent' }); + + expect(result.content[0].text).toContain('No Editor source code found'); + }); + + it('should format results with code snippet and metadata', async () => { + mockSearch.searchSourceCode.mockResolvedValue([ + { + filePath: 'editor/src/project/project.ts', + package: 'editor', + content: 'export class Project { constructor() { } }', + startLine: 10, + endLine: 50, + language: 'typescript', + imports: 'fs, path', + exports: 'Project', + url: 'https://github.com/BabylonJS/Editor/blob/master/editor/src/project/project.ts#L10-L50', + score: 0.92, + }, + ]); + + const result = await handler({ query: 'project' }); + const resultText = result.content[0].text; + + expect(resultText).toContain('"rank": 1'); + expect(resultText).toContain('"filePath": "editor/src/project/project.ts"'); + expect(resultText).toContain('"package": "editor"'); + expect(resultText).toContain('"relevance": "92.0%"'); + expect(resultText).toContain('"startLine": 10'); + expect(resultText).toContain('"endLine": 50'); + }); + + it('should truncate long code snippets', async () => { + mockSearch.searchSourceCode.mockResolvedValue([ + { + filePath: 'editor/src/large-file.ts', + package: 'editor', + content: 'x'.repeat(600), + startLine: 1, + endLine: 100, + language: 'typescript', + imports: '', + exports: '', + url: 'https://github.com/BabylonJS/Editor/blob/master/editor/src/large-file.ts#L1-L100', + score: 0.8, + }, + ]); + + const result = await handler({ query: 'large' }); + const resultText = result.content[0].text; + + expect(resultText).toContain('...'); + }); + + it('should handle search errors gracefully', async () => { + mockSearch.searchSourceCode.mockRejectedValue(new Error('Search failed')); + + const result = await handler({ query: 'test' }); + + expect(result.content[0].text).toContain('Error'); + expect(result.content[0].text).toContain('Search failed'); + }); + }); +}); diff --git a/src/mcp/handlers/editor/search-editor-source.handler.ts b/src/mcp/handlers/editor/search-editor-source.handler.ts new file mode 100644 index 0000000..cf911a8 --- /dev/null +++ b/src/mcp/handlers/editor/search-editor-source.handler.ts @@ -0,0 +1,72 @@ +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { getSearchInstance } from '../shared/search-instance.js'; +import { + formatJsonResponse, + formatNoResultsResponse, +} from '../shared/response-formatters.js'; +import { withErrorHandling } from '../shared/error-handlers.js'; + +export function register(server: McpServer): void { + server.registerTool( + 'search_babylon_editor_source', + { + description: 'Search Babylon.js Editor source code files', + inputSchema: { + query: z + .string() + .describe( + 'Search query for Editor source code (e.g., "inspector panel", "asset browser")' + ), + package: z + .string() + .optional() + .describe('Optional package filter (e.g., "editor", "tools", "website")'), + limit: z + .number() + .optional() + .default(5) + .describe('Maximum number of results to return (default: 5)'), + }, + }, + withErrorHandling( + async ({ query, package: packageFilter, limit = 5 }) => { + const search = await getSearchInstance(); + const options = { + package: packageFilter, + limit, + tableName: 'babylon_editor_source', + }; + const results = await search.searchSourceCode(query, options); + + if (results.length === 0) { + return formatNoResultsResponse(query, 'Editor source code'); + } + + const formattedResults = results.map((result, index) => ({ + rank: index + 1, + filePath: result.filePath, + package: result.package, + startLine: result.startLine, + endLine: result.endLine, + language: result.language, + codeSnippet: + result.content.substring(0, 500) + + /* c8 ignore next */ + (result.content.length > 500 ? '...' : ''), + imports: result.imports, + exports: result.exports, + url: result.url, + relevance: (result.score * 100).toFixed(1) + '%', + })); + + return formatJsonResponse({ + query, + totalResults: results.length, + results: formattedResults, + }); + }, + 'searching Editor source code' + ) + ); +} diff --git a/src/mcp/handlers/index.ts b/src/mcp/handlers/index.ts index 7903a07..d7fda0b 100644 --- a/src/mcp/handlers/index.ts +++ b/src/mcp/handlers/index.ts @@ -5,17 +5,21 @@ import * as searchApiHandler from './api/search-api.handler.js'; import * as searchSourceHandler from './source/search-source.handler.js'; import * as getSourceHandler from './source/get-source.handler.js'; import * as searchEditorDocsHandler from './editor/search-editor-docs.handler.js'; +import * as searchEditorSourceHandler from './editor/search-editor-source.handler.js'; +import * as getEditorSourceHandler from './editor/get-editor-source.handler.js'; /** * Register all MCP tool handlers with the server. * - * This function sets up all 6 Babylon.js MCP tools: + * This function sets up all 8 Babylon.js MCP tools: * - search_babylon_docs: Search documentation * - get_babylon_doc: Get specific documentation * - search_babylon_api: Search API documentation * - search_babylon_source: Search source code * - get_babylon_source: Get source code files * - search_babylon_editor_docs: Search Editor documentation + * - search_babylon_editor_source: Search Editor source code + * - get_babylon_editor_source: Get Editor source code files */ export function setupHandlers(server: McpServer): void { searchDocsHandler.register(server); @@ -24,4 +28,6 @@ export function setupHandlers(server: McpServer): void { searchSourceHandler.register(server); getSourceHandler.register(server); searchEditorDocsHandler.register(server); + searchEditorSourceHandler.register(server); + getEditorSourceHandler.register(server); } diff --git a/src/mcp/repository-config.ts b/src/mcp/repository-config.ts index 9795e20..04d3ebe 100644 --- a/src/mcp/repository-config.ts +++ b/src/mcp/repository-config.ts @@ -23,6 +23,7 @@ export const BABYLON_REPOSITORIES: RepositoryConfig[] = [ }, { name: 'Editor', + url: 'https://github.com/BabylonJS/Editor.git', shallow: true, branch: 'master', diff --git a/src/mcp/server.ts b/src/mcp/server.ts index a72e80a..79d4161 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -31,7 +31,7 @@ export class BabylonMCPServer { }, instructions: MCP_SERVER_CONFIG.instructions, } - ); + ) setupHandlers(this.server); this.setupErrorHandling(); diff --git a/src/search/lancedb-search.test.ts b/src/search/lancedb-search.test.ts index 536c338..b200937 100644 --- a/src/search/lancedb-search.test.ts +++ b/src/search/lancedb-search.test.ts @@ -179,6 +179,41 @@ describe('LanceDBSearch', () => { expect(doc).toBeDefined(); }); + it('should construct Editor URL for editor/ prefixed paths', async () => { + const mockTable = await (search as any).table; + // First call (exact path) returns empty, second call (Editor URL) returns doc + const whereFn = vi.fn() + .mockImplementationOnce(() => ({ + limit: vi.fn(() => ({ + toArray: vi.fn(() => Promise.resolve([])), + })), + })) + .mockImplementationOnce(() => ({ + limit: vi.fn(() => ({ + toArray: vi.fn(() => Promise.resolve([ + { + id: 'editor-docs_adding-scripts', + title: 'Adding Scripts', + content: 'Editor script content', + url: 'https://editor.babylonjs.com/documentation/adding-scripts', + filePath: '/test/editor/adding-scripts/page.tsx', + }, + ])), + })), + })); + mockTable.query = vi.fn(() => ({ where: whereFn })); + + const doc = await search.getDocumentByPath('editor/adding-scripts'); + expect(doc).toBeDefined(); + expect(doc!.title).toBe('Adding Scripts'); + // Verify both queries were made + expect(whereFn).toHaveBeenCalledTimes(2); + // Second call should use the constructed Editor URL + expect(whereFn).toHaveBeenNthCalledWith(2, + "url = 'https://editor.babylonjs.com/documentation/adding-scripts'" + ); + }); + it('should throw error if not initialized', async () => { const uninitSearch = new LanceDBSearch(); await expect(uninitSearch.getDocumentByPath('test')).rejects.toThrow( @@ -187,6 +222,32 @@ describe('LanceDBSearch', () => { }); }); + describe('pathToDocId', () => { + it('should detect editor-docs source for editor/ prefixed paths', () => { + const pathToDocId = (search as any).pathToDocId.bind(search); + const result = pathToDocId('editor/adding-scripts'); + expect(result).toBe('editor-docs_adding-scripts'); + }); + + it('should detect editor-docs source for paths containing /Editor/', () => { + const pathToDocId = (search as any).pathToDocId.bind(search); + const result = pathToDocId('/some/path/Editor/docs/page'); + expect(result).toBe('editor-docs__some_path_Editor_docs_page'); + }); + + it('should default to documentation source for other paths', () => { + const pathToDocId = (search as any).pathToDocId.bind(search); + const result = pathToDocId('features/materials'); + expect(result).toBe('documentation_features_materials'); + }); + + it('should strip page.tsx suffix for Editor docs', () => { + const pathToDocId = (search as any).pathToDocId.bind(search); + const result = pathToDocId('editor/adding-scripts/page.tsx'); + expect(result).toBe('editor-docs_adding-scripts'); + }); + }); + describe('close', () => { it('should close without error', async () => { await expect(search.close()).resolves.not.toThrow(); diff --git a/src/search/lancedb-search.ts b/src/search/lancedb-search.ts index 85b1536..2e5044d 100644 --- a/src/search/lancedb-search.ts +++ b/src/search/lancedb-search.ts @@ -103,13 +103,24 @@ export class LanceDBSearch { throw new Error('Search not initialized. Call initialize() first.'); } - // Try to find document by URL first + // Try to find document by exact URL first let results = await this.table .query() .where(`url = '${filePath}'`) .limit(1) .toArray(); + // If path looks like "editor/adding-scripts", try constructing full Editor URL + if (results.length === 0 && filePath.startsWith('editor/')) { + const editorPath = filePath.replace(/^editor\//, ''); + const editorUrl = `https://editor.babylonjs.com/documentation/${editorPath}`; + results = await this.table + .query() + .where(`url = '${editorUrl}'`) + .limit(1) + .toArray(); + } + if (results.length > 0) { const doc = results[0]; // Fetch fresh content from local file if available @@ -133,6 +144,7 @@ export class LanceDBSearch { path.join('./data/repositories/Documentation', filePath.replace(/^.*\/content\//, '')), path.join('./data/repositories/Babylon.js', filePath.replace(/^.*\/Babylon\.js\//, '')), path.join('./data/repositories/havok', filePath.replace(/^.*\/havok\//, '')), + path.join('./data/repositories/Editor/website/src/app/documentation', filePath.replace(/^.*\/documentation\//, '')), ]; for (const possiblePath of possiblePaths) { @@ -145,7 +157,7 @@ export class LanceDBSearch { } return null; - } catch (error) { + } catch { return null; } } @@ -198,17 +210,21 @@ export class LanceDBSearch { } private pathToDocId(filePath: string): string { - // Remove .md extension if present let normalizedPath = filePath.replace(/\.md$/, ''); + normalizedPath = normalizedPath.replace(/\/page\.tsx$/i, ''); + + // Detect source type from path + let sourcePrefix = 'documentation'; + if (filePath.startsWith('editor/') || filePath.includes('/Editor/')) { + sourcePrefix = 'editor-docs'; + normalizedPath = normalizedPath.replace(/^editor\//, ''); + } // Strip any leading path up to and including /content/ - // This handles both full paths and relative paths normalizedPath = normalizedPath.replace(/^.*\/content\//, ''); - // Convert slashes to underscores and prepend source name - // Note: source name is "documentation" (lowercase) as defined in index-docs.ts const pathWithUnderscores = normalizedPath.replace(/\//g, '_'); - return `documentation_${pathWithUnderscores}`; + return `${sourcePrefix}_${pathWithUnderscores}`; } async searchSourceCode( @@ -257,6 +273,26 @@ export class LanceDBSearch { } } + async getEditorSourceFile( + filePath: string, + startLine?: number, + endLine?: number + ): Promise { + try { + const fullPath = path.join('./data/repositories/Editor', filePath); + const content = await fs.readFile(fullPath, 'utf-8'); + + if (startLine !== undefined && endLine !== undefined) { + const lines = content.split('\n'); + return lines.slice(startLine - 1, endLine).join('\n'); + } + return content; + } catch (error) { + console.error(`Error reading Editor source file ${filePath}:`, error); + return null; + } + } + async close(): Promise { // LanceDB doesn't require explicit closing }