Add Editor source code tools and expand indexed Babylon.js packages

Adds search_babylon_editor_source and get_babylon_editor_source MCP tools
for searching and retrieving Editor source code. Expands source indexing
to include inspector, viewer, addons, accessibility, node-editor, and
procedural-textures packages. Improves pathToDocId to handle Editor paths
and adds Editor URL construction fallback in getDocumentByPath.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Michael Mainguy 2026-03-03 09:02:20 -06:00
parent 5483eedcc2
commit e6a3329c9b
13 changed files with 816 additions and 11 deletions

View File

@ -18,7 +18,8 @@
"index:docs": "tsx scripts/index-docs.ts", "index:docs": "tsx scripts/index-docs.ts",
"index:api": "tsx scripts/index-api.ts", "index:api": "tsx scripts/index-api.ts",
"index:source": "tsx scripts/index-source.ts", "index:source": "tsx scripts/index-source.ts",
"index:all": "npm run index:docs && npm run index:api && npm run index:source", "index:editor-source": "tsx scripts/index-editor-source.ts",
"index:all": "npm run index:docs && npm run index:api && npm run index:source && npm run index:editor-source",
"index-docs": "npm run index:docs", "index-docs": "npm run index:docs",
"index-api": "npm run index:api", "index-api": "npm run index:api",
"index-source": "npm run index:source" "index-source": "npm run index:source"

View File

@ -0,0 +1,229 @@
// MUST set environment variable before any imports that use @xenova/transformers
// This prevents onnxruntime-node from being loaded on Alpine Linux (musl libc)
if (process.env.TRANSFORMERS_BACKEND === 'wasm' || process.env.TRANSFORMERS_BACKEND === 'onnxruntime-web') {
process.env.ONNXRUNTIME_BACKEND = 'wasm';
}
import { connect } from '@lancedb/lancedb';
import { pipeline } from '@xenova/transformers';
import fs from 'fs/promises';
import path from 'path';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
interface SourceCodeChunk {
id: string;
filePath: string;
package: string;
content: string;
startLine: number;
endLine: number;
language: string;
imports: string;
exports: string;
url: string;
vector: number[];
}
const CHUNK_SIZE = 200;
const CHUNK_OVERLAP = 20;
async function getAllSourceFiles(dir: string): Promise<string[]> {
const files: string[] = [];
try {
const entries = await fs.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
if (!['node_modules', 'dist', 'build', 'lib', '.git', 'declaration'].includes(entry.name)) {
const subFiles = await getAllSourceFiles(fullPath);
files.push(...subFiles);
}
} else if (entry.isFile()) {
if (/\.(ts|tsx|js|jsx)$/.test(entry.name)) {
files.push(fullPath);
}
}
}
} catch {
return [];
}
return files;
}
function extractImports(content: string): string {
const imports: string[] = [];
const importRegex = /import\s+(?:{[^}]+}|[^;]+)\s+from\s+['"]([^'"]+)['"]/g;
let match;
while ((match = importRegex.exec(content)) !== null) {
if (match[1]) {
imports.push(match[1]);
}
}
return imports.slice(0, 20).join(', ');
}
function extractExports(content: string): string {
const exports: string[] = [];
const exportRegex = /export\s+(?:class|function|interface|type|const|let|var|enum|default)\s+([A-Za-z_$][A-Za-z0-9_$]*)/g;
let match;
while ((match = exportRegex.exec(content)) !== null) {
if (match[1]) {
exports.push(match[1]);
}
}
return exports.slice(0, 20).join(', ');
}
function extractComments(code: string): string {
const comments: string[] = [];
const singleLineRegex = /\/\/\s*(.+)$/gm;
let match;
while ((match = singleLineRegex.exec(code)) !== null) {
if (match[1]) {
comments.push(match[1].trim());
}
}
const multiLineRegex = /\/\*\*?([\s\S]*?)\*\//g;
while ((match = multiLineRegex.exec(code)) !== null) {
if (match[1]) {
comments.push(match[1].trim());
}
}
return comments.slice(0, 5).join(' ');
}
async function main() {
const projectRoot = path.join(__dirname, '..');
const dbPath = path.join(projectRoot, 'data', 'lancedb');
const repositoryPath = path.join(projectRoot, 'data', 'repositories', 'Editor');
const tableName = 'babylon_editor_source';
// Editor packages with their source paths (relative to repo root)
const packages = [
{ name: 'editor', srcPath: 'editor/src' },
{ name: 'tools', srcPath: 'tools/src' },
{ name: 'website', srcPath: 'website/src' },
];
console.log('Starting Editor source code indexing...');
console.log(`Database path: ${dbPath}`);
console.log(`Repository path: ${repositoryPath}`);
console.log(`Packages: ${packages.map(p => p.name).join(', ')}`);
console.log();
// Initialize LanceDB
console.log('Initializing LanceDB connection...');
const db = await connect(dbPath);
// Load embedding model
console.log('Loading embedding model...');
const embedder = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
console.log('Embedding model loaded');
const chunks: SourceCodeChunk[] = [];
let totalFiles = 0;
for (const pkg of packages) {
console.log(`\nIndexing package: ${pkg.name}...`);
const packagePath = path.join(repositoryPath, pkg.srcPath);
try {
const files = await getAllSourceFiles(packagePath);
console.log(`Found ${files.length} source files in ${pkg.name}`);
for (let i = 0; i < files.length; i++) {
const file = files[i]!;
try {
const content = await fs.readFile(file, 'utf-8');
const lines = content.split('\n');
const imports = extractImports(content);
const exports = extractExports(content);
const language = file.endsWith('.ts') || file.endsWith('.tsx') ? 'typescript' : 'javascript';
const relativePath = path.relative(repositoryPath, file);
// Chunk the file
for (let j = 0; j < lines.length; j += CHUNK_SIZE - CHUNK_OVERLAP) {
const startLine = j + 1;
const endLine = Math.min(j + CHUNK_SIZE, lines.length);
const chunkLines = lines.slice(j, endLine);
const chunkContent = chunkLines.join('\n');
if (chunkContent.trim().length === 0) {
continue;
}
// Create embedding text
const fileName = path.basename(file);
const dirName = path.dirname(relativePath).split('/').pop() || '';
const comments = extractComments(chunkContent);
const embeddingText = `${fileName} ${dirName} ${comments} ${chunkContent.substring(0, 1000)}`;
// Generate embedding
const result = await embedder(embeddingText, {
pooling: 'mean',
normalize: true,
});
const vector = Array.from(result.data) as number[];
// Generate GitHub URL for Editor repo
const url = `https://github.com/BabylonJS/Editor/blob/master/${relativePath}#L${startLine}-L${endLine}`;
chunks.push({
id: `${relativePath}:${startLine}-${endLine}`,
filePath: relativePath,
package: pkg.name,
content: chunkContent,
startLine,
endLine,
language,
imports,
exports,
url,
vector,
});
}
totalFiles++;
if (totalFiles % 50 === 0) {
console.log(`Processed ${totalFiles} files, ${chunks.length} chunks...`);
}
} catch (error) {
console.error(`Error processing ${file}:`, error);
}
}
} catch (error) {
console.error(`Error indexing package ${pkg.name}:`, error);
}
}
console.log(`\nTotal files processed: ${totalFiles}`);
console.log(`Total source code chunks: ${chunks.length}`);
console.log('Creating LanceDB table...');
// Drop existing table if it exists
const tableNames = await db.tableNames();
if (tableNames.includes(tableName)) {
await db.dropTable(tableName);
}
// Create new table
await db.createTable(tableName, chunks);
console.log('\n✓ Editor source code indexing completed successfully!');
}
main().catch(console.error);

View File

@ -14,6 +14,15 @@ async function main() {
'materials', 'materials',
'loaders', 'loaders',
'serializers', 'serializers',
'inspector',
'inspector-v2',
'ui-controls',
'viewer',
'addons',
'accessibility',
'node-editor',
'node-particle-editor',
'procedural-textures'
]; ];
console.log('Starting source code indexing for Babylon.js packages...'); console.log('Starting source code indexing for Babylon.js packages...');

View File

@ -17,7 +17,7 @@ describe('MCP Handlers', () => {
it('should register all required tools', () => { it('should register all required tools', () => {
setupHandlers(mockServer); setupHandlers(mockServer);
expect(registerToolSpy).toHaveBeenCalledTimes(6); expect(registerToolSpy).toHaveBeenCalledTimes(8);
}); });
it('should register search_babylon_docs tool', () => { it('should register search_babylon_docs tool', () => {

View File

@ -0,0 +1,138 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import * as getEditorSourceHandler from './get-editor-source.handler.js';
import * as searchInstance from '../shared/search-instance.js';
vi.mock('../shared/search-instance.js', () => ({
getSearchInstance: vi.fn(),
}));
describe('get-editor-source.handler', () => {
let mockServer: McpServer;
let mockSearch: any;
beforeEach(() => {
mockServer = {
registerTool: vi.fn(),
} as any;
mockSearch = {
getEditorSourceFile: vi.fn(),
};
vi.mocked(searchInstance.getSearchInstance).mockResolvedValue(mockSearch);
});
describe('register', () => {
it('should register get_babylon_editor_source tool with correct metadata', () => {
getEditorSourceHandler.register(mockServer);
expect(mockServer.registerTool).toHaveBeenCalledWith(
'get_babylon_editor_source',
expect.objectContaining({
description:
'Retrieve full Babylon.js Editor source code file or specific line range',
inputSchema: expect.any(Object),
}),
expect.any(Function)
);
});
it('should define input schema with filePath, startLine, and endLine', () => {
getEditorSourceHandler.register(mockServer);
const callArgs = vi.mocked(mockServer.registerTool).mock.calls[0];
const schema = callArgs![1];
expect(schema.inputSchema).toHaveProperty('filePath');
expect(schema.inputSchema).toHaveProperty('startLine');
expect(schema.inputSchema).toHaveProperty('endLine');
});
});
describe('handler execution', () => {
let handler: Function;
beforeEach(() => {
getEditorSourceHandler.register(mockServer);
const callArgs = vi.mocked(mockServer.registerTool).mock.calls[0];
handler = callArgs![2];
});
it('should retrieve full source file', async () => {
const sourceCode = 'export class Editor {\n constructor() {}\n}';
mockSearch.getEditorSourceFile.mockResolvedValue(sourceCode);
const result = await handler({ filePath: 'editor/src/editor.ts' });
expect(mockSearch.getEditorSourceFile).toHaveBeenCalledWith(
'editor/src/editor.ts',
undefined,
undefined
);
expect(result.content[0].text).toContain('export class Editor');
expect(result.content[0].text).toContain('"totalLines": 3');
});
it('should retrieve specific line range', async () => {
const sourceCode = 'line 10\nline 11\nline 12';
mockSearch.getEditorSourceFile.mockResolvedValue(sourceCode);
const result = await handler({
filePath: 'editor/src/main.tsx',
startLine: 10,
endLine: 12,
});
expect(mockSearch.getEditorSourceFile).toHaveBeenCalledWith(
'editor/src/main.tsx',
10,
12
);
expect(result.content[0].text).toContain('"startLine": 10');
expect(result.content[0].text).toContain('"endLine": 12');
});
it('should return not found for non-existent file', async () => {
mockSearch.getEditorSourceFile.mockResolvedValue(null);
const result = await handler({ filePath: 'editor/src/nonexistent.ts' });
expect(result.content[0].text).toContain('Editor source file not found');
expect(result.content[0].text).toContain('editor/src/nonexistent.ts');
});
it('should detect typescript language from .ts extension', async () => {
mockSearch.getEditorSourceFile.mockResolvedValue('const x = 1;');
const result = await handler({ filePath: 'editor/src/file.ts' });
expect(result.content[0].text).toContain('"language": "typescript"');
});
it('should detect typescript language from .tsx extension', async () => {
mockSearch.getEditorSourceFile.mockResolvedValue('const App = () => <div/>;');
const result = await handler({ filePath: 'editor/src/app.tsx' });
expect(result.content[0].text).toContain('"language": "typescript"');
});
it('should detect javascript language from .js extension', async () => {
mockSearch.getEditorSourceFile.mockResolvedValue('var x = 1;');
const result = await handler({ filePath: 'editor/src/file.js' });
expect(result.content[0].text).toContain('"language": "javascript"');
});
it('should handle errors gracefully', async () => {
mockSearch.getEditorSourceFile.mockRejectedValue(new Error('File read error'));
const result = await handler({ filePath: 'editor/src/error.ts' });
expect(result.content[0].text).toContain('Error');
expect(result.content[0].text).toContain('File read error');
});
});
});

View File

@ -0,0 +1,61 @@
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { z } from 'zod';
import { getSearchInstance } from '../shared/search-instance.js';
import {
formatJsonResponse,
formatNotFoundResponse,
} from '../shared/response-formatters.js';
import { withErrorHandling } from '../shared/error-handlers.js';
export function register(server: McpServer): void {
server.registerTool(
'get_babylon_editor_source',
{
description:
'Retrieve full Babylon.js Editor source code file or specific line range',
inputSchema: {
filePath: z
.string()
.describe(
'Relative file path from Editor repository root (e.g., "editor/src/editor/layout/main.tsx")'
),
startLine: z
.number()
.optional()
.describe('Optional start line number (1-indexed)'),
endLine: z
.number()
.optional()
.describe('Optional end line number (1-indexed)'),
},
},
withErrorHandling(
async ({ filePath, startLine, endLine }) => {
const search = await getSearchInstance();
const sourceCode = await search.getEditorSourceFile(filePath, startLine, endLine);
if (!sourceCode) {
return formatNotFoundResponse(
filePath,
'Editor source file',
'The path may be incorrect or the file does not exist in the Editor repository.'
);
}
return formatJsonResponse({
filePath,
startLine: startLine || 1,
endLine: endLine || sourceCode.split('\n').length,
totalLines: sourceCode.split('\n').length,
/* c8 ignore next 3 */
language:
filePath.endsWith('.ts') || filePath.endsWith('.tsx')
? 'typescript'
: 'javascript',
content: sourceCode,
});
},
'retrieving Editor source file'
)
);
}

View File

@ -0,0 +1,191 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import * as searchEditorSourceHandler from './search-editor-source.handler.js';
import * as searchInstance from '../shared/search-instance.js';
vi.mock('../shared/search-instance.js', () => ({
getSearchInstance: vi.fn(),
}));
describe('search-editor-source.handler', () => {
let mockServer: McpServer;
let mockSearch: any;
beforeEach(() => {
mockServer = {
registerTool: vi.fn(),
} as any;
mockSearch = {
searchSourceCode: vi.fn(),
};
vi.mocked(searchInstance.getSearchInstance).mockResolvedValue(mockSearch);
});
describe('register', () => {
it('should register search_babylon_editor_source tool with correct metadata', () => {
searchEditorSourceHandler.register(mockServer);
expect(mockServer.registerTool).toHaveBeenCalledWith(
'search_babylon_editor_source',
expect.objectContaining({
description: 'Search Babylon.js Editor source code files',
inputSchema: expect.any(Object),
}),
expect.any(Function)
);
});
it('should define input schema with query, package, and limit', () => {
searchEditorSourceHandler.register(mockServer);
const callArgs = vi.mocked(mockServer.registerTool).mock.calls[0];
const schema = callArgs![1];
expect(schema.inputSchema).toHaveProperty('query');
expect(schema.inputSchema).toHaveProperty('package');
expect(schema.inputSchema).toHaveProperty('limit');
});
});
describe('handler execution', () => {
let handler: Function;
beforeEach(() => {
searchEditorSourceHandler.register(mockServer);
const callArgs = vi.mocked(mockServer.registerTool).mock.calls[0];
handler = callArgs![2];
});
it('should search with query using babylon_editor_source table', async () => {
mockSearch.searchSourceCode.mockResolvedValue([
{
filePath: 'editor/src/editor/layout/main.tsx',
package: 'editor',
content: 'export function MainLayout() {',
startLine: 1,
endLine: 200,
language: 'typescript',
imports: 'react, @babylonjs/core',
exports: 'MainLayout',
url: 'https://github.com/BabylonJS/Editor/blob/master/editor/src/editor/layout/main.tsx#L1-L200',
score: 0.85,
},
]);
const result = await handler({ query: 'main layout' });
expect(mockSearch.searchSourceCode).toHaveBeenCalledWith('main layout', {
package: undefined,
limit: 5,
tableName: 'babylon_editor_source',
});
expect(result.content[0].text).toContain('MainLayout');
expect(result.content[0].text).toContain('editor/src/editor/layout/main.tsx');
});
it('should apply package filter', async () => {
mockSearch.searchSourceCode.mockResolvedValue([
{
filePath: 'tools/src/tools/mesh-tool.ts',
package: 'tools',
content: 'export class MeshTool {',
startLine: 1,
endLine: 100,
language: 'typescript',
imports: '@babylonjs/core',
exports: 'MeshTool',
url: 'https://github.com/BabylonJS/Editor/blob/master/tools/src/tools/mesh-tool.ts#L1-L100',
score: 0.9,
},
]);
await handler({ query: 'mesh', package: 'tools' });
expect(mockSearch.searchSourceCode).toHaveBeenCalledWith('mesh', {
package: 'tools',
limit: 5,
tableName: 'babylon_editor_source',
});
});
it('should respect limit parameter', async () => {
mockSearch.searchSourceCode.mockResolvedValue([]);
await handler({ query: 'test', limit: 10 });
expect(mockSearch.searchSourceCode).toHaveBeenCalledWith('test', {
package: undefined,
limit: 10,
tableName: 'babylon_editor_source',
});
});
it('should return no results message when empty', async () => {
mockSearch.searchSourceCode.mockResolvedValue([]);
const result = await handler({ query: 'nonexistent' });
expect(result.content[0].text).toContain('No Editor source code found');
});
it('should format results with code snippet and metadata', async () => {
mockSearch.searchSourceCode.mockResolvedValue([
{
filePath: 'editor/src/project/project.ts',
package: 'editor',
content: 'export class Project { constructor() { } }',
startLine: 10,
endLine: 50,
language: 'typescript',
imports: 'fs, path',
exports: 'Project',
url: 'https://github.com/BabylonJS/Editor/blob/master/editor/src/project/project.ts#L10-L50',
score: 0.92,
},
]);
const result = await handler({ query: 'project' });
const resultText = result.content[0].text;
expect(resultText).toContain('"rank": 1');
expect(resultText).toContain('"filePath": "editor/src/project/project.ts"');
expect(resultText).toContain('"package": "editor"');
expect(resultText).toContain('"relevance": "92.0%"');
expect(resultText).toContain('"startLine": 10');
expect(resultText).toContain('"endLine": 50');
});
it('should truncate long code snippets', async () => {
mockSearch.searchSourceCode.mockResolvedValue([
{
filePath: 'editor/src/large-file.ts',
package: 'editor',
content: 'x'.repeat(600),
startLine: 1,
endLine: 100,
language: 'typescript',
imports: '',
exports: '',
url: 'https://github.com/BabylonJS/Editor/blob/master/editor/src/large-file.ts#L1-L100',
score: 0.8,
},
]);
const result = await handler({ query: 'large' });
const resultText = result.content[0].text;
expect(resultText).toContain('...');
});
it('should handle search errors gracefully', async () => {
mockSearch.searchSourceCode.mockRejectedValue(new Error('Search failed'));
const result = await handler({ query: 'test' });
expect(result.content[0].text).toContain('Error');
expect(result.content[0].text).toContain('Search failed');
});
});
});

View File

@ -0,0 +1,72 @@
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { z } from 'zod';
import { getSearchInstance } from '../shared/search-instance.js';
import {
formatJsonResponse,
formatNoResultsResponse,
} from '../shared/response-formatters.js';
import { withErrorHandling } from '../shared/error-handlers.js';
export function register(server: McpServer): void {
server.registerTool(
'search_babylon_editor_source',
{
description: 'Search Babylon.js Editor source code files',
inputSchema: {
query: z
.string()
.describe(
'Search query for Editor source code (e.g., "inspector panel", "asset browser")'
),
package: z
.string()
.optional()
.describe('Optional package filter (e.g., "editor", "tools", "website")'),
limit: z
.number()
.optional()
.default(5)
.describe('Maximum number of results to return (default: 5)'),
},
},
withErrorHandling(
async ({ query, package: packageFilter, limit = 5 }) => {
const search = await getSearchInstance();
const options = {
package: packageFilter,
limit,
tableName: 'babylon_editor_source',
};
const results = await search.searchSourceCode(query, options);
if (results.length === 0) {
return formatNoResultsResponse(query, 'Editor source code');
}
const formattedResults = results.map((result, index) => ({
rank: index + 1,
filePath: result.filePath,
package: result.package,
startLine: result.startLine,
endLine: result.endLine,
language: result.language,
codeSnippet:
result.content.substring(0, 500) +
/* c8 ignore next */
(result.content.length > 500 ? '...' : ''),
imports: result.imports,
exports: result.exports,
url: result.url,
relevance: (result.score * 100).toFixed(1) + '%',
}));
return formatJsonResponse({
query,
totalResults: results.length,
results: formattedResults,
});
},
'searching Editor source code'
)
);
}

View File

@ -5,17 +5,21 @@ import * as searchApiHandler from './api/search-api.handler.js';
import * as searchSourceHandler from './source/search-source.handler.js'; import * as searchSourceHandler from './source/search-source.handler.js';
import * as getSourceHandler from './source/get-source.handler.js'; import * as getSourceHandler from './source/get-source.handler.js';
import * as searchEditorDocsHandler from './editor/search-editor-docs.handler.js'; import * as searchEditorDocsHandler from './editor/search-editor-docs.handler.js';
import * as searchEditorSourceHandler from './editor/search-editor-source.handler.js';
import * as getEditorSourceHandler from './editor/get-editor-source.handler.js';
/** /**
* Register all MCP tool handlers with the server. * Register all MCP tool handlers with the server.
* *
* This function sets up all 6 Babylon.js MCP tools: * This function sets up all 8 Babylon.js MCP tools:
* - search_babylon_docs: Search documentation * - search_babylon_docs: Search documentation
* - get_babylon_doc: Get specific documentation * - get_babylon_doc: Get specific documentation
* - search_babylon_api: Search API documentation * - search_babylon_api: Search API documentation
* - search_babylon_source: Search source code * - search_babylon_source: Search source code
* - get_babylon_source: Get source code files * - get_babylon_source: Get source code files
* - search_babylon_editor_docs: Search Editor documentation * - search_babylon_editor_docs: Search Editor documentation
* - search_babylon_editor_source: Search Editor source code
* - get_babylon_editor_source: Get Editor source code files
*/ */
export function setupHandlers(server: McpServer): void { export function setupHandlers(server: McpServer): void {
searchDocsHandler.register(server); searchDocsHandler.register(server);
@ -24,4 +28,6 @@ export function setupHandlers(server: McpServer): void {
searchSourceHandler.register(server); searchSourceHandler.register(server);
getSourceHandler.register(server); getSourceHandler.register(server);
searchEditorDocsHandler.register(server); searchEditorDocsHandler.register(server);
searchEditorSourceHandler.register(server);
getEditorSourceHandler.register(server);
} }

View File

@ -23,6 +23,7 @@ export const BABYLON_REPOSITORIES: RepositoryConfig[] = [
}, },
{ {
name: 'Editor', name: 'Editor',
url: 'https://github.com/BabylonJS/Editor.git', url: 'https://github.com/BabylonJS/Editor.git',
shallow: true, shallow: true,
branch: 'master', branch: 'master',

View File

@ -31,7 +31,7 @@ export class BabylonMCPServer {
}, },
instructions: MCP_SERVER_CONFIG.instructions, instructions: MCP_SERVER_CONFIG.instructions,
} }
); )
setupHandlers(this.server); setupHandlers(this.server);
this.setupErrorHandling(); this.setupErrorHandling();

View File

@ -179,6 +179,41 @@ describe('LanceDBSearch', () => {
expect(doc).toBeDefined(); expect(doc).toBeDefined();
}); });
it('should construct Editor URL for editor/ prefixed paths', async () => {
const mockTable = await (search as any).table;
// First call (exact path) returns empty, second call (Editor URL) returns doc
const whereFn = vi.fn()
.mockImplementationOnce(() => ({
limit: vi.fn(() => ({
toArray: vi.fn(() => Promise.resolve([])),
})),
}))
.mockImplementationOnce(() => ({
limit: vi.fn(() => ({
toArray: vi.fn(() => Promise.resolve([
{
id: 'editor-docs_adding-scripts',
title: 'Adding Scripts',
content: 'Editor script content',
url: 'https://editor.babylonjs.com/documentation/adding-scripts',
filePath: '/test/editor/adding-scripts/page.tsx',
},
])),
})),
}));
mockTable.query = vi.fn(() => ({ where: whereFn }));
const doc = await search.getDocumentByPath('editor/adding-scripts');
expect(doc).toBeDefined();
expect(doc!.title).toBe('Adding Scripts');
// Verify both queries were made
expect(whereFn).toHaveBeenCalledTimes(2);
// Second call should use the constructed Editor URL
expect(whereFn).toHaveBeenNthCalledWith(2,
"url = 'https://editor.babylonjs.com/documentation/adding-scripts'"
);
});
it('should throw error if not initialized', async () => { it('should throw error if not initialized', async () => {
const uninitSearch = new LanceDBSearch(); const uninitSearch = new LanceDBSearch();
await expect(uninitSearch.getDocumentByPath('test')).rejects.toThrow( await expect(uninitSearch.getDocumentByPath('test')).rejects.toThrow(
@ -187,6 +222,32 @@ describe('LanceDBSearch', () => {
}); });
}); });
describe('pathToDocId', () => {
it('should detect editor-docs source for editor/ prefixed paths', () => {
const pathToDocId = (search as any).pathToDocId.bind(search);
const result = pathToDocId('editor/adding-scripts');
expect(result).toBe('editor-docs_adding-scripts');
});
it('should detect editor-docs source for paths containing /Editor/', () => {
const pathToDocId = (search as any).pathToDocId.bind(search);
const result = pathToDocId('/some/path/Editor/docs/page');
expect(result).toBe('editor-docs__some_path_Editor_docs_page');
});
it('should default to documentation source for other paths', () => {
const pathToDocId = (search as any).pathToDocId.bind(search);
const result = pathToDocId('features/materials');
expect(result).toBe('documentation_features_materials');
});
it('should strip page.tsx suffix for Editor docs', () => {
const pathToDocId = (search as any).pathToDocId.bind(search);
const result = pathToDocId('editor/adding-scripts/page.tsx');
expect(result).toBe('editor-docs_adding-scripts');
});
});
describe('close', () => { describe('close', () => {
it('should close without error', async () => { it('should close without error', async () => {
await expect(search.close()).resolves.not.toThrow(); await expect(search.close()).resolves.not.toThrow();

View File

@ -103,13 +103,24 @@ export class LanceDBSearch {
throw new Error('Search not initialized. Call initialize() first.'); throw new Error('Search not initialized. Call initialize() first.');
} }
// Try to find document by URL first // Try to find document by exact URL first
let results = await this.table let results = await this.table
.query() .query()
.where(`url = '${filePath}'`) .where(`url = '${filePath}'`)
.limit(1) .limit(1)
.toArray(); .toArray();
// If path looks like "editor/adding-scripts", try constructing full Editor URL
if (results.length === 0 && filePath.startsWith('editor/')) {
const editorPath = filePath.replace(/^editor\//, '');
const editorUrl = `https://editor.babylonjs.com/documentation/${editorPath}`;
results = await this.table
.query()
.where(`url = '${editorUrl}'`)
.limit(1)
.toArray();
}
if (results.length > 0) { if (results.length > 0) {
const doc = results[0]; const doc = results[0];
// Fetch fresh content from local file if available // Fetch fresh content from local file if available
@ -133,6 +144,7 @@ export class LanceDBSearch {
path.join('./data/repositories/Documentation', filePath.replace(/^.*\/content\//, '')), path.join('./data/repositories/Documentation', filePath.replace(/^.*\/content\//, '')),
path.join('./data/repositories/Babylon.js', filePath.replace(/^.*\/Babylon\.js\//, '')), path.join('./data/repositories/Babylon.js', filePath.replace(/^.*\/Babylon\.js\//, '')),
path.join('./data/repositories/havok', filePath.replace(/^.*\/havok\//, '')), path.join('./data/repositories/havok', filePath.replace(/^.*\/havok\//, '')),
path.join('./data/repositories/Editor/website/src/app/documentation', filePath.replace(/^.*\/documentation\//, '')),
]; ];
for (const possiblePath of possiblePaths) { for (const possiblePath of possiblePaths) {
@ -145,7 +157,7 @@ export class LanceDBSearch {
} }
return null; return null;
} catch (error) { } catch {
return null; return null;
} }
} }
@ -198,17 +210,21 @@ export class LanceDBSearch {
} }
private pathToDocId(filePath: string): string { private pathToDocId(filePath: string): string {
// Remove .md extension if present
let normalizedPath = filePath.replace(/\.md$/, ''); let normalizedPath = filePath.replace(/\.md$/, '');
normalizedPath = normalizedPath.replace(/\/page\.tsx$/i, '');
// Detect source type from path
let sourcePrefix = 'documentation';
if (filePath.startsWith('editor/') || filePath.includes('/Editor/')) {
sourcePrefix = 'editor-docs';
normalizedPath = normalizedPath.replace(/^editor\//, '');
}
// Strip any leading path up to and including /content/ // Strip any leading path up to and including /content/
// This handles both full paths and relative paths
normalizedPath = normalizedPath.replace(/^.*\/content\//, ''); normalizedPath = normalizedPath.replace(/^.*\/content\//, '');
// Convert slashes to underscores and prepend source name
// Note: source name is "documentation" (lowercase) as defined in index-docs.ts
const pathWithUnderscores = normalizedPath.replace(/\//g, '_'); const pathWithUnderscores = normalizedPath.replace(/\//g, '_');
return `documentation_${pathWithUnderscores}`; return `${sourcePrefix}_${pathWithUnderscores}`;
} }
async searchSourceCode( async searchSourceCode(
@ -257,6 +273,26 @@ export class LanceDBSearch {
} }
} }
async getEditorSourceFile(
filePath: string,
startLine?: number,
endLine?: number
): Promise<string | null> {
try {
const fullPath = path.join('./data/repositories/Editor', filePath);
const content = await fs.readFile(fullPath, 'utf-8');
if (startLine !== undefined && endLine !== undefined) {
const lines = content.split('\n');
return lines.slice(startLine - 1, endLine).join('\n');
}
return content;
} catch (error) {
console.error(`Error reading Editor source file ${filePath}:`, error);
return null;
}
}
async close(): Promise<void> { async close(): Promise<void> {
// LanceDB doesn't require explicit closing // LanceDB doesn't require explicit closing
} }