babylon-mcp/scripts/test-source-search.ts
Michael Mainguy 779fa53363 Add source code indexing and search with comprehensive documentation
Features:
- Implemented SourceCodeIndexer class for indexing TypeScript/JavaScript source files
  - Chunks large files into 200-line segments with 20-line overlap
  - Extracts imports, exports, and metadata
  - Generates semantic embeddings using Xenova/all-MiniLM-L6-v2
  - Creates GitHub URLs with line numbers for easy navigation

- Enhanced LanceDBSearch with source code search capabilities
  - Added searchSourceCode() method for semantic source code search
  - Added getSourceFile() method for retrieving specific files or line ranges
  - Supports package filtering and configurable table names
  - Fixed score calculation to ensure values between 0-100%

- Added two new MCP tools
  - search_babylon_source: Search Babylon.js source code with semantic search
  - get_babylon_source: Retrieve full source files or specific line ranges
  - Both tools include comprehensive error handling and JSON responses

- Created indexing and testing scripts
  - scripts/index-source.ts: Production script for indexing all packages
  - scripts/test-source-indexing.ts: Test script for core package only
  - scripts/test-source-search.ts: Test script for search functionality

- Updated package.json with comprehensive indexing commands
  - npm run index:docs - Index documentation only
  - npm run index:api - Index API documentation only
  - npm run index:source - Index source code only
  - npm run index:all - Master script to index everything

- Created comprehensive README.md
  - Complete setup and installation instructions
  - Claude Desktop integration guide with configuration examples
  - Documentation of all 5 MCP tools with parameters and examples
  - Project structure, development commands, and troubleshooting guide
  - Architecture overview and disk space requirements

Testing:
- All 118 tests passing
- TypeScript compilation successful
- Source code search verified with real queries
- Successfully indexed 1,561 files into 5,650 searchable chunks

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-11-23 06:34:00 -06:00

72 lines
2.4 KiB
TypeScript

import { LanceDBSearch } from '../src/search/lancedb-search.js';
async function main() {
console.log('Testing source code search...\n');
// Note: We use babylon_docs as the main table, but specify babylon_source_test for source code search
const search = new LanceDBSearch('./data/lancedb', 'babylon_docs');
await search.initialize();
try {
// Test 1: Search for getMeshByName implementation
console.log('='.repeat(80));
console.log('Test 1: Searching for "getMeshByName implementation"');
console.log('='.repeat(80));
const results1 = await search.searchSourceCode('getMeshByName implementation', {
limit: 3,
tableName: 'babylon_source_test'
});
console.log(`Found ${results1.length} results:\n`);
for (const result of results1) {
console.log(`File: ${result.filePath}`);
console.log(`Lines: ${result.startLine}-${result.endLine}`);
console.log(`Score: ${(result.score * 100).toFixed(1)}%`);
console.log(`Preview: ${result.content.substring(0, 200)}...`);
console.log(`URL: ${result.url}`);
console.log('-'.repeat(80));
}
// Test 2: Get specific source file
console.log('\n');
console.log('='.repeat(80));
console.log('Test 2: Getting source file scene.ts lines 4100-4110');
console.log('='.repeat(80));
const sourceCode = await search.getSourceFile('packages/dev/core/src/scene.ts', 4100, 4110);
if (sourceCode) {
console.log(sourceCode);
} else {
console.log('File not found');
}
// Test 3: Search for mesh management
console.log('\n');
console.log('='.repeat(80));
console.log('Test 3: Searching for "mesh management scene"');
console.log('='.repeat(80));
const results3 = await search.searchSourceCode('mesh management scene', {
limit: 2,
tableName: 'babylon_source_test'
});
console.log(`Found ${results3.length} results:\n`);
for (const result of results3) {
console.log(`File: ${result.filePath}`);
console.log(`Lines: ${result.startLine}-${result.endLine}`);
console.log(`Exports: ${result.exports}`);
console.log(`Score: ${(result.score * 100).toFixed(1)}%`);
console.log('-'.repeat(80));
}
} catch (error) {
console.error('Error during search:', error);
if (error instanceof Error) {
console.error('Stack:', error.stack);
}
}
await search.close();
}
main().catch(console.error);