Features: - Implemented SourceCodeIndexer class for indexing TypeScript/JavaScript source files - Chunks large files into 200-line segments with 20-line overlap - Extracts imports, exports, and metadata - Generates semantic embeddings using Xenova/all-MiniLM-L6-v2 - Creates GitHub URLs with line numbers for easy navigation - Enhanced LanceDBSearch with source code search capabilities - Added searchSourceCode() method for semantic source code search - Added getSourceFile() method for retrieving specific files or line ranges - Supports package filtering and configurable table names - Fixed score calculation to ensure values between 0-100% - Added two new MCP tools - search_babylon_source: Search Babylon.js source code with semantic search - get_babylon_source: Retrieve full source files or specific line ranges - Both tools include comprehensive error handling and JSON responses - Created indexing and testing scripts - scripts/index-source.ts: Production script for indexing all packages - scripts/test-source-indexing.ts: Test script for core package only - scripts/test-source-search.ts: Test script for search functionality - Updated package.json with comprehensive indexing commands - npm run index:docs - Index documentation only - npm run index:api - Index API documentation only - npm run index:source - Index source code only - npm run index:all - Master script to index everything - Created comprehensive README.md - Complete setup and installation instructions - Claude Desktop integration guide with configuration examples - Documentation of all 5 MCP tools with parameters and examples - Project structure, development commands, and troubleshooting guide - Architecture overview and disk space requirements Testing: - All 118 tests passing - TypeScript compilation successful - Source code search verified with real queries - Successfully indexed 1,561 files into 5,650 searchable chunks 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
72 lines
2.4 KiB
TypeScript
72 lines
2.4 KiB
TypeScript
import { LanceDBSearch } from '../src/search/lancedb-search.js';
|
|
|
|
async function main() {
|
|
console.log('Testing source code search...\n');
|
|
|
|
// Note: We use babylon_docs as the main table, but specify babylon_source_test for source code search
|
|
const search = new LanceDBSearch('./data/lancedb', 'babylon_docs');
|
|
await search.initialize();
|
|
|
|
try {
|
|
// Test 1: Search for getMeshByName implementation
|
|
console.log('='.repeat(80));
|
|
console.log('Test 1: Searching for "getMeshByName implementation"');
|
|
console.log('='.repeat(80));
|
|
const results1 = await search.searchSourceCode('getMeshByName implementation', {
|
|
limit: 3,
|
|
tableName: 'babylon_source_test'
|
|
});
|
|
console.log(`Found ${results1.length} results:\n`);
|
|
|
|
for (const result of results1) {
|
|
console.log(`File: ${result.filePath}`);
|
|
console.log(`Lines: ${result.startLine}-${result.endLine}`);
|
|
console.log(`Score: ${(result.score * 100).toFixed(1)}%`);
|
|
console.log(`Preview: ${result.content.substring(0, 200)}...`);
|
|
console.log(`URL: ${result.url}`);
|
|
console.log('-'.repeat(80));
|
|
}
|
|
|
|
// Test 2: Get specific source file
|
|
console.log('\n');
|
|
console.log('='.repeat(80));
|
|
console.log('Test 2: Getting source file scene.ts lines 4100-4110');
|
|
console.log('='.repeat(80));
|
|
const sourceCode = await search.getSourceFile('packages/dev/core/src/scene.ts', 4100, 4110);
|
|
if (sourceCode) {
|
|
console.log(sourceCode);
|
|
} else {
|
|
console.log('File not found');
|
|
}
|
|
|
|
// Test 3: Search for mesh management
|
|
console.log('\n');
|
|
console.log('='.repeat(80));
|
|
console.log('Test 3: Searching for "mesh management scene"');
|
|
console.log('='.repeat(80));
|
|
const results3 = await search.searchSourceCode('mesh management scene', {
|
|
limit: 2,
|
|
tableName: 'babylon_source_test'
|
|
});
|
|
console.log(`Found ${results3.length} results:\n`);
|
|
|
|
for (const result of results3) {
|
|
console.log(`File: ${result.filePath}`);
|
|
console.log(`Lines: ${result.startLine}-${result.endLine}`);
|
|
console.log(`Exports: ${result.exports}`);
|
|
console.log(`Score: ${(result.score * 100).toFixed(1)}%`);
|
|
console.log('-'.repeat(80));
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error('Error during search:', error);
|
|
if (error instanceof Error) {
|
|
console.error('Stack:', error.stack);
|
|
}
|
|
}
|
|
|
|
await search.close();
|
|
}
|
|
|
|
main().catch(console.error);
|