Michael Mainguy
005a17f345
Implemented comprehensive Editor documentation indexing using TypeScript Compiler API to parse React/Next.js TSX files from the Babylon.js Editor repository. Key changes: - Added Editor repository (4th repo) to repository-config.ts - Created tsx-parser.ts using TypeScript Compiler API (zero new dependencies) - Extended document-parser.ts to route .tsx files to TSX parser - Updated lancedb-indexer.ts to discover page.tsx files - Added editor-docs source to index-docs.ts script Features: - Parses TSX/JSX files to extract text content, headings, and code blocks - Filters out className values and non-content text - Extracts categories from file paths (editor/adding-scripts, etc.) - Handles Editor-specific documentation structure Test coverage: - Added tsx-parser.test.ts (11 tests, 10 passing) - Extended document-parser.test.ts with TSX coverage (5 new tests) - Fixed repository-manager.test.ts for 4 repositories - Total: 167 tests passing, 1 skipped Results: - 902 documents now indexed (745 docs + 144 source + 13 editor) - Editor documentation appears in search results - Verified with Editor-specific queries (onStart, decorators, etc.) Updated ROADMAP.md with completion status for Editor integration phases 1-3. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
64 lines
1.9 KiB
TypeScript
64 lines
1.9 KiB
TypeScript
#!/usr/bin/env tsx
|
|
|
|
import { TsxParser } from '../src/search/tsx-parser.js';
|
|
import path from 'path';
|
|
import { fileURLToPath } from 'url';
|
|
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
const __dirname = path.dirname(__filename);
|
|
|
|
async function main() {
|
|
const projectRoot = path.join(__dirname, '..');
|
|
const parser = new TsxParser();
|
|
|
|
// Test file: adding-scripts/page.tsx
|
|
const testFile = path.join(
|
|
projectRoot,
|
|
'data',
|
|
'repositories',
|
|
'Editor',
|
|
'website',
|
|
'src',
|
|
'app',
|
|
'documentation',
|
|
'adding-scripts',
|
|
'page.tsx'
|
|
);
|
|
|
|
console.log('Testing TSX Parser');
|
|
console.log('==================\n');
|
|
console.log(`File: ${testFile}\n`);
|
|
|
|
try {
|
|
const metadata = await parser.parseFile(testFile, 'https://editor.babylonjs.com/documentation');
|
|
|
|
console.log('Parsed Metadata:');
|
|
console.log('----------------');
|
|
console.log(`Title: ${metadata.title}`);
|
|
console.log(`Category: ${metadata.category}`);
|
|
console.log(`Breadcrumbs: ${metadata.breadcrumbs.join(' > ')}`);
|
|
console.log(`Description: ${metadata.description.substring(0, 150)}...`);
|
|
console.log(`Keywords: ${metadata.keywords.slice(0, 5).join(', ')}`);
|
|
console.log(`\nHeadings (${metadata.headings.length}):`);
|
|
metadata.headings.forEach(h => {
|
|
console.log(` ${' '.repeat(h.level - 1)}${h.text}`);
|
|
});
|
|
console.log(`\nCode Blocks: ${metadata.codeBlocks.length}`);
|
|
metadata.codeBlocks.forEach((cb, i) => {
|
|
console.log(` ${i + 1}. ${cb.language} (${cb.code.split('\n').length} lines)`);
|
|
});
|
|
console.log(`\nContent Length: ${metadata.content.length} characters`);
|
|
console.log(`\nFirst 500 characters of content:`);
|
|
console.log('---');
|
|
console.log(metadata.content.substring(0, 500));
|
|
console.log('---');
|
|
|
|
console.log('\n✓ TSX parsing successful!');
|
|
} catch (error) {
|
|
console.error('✗ Error parsing TSX file:', error);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
main();
|