diff --git a/src/cli/commands/index-cmd.ts b/src/cli/commands/index-cmd.ts new file mode 100644 index 0000000..84a6fed --- /dev/null +++ b/src/cli/commands/index-cmd.ts @@ -0,0 +1,61 @@ +import { Command } from 'commander'; +import chalk from 'chalk'; +import { indexProject } from '../../core/indexer'; + +export const indexCommand = new Command('index') + .description('Index a codebase to create component nodes') + .argument('[path]', 'Path to index', '.') + .option('--update', 'Only update changed files (incremental)') + .option('--dry-run', 'Preview what would be indexed without making changes') + .option('--depth ', 'Maximum directory depth', '10') + .option('--lang ', 'Only index specific language (ts, js, py)') + .option('--ignore ', 'Additional ignore patterns (comma-separated)') + .action(async (inputPath: string, opts) => { + const startTime = Date.now(); + + if (opts.dryRun) { + console.log(chalk.yellow('Dry run mode - no changes will be made\n')); + } + + console.log(chalk.cyan(`Indexing ${inputPath}...`)); + + try { + const result = await indexProject(inputPath, { + update: opts.update, + dryRun: opts.dryRun, + maxDepth: parseInt(opts.depth), + language: opts.lang, + ignore: opts.ignore?.split(',').map((s: string) => s.trim()), + }); + + const elapsed = ((Date.now() - startTime) / 1000).toFixed(2); + + console.log(); + console.log(chalk.green(`✓ Indexed ${result.projectName} (${result.projectType})`)); + console.log(); + console.log(` Files scanned: ${result.files.length}`); + console.log(` Components created: ${result.componentsCreated}`); + console.log(` Components updated: ${result.componentsUpdated}`); + console.log(` Components removed: ${result.componentsRemoved}`); + console.log(` Relationships: ${result.relationshipsCreated}`); + if (result.architectureNodeId) { + console.log(` Architecture node: ${result.architectureNodeId.slice(0, 8)}`); + } + console.log(); + console.log(chalk.dim(`Completed in ${elapsed}s`)); + + if (opts.dryRun && result.files.length > 0) { + console.log(); + console.log(chalk.yellow('Files that would be indexed:')); + for (const file of result.files.slice(0, 20)) { + console.log(` ${file}`); + } + if (result.files.length > 20) { + console.log(chalk.dim(` ... and ${result.files.length - 20} more`)); + } + } + } catch (err: any) { + console.error(chalk.red(`Error: ${err.message}`)); + process.exit(1); + } + }); diff --git a/src/cli/index.ts b/src/cli/index.ts index 8bc0b6c..27602a8 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -16,6 +16,7 @@ import { diffCommand } from './commands/diff'; import { restoreCommand } from './commands/restore'; import { captureCommand, captureHookCommand, configCommand } from './commands/capture'; import { contextCommand, contextHookCommand } from './commands/context'; +import { indexCommand } from './commands/index-cmd'; import { closeDb } from '../core/db'; const program = new Command(); @@ -44,6 +45,7 @@ program.addCommand(captureHookCommand); program.addCommand(contextCommand); program.addCommand(contextHookCommand); program.addCommand(configCommand); +program.addCommand(indexCommand); program.hook('postAction', () => { closeDb(); diff --git a/src/core/indexer/architecture.ts b/src/core/indexer/architecture.ts new file mode 100644 index 0000000..e79c00a --- /dev/null +++ b/src/core/indexer/architecture.ts @@ -0,0 +1,79 @@ +import * as path from 'path'; +import { Node } from '../../types'; +import { ProjectInfo } from './detector'; +import { getDirectoryTree } from './scanner'; + +export interface ArchitectureSummary { + projectName: string; + projectType: string; + description: string; + techStack: string[]; + keyComponents: { name: string; path: string; exports: number }[]; + directoryStructure: string; +} + +export function generateArchitectureSummary( + projectRoot: string, + projectInfo: ProjectInfo, + components: Node[] +): ArchitectureSummary { + // Determine tech stack from dependencies + const techStack: string[] = [projectInfo.type]; + const deps = [...projectInfo.dependencies, ...(projectInfo.devDependencies || [])]; + + // Detect frameworks/libraries + if (deps.includes('react') || deps.includes('react-dom')) techStack.push('React'); + if (deps.includes('vue')) techStack.push('Vue'); + if (deps.includes('express')) techStack.push('Express'); + if (deps.includes('fastify')) techStack.push('Fastify'); + if (deps.includes('next')) techStack.push('Next.js'); + if (deps.includes('typescript')) techStack.push('TypeScript'); + if (deps.includes('prisma')) techStack.push('Prisma'); + if (deps.includes('sequelize')) techStack.push('Sequelize'); + if (deps.includes('mongoose')) techStack.push('MongoDB'); + if (deps.includes('better-sqlite3') || deps.includes('sqlite3')) techStack.push('SQLite'); + + // Get key components (most exports or largest) + const keyComponents = components + .map(n => ({ + name: n.title, + path: n.metadata?.filePath as string || '', + exports: (n.metadata?.exports as string[])?.length || 0, + })) + .sort((a, b) => b.exports - a.exports) + .slice(0, 10); + + return { + projectName: projectInfo.name, + projectType: projectInfo.type, + description: projectInfo.description || `A ${projectInfo.type} project`, + techStack, + keyComponents, + directoryStructure: getDirectoryTree(projectRoot, 3), + }; +} + +export function formatArchitectureAsMarkdown(summary: ArchitectureSummary): string { + const sections: string[] = []; + + sections.push(`# ${summary.projectName} Architecture\n`); + sections.push(`${summary.description}\n`); + + sections.push(`## Tech Stack\n`); + sections.push(summary.techStack.map(t => `- ${t}`).join('\n') + '\n'); + + if (summary.keyComponents.length > 0) { + sections.push(`## Key Components\n`); + for (const comp of summary.keyComponents) { + sections.push(`- **${comp.name}** (${comp.path}) - ${comp.exports} exports`); + } + sections.push(''); + } + + sections.push(`## Directory Structure\n`); + sections.push('```'); + sections.push(summary.directoryStructure); + sections.push('```'); + + return sections.join('\n'); +} diff --git a/src/core/indexer/detector.ts b/src/core/indexer/detector.ts new file mode 100644 index 0000000..a2f710e --- /dev/null +++ b/src/core/indexer/detector.ts @@ -0,0 +1,145 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +export type ProjectType = 'nodejs' | 'python' | 'rust' | 'go' | 'generic'; + +export interface ProjectInfo { + type: ProjectType; + name: string; + description?: string; + dependencies: string[]; + devDependencies?: string[]; + scripts?: Record; + entryPoints: string[]; +} + +export async function detectProjectType(root: string): Promise { + const absRoot = path.resolve(root); + + // Check for Node.js (package.json) + const packageJsonPath = path.join(absRoot, 'package.json'); + if (fs.existsSync(packageJsonPath)) { + return parseNodeProject(absRoot, packageJsonPath); + } + + // Check for Python + const pyprojectPath = path.join(absRoot, 'pyproject.toml'); + const setupPyPath = path.join(absRoot, 'setup.py'); + const requirementsPath = path.join(absRoot, 'requirements.txt'); + if (fs.existsSync(pyprojectPath) || fs.existsSync(setupPyPath) || fs.existsSync(requirementsPath)) { + return parsePythonProject(absRoot); + } + + // Check for Rust (Cargo.toml) + const cargoPath = path.join(absRoot, 'Cargo.toml'); + if (fs.existsSync(cargoPath)) { + return parseRustProject(absRoot, cargoPath); + } + + // Check for Go (go.mod) + const goModPath = path.join(absRoot, 'go.mod'); + if (fs.existsSync(goModPath)) { + return parseGoProject(absRoot, goModPath); + } + + return parseGenericProject(absRoot); +} + +function parseNodeProject(root: string, packageJsonPath: string): ProjectInfo { + const content = fs.readFileSync(packageJsonPath, 'utf-8'); + const pkg = JSON.parse(content); + const deps = Object.keys(pkg.dependencies || {}); + const devDeps = Object.keys(pkg.devDependencies || {}); + const entryPoints: string[] = []; + if (pkg.main) entryPoints.push(pkg.main); + if (pkg.bin) { + if (typeof pkg.bin === 'string') entryPoints.push(pkg.bin); + else entryPoints.push(...Object.values(pkg.bin) as string[]); + } + return { + type: 'nodejs', + name: pkg.name || path.basename(root), + description: pkg.description, + dependencies: deps, + devDependencies: devDeps, + scripts: pkg.scripts, + entryPoints, + }; +} + +function parsePythonProject(root: string): ProjectInfo { + const name = path.basename(root); + const deps: string[] = []; + const entryPoints: string[] = []; + const reqPath = path.join(root, 'requirements.txt'); + if (fs.existsSync(reqPath)) { + const content = fs.readFileSync(reqPath, 'utf-8'); + for (const line of content.split('\n')) { + const trimmed = line.trim(); + if (trimmed && !trimmed.startsWith('#')) { + const pkg = trimmed.split(/[=<>!]/)[0].trim(); + if (pkg) deps.push(pkg); + } + } + } + for (const entry of ['main.py', 'app.py', '__main__.py']) { + if (fs.existsSync(path.join(root, entry))) entryPoints.push(entry); + } + return { type: 'python', name, dependencies: deps, entryPoints }; +} + +function parseRustProject(root: string, cargoPath: string): ProjectInfo { + const content = fs.readFileSync(cargoPath, 'utf-8'); + const nameMatch = content.match(/^name\s*=\s*["']([^"']+)["']/m); + const descMatch = content.match(/^description\s*=\s*["']([^"']+)["']/m); + const deps: string[] = []; + const depsSection = content.match(/\[dependencies\]([\s\S]*?)(?:\[|$)/); + if (depsSection) { + const depLines = depsSection[1].match(/^(\w[\w-]*)\s*=/gm); + if (depLines) deps.push(...depLines.map(d => d.replace(/\s*=.*/, ''))); + } + const entryPoints: string[] = []; + if (fs.existsSync(path.join(root, 'src/main.rs'))) entryPoints.push('src/main.rs'); + if (fs.existsSync(path.join(root, 'src/lib.rs'))) entryPoints.push('src/lib.rs'); + return { + type: 'rust', + name: nameMatch?.[1] || path.basename(root), + description: descMatch?.[1], + dependencies: deps, + entryPoints, + }; +} + +function parseGoProject(root: string, goModPath: string): ProjectInfo { + const content = fs.readFileSync(goModPath, 'utf-8'); + const moduleMatch = content.match(/^module\s+(\S+)/m); + const deps: string[] = []; + const requireMatch = content.match(/require\s*\(([\s\S]*?)\)/); + if (requireMatch) { + const reqLines = requireMatch[1].match(/^\s*(\S+)\s+v/gm); + if (reqLines) deps.push(...reqLines.map(d => d.trim().split(/\s/)[0])); + } + const entryPoints: string[] = []; + if (fs.existsSync(path.join(root, 'main.go'))) entryPoints.push('main.go'); + return { + type: 'go', + name: moduleMatch?.[1]?.split('/').pop() || path.basename(root), + dependencies: deps, + entryPoints, + }; +} + +function parseGenericProject(root: string): ProjectInfo { + const name = path.basename(root); + let description: string | undefined; + for (const readme of ['README.md', 'README.txt', 'README']) { + const readmePath = path.join(root, readme); + if (fs.existsSync(readmePath)) { + const content = fs.readFileSync(readmePath, 'utf-8'); + const firstPara = content.split(/\n\n/)[0].replace(/^#.*\n/, '').trim(); + if (firstPara) description = firstPara.slice(0, 200); + break; + } + } + return { type: 'generic', name, description, dependencies: [], entryPoints: [] }; +} diff --git a/src/core/indexer/index.ts b/src/core/indexer/index.ts new file mode 100644 index 0000000..37a07b3 --- /dev/null +++ b/src/core/indexer/index.ts @@ -0,0 +1,6 @@ +export { detectProjectType, type ProjectType, type ProjectInfo } from './detector'; +export { scanFiles, getDirectoryTree, type ScanOptions, type ScannedFile } from './scanner'; +export { mapRelationships, resolveImportPath, loadIndexState, saveIndexState, type ComponentNode, type MappedRelationship, type IndexState } from './mapper'; +export { generateArchitectureSummary, formatArchitectureAsMarkdown, type ArchitectureSummary } from './architecture'; +export { indexProject, type IndexOptions, type IndexResult } from './indexProject'; +export * from './parsers'; diff --git a/src/core/indexer/indexProject.ts b/src/core/indexer/indexProject.ts new file mode 100644 index 0000000..6990809 --- /dev/null +++ b/src/core/indexer/indexProject.ts @@ -0,0 +1,246 @@ +import * as path from 'path'; +import { addNode, addEdge, listNodes, updateNode, removeNode } from '../store'; +import { detectProjectType, ProjectInfo } from './detector'; +import { scanFiles, ScannedFile } from './scanner'; +import { parseTypeScript, parsePython } from './parsers'; +import { mapRelationships, loadIndexState, saveIndexState, IndexState, ComponentNode } from './mapper'; +import { generateArchitectureSummary, formatArchitectureAsMarkdown } from './architecture'; +import { Node } from '../../types'; + +export interface IndexOptions { + update?: boolean; + dryRun?: boolean; + maxDepth?: number; + language?: string; + ignore?: string[]; +} + +export interface IndexResult { + projectName: string; + projectType: string; + componentsCreated: number; + componentsUpdated: number; + componentsRemoved: number; + relationshipsCreated: number; + architectureNodeId: string | null; + files: string[]; +} + +export async function indexProject(root: string, options: IndexOptions = {}): Promise { + const absRoot = path.resolve(root); + const projectInfo = await detectProjectType(absRoot); + const state = options.update ? loadIndexState(absRoot) : null; + + // Filter extensions by language if specified + let extensions: string[] | undefined; + if (options.language) { + const langMap: Record = { + ts: ['.ts', '.tsx'], + typescript: ['.ts', '.tsx'], + js: ['.js', '.jsx', '.mjs', '.cjs'], + javascript: ['.js', '.jsx', '.mjs', '.cjs'], + py: ['.py'], + python: ['.py'], + }; + extensions = langMap[options.language.toLowerCase()]; + } + + // Scan files + const files = await scanFiles(absRoot, { + maxDepth: options.maxDepth, + ignore: options.ignore, + extensions, + }); + + if (options.dryRun) { + return { + projectName: projectInfo.name, + projectType: projectInfo.type, + componentsCreated: files.length, + componentsUpdated: 0, + componentsRemoved: 0, + relationshipsCreated: 0, + architectureNodeId: null, + files: files.map(f => f.relativePath), + }; + } + + // Track what we're creating + const newState: IndexState = { + projectPath: absRoot, + projectName: projectInfo.name, + lastIndexed: Date.now(), + fileHashes: {}, + nodeIds: {}, + }; + + const componentNodes: ComponentNode[] = []; + let created = 0; + let updated = 0; + + // Process each file + for (const file of files) { + // Check if file changed (for incremental updates) + if (state && state.fileHashes[file.relativePath] === file.hash) { + // File unchanged, keep existing node + if (state.nodeIds[file.relativePath]) { + newState.fileHashes[file.relativePath] = file.hash; + newState.nodeIds[file.relativePath] = state.nodeIds[file.relativePath]; + // Still need to add to componentNodes for relationship mapping + const existingNode = listNodes({ limit: 1 }).find(n => n.id === state.nodeIds[file.relativePath]); + if (existingNode) { + componentNodes.push({ + node: existingNode, + filePath: file.relativePath, + imports: (existingNode.metadata?.imports as string[]) || [], + }); + } + } + continue; + } + + // Parse file based on extension + let parsed: { exports: any[]; imports: any[]; classes: any[]; functions: any[]; loc: number } | null = null; + try { + if (['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'].includes(file.extension)) { + parsed = await parseTypeScript(file.path); + } else if (file.extension === '.py') { + parsed = await parsePython(file.path); + } + } catch { + // Skip files that fail to parse + continue; + } + + if (!parsed) continue; + + // Create component title from file path + const fileName = path.basename(file.relativePath, file.extension); + const dirName = path.dirname(file.relativePath); + const title = dirName === '.' ? fileName : `${dirName}/${fileName}`; + + // Build content summary + const exportNames = parsed.exports.map((e: any) => e.name || e).slice(0, 20); + const classNames = parsed.classes.map((c: any) => c.name); + const funcNames = parsed.functions.map((f: any) => f.name).slice(0, 10); + + const contentParts: string[] = []; + if (exportNames.length) contentParts.push(`Exports: ${exportNames.join(', ')}`); + if (classNames.length) contentParts.push(`Classes: ${classNames.join(', ')}`); + if (funcNames.length) contentParts.push(`Functions: ${funcNames.join(', ')}`); + contentParts.push(`Lines: ${parsed.loc}`); + + const content = contentParts.join('\n'); + const imports = parsed.imports.map((i: any) => i.source || i.module); + + // Check if node exists + const existingNodeId = state?.nodeIds[file.relativePath]; + let node: Node; + + if (existingNodeId) { + // Update existing + const updatedNode = await updateNode(existingNodeId, { + content, + metadata: { + filePath: file.relativePath, + extension: file.extension, + exports: exportNames, + imports, + loc: parsed.loc, + indexedAt: Date.now(), + }, + }); + if (updatedNode) { + node = updatedNode; + updated++; + } else { + continue; + } + } else { + // Create new + node = await addNode({ + kind: 'component', + title, + content, + tags: [projectInfo.name, file.extension.slice(1), 'indexed'], + metadata: { + filePath: file.relativePath, + extension: file.extension, + exports: exportNames, + imports, + loc: parsed.loc, + indexedAt: Date.now(), + }, + }); + created++; + } + + newState.fileHashes[file.relativePath] = file.hash; + newState.nodeIds[file.relativePath] = node.id; + componentNodes.push({ node, filePath: file.relativePath, imports }); + } + + // Remove nodes for deleted files + let removed = 0; + if (state) { + for (const [filePath, nodeId] of Object.entries(state.nodeIds)) { + if (!newState.nodeIds[filePath]) { + removeNode(nodeId, true); + removed++; + } + } + } + + // Map relationships + const relationships = mapRelationships(componentNodes, absRoot); + let relCreated = 0; + for (const rel of relationships) { + try { + addEdge(rel.fromId, rel.toId, rel.type); + relCreated++; + } catch { + // Edge might already exist + } + } + + // Create/update architecture summary node + let archNodeId: string | null = null; + const archTitle = `${projectInfo.name} Architecture`; + const existingArch = listNodes({ kind: 'component', tags: [projectInfo.name, 'architecture'] }); + + const summary = generateArchitectureSummary(absRoot, projectInfo, componentNodes.map(c => c.node)); + const archContent = formatArchitectureAsMarkdown(summary); + + if (existingArch.length > 0) { + await updateNode(existingArch[0].id, { content: archContent }); + archNodeId = existingArch[0].id; + } else { + const archNode = await addNode({ + kind: 'component', + title: archTitle, + content: archContent, + tags: [projectInfo.name, 'architecture', 'indexed'], + metadata: { + projectType: projectInfo.type, + techStack: summary.techStack, + componentCount: componentNodes.length, + indexedAt: Date.now(), + }, + }); + archNodeId = archNode.id; + } + + // Save state + saveIndexState(absRoot, newState); + + return { + projectName: projectInfo.name, + projectType: projectInfo.type, + componentsCreated: created, + componentsUpdated: updated, + componentsRemoved: removed, + relationshipsCreated: relCreated, + architectureNodeId: archNodeId, + files: files.map(f => f.relativePath), + }; +} diff --git a/src/core/indexer/mapper.ts b/src/core/indexer/mapper.ts new file mode 100644 index 0000000..0643406 --- /dev/null +++ b/src/core/indexer/mapper.ts @@ -0,0 +1,109 @@ +import * as path from 'path'; +import * as fs from 'fs'; +import { Node } from '../../types'; + +export interface ComponentNode { + node: Node; + filePath: string; + imports: string[]; +} + +export interface MappedRelationship { + fromId: string; + toId: string; + fromPath: string; + toPath: string; + type: 'depends_on' | 'contains'; +} + +export function mapRelationships(components: ComponentNode[], projectRoot: string): MappedRelationship[] { + const relationships: MappedRelationship[] = []; + const pathToComponent = new Map(); + + // Build lookup map + for (const comp of components) { + pathToComponent.set(comp.filePath, comp); + // Also map without extension for JS/TS imports + const withoutExt = comp.filePath.replace(/\.(ts|tsx|js|jsx|mjs|cjs)$/, ''); + pathToComponent.set(withoutExt, comp); + } + + // Map import relationships + for (const comp of components) { + for (const importPath of comp.imports) { + const resolved = resolveImportPath(importPath, comp.filePath, projectRoot); + if (resolved) { + const target = pathToComponent.get(resolved) || pathToComponent.get(resolved.replace(/\.(ts|tsx|js|jsx)$/, '')); + if (target && target.node.id !== comp.node.id) { + relationships.push({ + fromId: comp.node.id, + toId: target.node.id, + fromPath: comp.filePath, + toPath: target.filePath, + type: 'depends_on', + }); + } + } + } + } + + return relationships; +} + +export function resolveImportPath(importSource: string, fromFile: string, projectRoot: string): string | null { + // Skip external packages + if (!importSource.startsWith('.') && !importSource.startsWith('/')) { + return null; + } + + const fromDir = path.dirname(fromFile); + let resolved: string; + + if (importSource.startsWith('.')) { + resolved = path.resolve(fromDir, importSource); + } else { + resolved = path.resolve(projectRoot, importSource); + } + + // Try with various extensions + const extensions = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '/index.ts', '/index.js']; + for (const ext of extensions) { + const withExt = resolved + ext; + if (fs.existsSync(withExt)) { + return path.relative(projectRoot, withExt); + } + } + + // Check if it exists as-is + if (fs.existsSync(resolved)) { + return path.relative(projectRoot, resolved); + } + + return null; +} + +export interface IndexState { + projectPath: string; + projectName: string; + lastIndexed: number; + fileHashes: Record; + nodeIds: Record; +} + +const STATE_FILE = '.cortex-index.json'; + +export function loadIndexState(projectRoot: string): IndexState | null { + const statePath = path.join(projectRoot, STATE_FILE); + if (!fs.existsSync(statePath)) return null; + try { + const content = fs.readFileSync(statePath, 'utf-8'); + return JSON.parse(content); + } catch { + return null; + } +} + +export function saveIndexState(projectRoot: string, state: IndexState): void { + const statePath = path.join(projectRoot, STATE_FILE); + fs.writeFileSync(statePath, JSON.stringify(state, null, 2)); +} diff --git a/src/core/indexer/parsers/index.ts b/src/core/indexer/parsers/index.ts new file mode 100644 index 0000000..a85903d --- /dev/null +++ b/src/core/indexer/parsers/index.ts @@ -0,0 +1,2 @@ +export { parseTypeScript, type ParsedTSFile, type ExportInfo, type ImportInfo, type ClassInfo, type FunctionInfo } from './typescript'; +export { parsePython, type ParsedPyFile, type PyImport, type PyClass, type PyFunction } from './python'; diff --git a/src/core/indexer/parsers/python.ts b/src/core/indexer/parsers/python.ts new file mode 100644 index 0000000..8c12fe3 --- /dev/null +++ b/src/core/indexer/parsers/python.ts @@ -0,0 +1,120 @@ +import * as fs from 'fs'; + +export interface ParsedPyFile { + filePath: string; + language: 'python'; + imports: PyImport[]; + classes: PyClass[]; + functions: PyFunction[]; + exports: string[]; + loc: number; +} + +export interface PyImport { + module: string; + names: string[]; + isRelative: boolean; + line: number; +} + +export interface PyClass { + name: string; + bases: string[]; + methods: string[]; + decorators: string[]; + line: number; +} + +export interface PyFunction { + name: string; + isAsync: boolean; + decorators: string[]; + line: number; +} + +export async function parsePython(filePath: string): Promise { + const content = fs.readFileSync(filePath, 'utf-8'); + const lines = content.split('\n'); + + const imports: PyImport[] = []; + const classes: PyClass[] = []; + const functions: PyFunction[] = []; + let exports: string[] = []; + + let currentDecorators: string[] = []; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const lineNum = i + 1; + + // __all__ exports + const allMatch = line.match(/^__all__\s*=\s*\[([^\]]+)\]/); + if (allMatch) { + exports = allMatch[1].match(/['"](\w+)['"]/g)?.map(s => s.slice(1, -1)) || []; + } + + // Decorators + const decoMatch = line.match(/^@(\w+)/); + if (decoMatch) { + currentDecorators.push(decoMatch[1]); + continue; + } + + // Import statements + const importMatch = line.match(/^import\s+(\S+)/); + if (importMatch) { + imports.push({ module: importMatch[1], names: [importMatch[1]], isRelative: false, line: lineNum }); + currentDecorators = []; + continue; + } + + const fromMatch = line.match(/^from\s+(\S+)\s+import\s+(.+)/); + if (fromMatch) { + const module = fromMatch[1]; + const names = fromMatch[2].split(',').map(n => n.trim().split(/\s+as\s+/)[0]).filter(Boolean); + imports.push({ module, names, isRelative: module.startsWith('.'), line: lineNum }); + currentDecorators = []; + continue; + } + + // Class definitions + const classMatch = line.match(/^class\s+(\w+)(?:\(([^)]*)\))?:/); + if (classMatch) { + const className = classMatch[1]; + const bases = classMatch[2]?.split(',').map(b => b.trim()).filter(Boolean) || []; + const methods: string[] = []; + + // Find methods + for (let j = i + 1; j < lines.length; j++) { + const methodLine = lines[j]; + if (methodLine.match(/^\S/) && !methodLine.match(/^\s*#/) && !methodLine.match(/^\s*$/)) break; + const methodMatch = methodLine.match(/^\s+(?:async\s+)?def\s+(\w+)/); + if (methodMatch) methods.push(methodMatch[1]); + } + + classes.push({ name: className, bases, methods, decorators: currentDecorators, line: lineNum }); + currentDecorators = []; + continue; + } + + // Function definitions (top-level only) + const funcMatch = line.match(/^(async\s+)?def\s+(\w+)/); + if (funcMatch) { + functions.push({ + name: funcMatch[2], + isAsync: !!funcMatch[1], + decorators: currentDecorators, + line: lineNum, + }); + currentDecorators = []; + continue; + } + + // Reset decorators if we hit a non-decorator, non-function/class line + if (!line.match(/^\s*$/) && !line.match(/^\s*#/)) { + currentDecorators = []; + } + } + + return { filePath, language: 'python', imports, classes, functions, exports, loc: lines.length }; +} diff --git a/src/core/indexer/parsers/typescript.ts b/src/core/indexer/parsers/typescript.ts new file mode 100644 index 0000000..375346f --- /dev/null +++ b/src/core/indexer/parsers/typescript.ts @@ -0,0 +1,132 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +export interface ParsedTSFile { + filePath: string; + language: 'typescript' | 'javascript'; + exports: ExportInfo[]; + imports: ImportInfo[]; + classes: ClassInfo[]; + functions: FunctionInfo[]; + loc: number; +} + +export interface ExportInfo { + name: string; + kind: 'function' | 'class' | 'interface' | 'type' | 'const' | 'default' | 'enum'; + line: number; +} + +export interface ImportInfo { + source: string; + names: string[]; + line: number; +} + +export interface ClassInfo { + name: string; + extends?: string; + implements: string[]; + methods: string[]; + isExported: boolean; + line: number; +} + +export interface FunctionInfo { + name: string; + isAsync: boolean; + isExported: boolean; + line: number; +} + +export async function parseTypeScript(filePath: string): Promise { + const content = fs.readFileSync(filePath, 'utf-8'); + const ext = path.extname(filePath).toLowerCase(); + const language = ext === '.ts' || ext === '.tsx' ? 'typescript' : 'javascript'; + return parseContent(filePath, content, language); +} + +function parseContent(filePath: string, content: string, language: 'typescript' | 'javascript'): ParsedTSFile { + const lines = content.split('\n'); + const exports: ExportInfo[] = []; + const imports: ImportInfo[] = []; + const classes: ClassInfo[] = []; + const functions: FunctionInfo[] = []; + + // Parse imports + const importRegex = /^import\s+.*?\s+from\s+['"]([^'"]+)['"]/gm; + let match: RegExpExecArray | null; + while ((match = importRegex.exec(content)) !== null) { + const line = content.slice(0, match.index).split('\n').length; + const clause = match[0].replace(/^import\s+/, '').replace(/\s+from.*/, ''); + const names: string[] = []; + const namedMatch = clause.match(/\{([^}]+)\}/); + if (namedMatch) names.push(...namedMatch[1].split(',').map(n => n.trim().split(/\s+as\s+/)[0]).filter(Boolean)); + const defaultMatch = clause.match(/^(\w+)/); + if (defaultMatch && !clause.startsWith('{') && !clause.startsWith('*')) names.push(defaultMatch[1]); + if (clause.includes('*')) names.push('*'); + imports.push({ source: match[1], names, line }); + } + + // Parse line by line + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const lineNum = i + 1; + + // Export const/let + const constMatch = line.match(/^export\s+(const|let|var)\s+(\w+)/); + if (constMatch) exports.push({ name: constMatch[2], kind: 'const', line: lineNum }); + + // Export default + if (line.match(/^export\s+default\s+/)) { + const m = line.match(/^export\s+default\s+(?:class|function)?\s*(\w+)?/); + exports.push({ name: m?.[1] || 'default', kind: 'default', line: lineNum }); + } + + // Export type/interface + const typeMatch = line.match(/^export\s+(type|interface)\s+(\w+)/); + if (typeMatch) exports.push({ name: typeMatch[2], kind: typeMatch[1] as 'type' | 'interface', line: lineNum }); + + // Export enum + const enumMatch = line.match(/^export\s+enum\s+(\w+)/); + if (enumMatch) exports.push({ name: enumMatch[1], kind: 'enum', line: lineNum }); + + // Classes + const classMatch = line.match(/^(export\s+)?(abstract\s+)?class\s+(\w+)(?:\s+extends\s+(\w+))?(?:\s+implements\s+([^{]+))?/); + if (classMatch) { + const isExported = !!classMatch[1]; + const className = classMatch[3]; + const methods: string[] = []; + for (let j = i + 1; j < lines.length && j < i + 100; j++) { + if (lines[j].match(/^\s*\}/) && !lines[j].match(/^\s*\}\s*\)/)) break; + const methodMatch = lines[j].match(/^\s+(async\s+)?(\w+)\s*\(/); + if (methodMatch && methodMatch[2] !== 'constructor') methods.push(methodMatch[2]); + } + classes.push({ + name: className, + extends: classMatch[4], + implements: classMatch[5]?.split(',').map(s => s.trim()).filter(Boolean) || [], + methods, + isExported, + line: lineNum, + }); + if (isExported) exports.push({ name: className, kind: 'class', line: lineNum }); + } + + // Functions + const funcMatch = line.match(/^(export\s+)?(async\s+)?function\s+(\w+)/); + if (funcMatch) { + const isExported = !!funcMatch[1]; + functions.push({ name: funcMatch[3], isAsync: !!funcMatch[2], isExported, line: lineNum }); + if (isExported) exports.push({ name: funcMatch[3], kind: 'function', line: lineNum }); + } + + // Arrow function exports + const arrowMatch = line.match(/^export\s+(const|let)\s+(\w+)\s*=\s*(async\s+)?[\(]/); + if (arrowMatch) { + functions.push({ name: arrowMatch[2], isAsync: !!arrowMatch[3], isExported: true, line: lineNum }); + } + } + + return { filePath, language, exports, imports, classes, functions, loc: lines.length }; +} diff --git a/src/core/indexer/scanner.ts b/src/core/indexer/scanner.ts new file mode 100644 index 0000000..771565d --- /dev/null +++ b/src/core/indexer/scanner.ts @@ -0,0 +1,108 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import * as crypto from 'crypto'; + +export interface ScanOptions { + maxDepth?: number; + ignore?: string[]; + extensions?: string[]; +} + +export interface ScannedFile { + path: string; + relativePath: string; + extension: string; + size: number; + hash: string; +} + +const DEFAULT_IGNORE = [ + 'node_modules', '.git', 'dist', 'build', '__pycache__', '.pytest_cache', + '.mypy_cache', '.env', 'coverage', '.next', '.nuxt', 'target', 'vendor', + '.idea', '.vscode', '.DS_Store', 'Thumbs.db', +]; + +const CODE_EXTENSIONS = [ + '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', + '.py', '.rs', '.go', '.java', '.kt', '.c', '.cpp', '.h', '.hpp', + '.cs', '.rb', '.php', '.swift', '.vue', '.svelte', +]; + +export async function scanFiles(root: string, options: ScanOptions = {}): Promise { + const absRoot = path.resolve(root); + const maxDepth = options.maxDepth ?? 10; + const ignorePatterns = [...DEFAULT_IGNORE, ...(options.ignore || [])]; + const extensions = options.extensions || CODE_EXTENSIONS; + const files: ScannedFile[] = []; + + function shouldIgnore(name: string): boolean { + for (const pattern of ignorePatterns) { + if (pattern.startsWith('*.')) { + if (name.endsWith(pattern.slice(1))) return true; + } else if (name === pattern) return true; + } + return false; + } + + function walk(dir: string, depth: number): void { + if (depth > maxDepth) return; + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { return; } + + for (const entry of entries) { + if (shouldIgnore(entry.name)) continue; + const fullPath = path.join(dir, entry.name); + + if (entry.isDirectory()) { + walk(fullPath, depth + 1); + } else if (entry.isFile()) { + const ext = path.extname(entry.name).toLowerCase(); + if (extensions.includes(ext)) { + try { + const stat = fs.statSync(fullPath); + const content = fs.readFileSync(fullPath); + files.push({ + path: fullPath, + relativePath: path.relative(absRoot, fullPath), + extension: ext, + size: stat.size, + hash: crypto.createHash('md5').update(content).digest('hex'), + }); + } catch { /* skip */ } + } + } + } + } + + walk(absRoot, 0); + return files; +} + +export function getDirectoryTree(root: string, maxDepth: number = 3): string { + const absRoot = path.resolve(root); + const lines: string[] = [path.basename(absRoot) + '/']; + + function walk(dir: string, prefix: string, depth: number): void { + if (depth > maxDepth) return; + let entries: fs.Dirent[]; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + + const filtered = entries.filter(e => !DEFAULT_IGNORE.includes(e.name) && !e.name.startsWith('.')); + const dirs = filtered.filter(e => e.isDirectory()).sort((a, b) => a.name.localeCompare(b.name)); + const files = filtered.filter(e => e.isFile()).sort((a, b) => a.name.localeCompare(b.name)); + const sorted = [...dirs, ...files]; + + sorted.forEach((entry, i) => { + const isLast = i === sorted.length - 1; + lines.push(prefix + (isLast ? '└── ' : '├── ') + entry.name + (entry.isDirectory() ? '/' : '')); + if (entry.isDirectory()) { + walk(path.join(dir, entry.name), prefix + (isLast ? ' ' : '│ '), depth + 1); + } + }); + } + + walk(absRoot, '', 0); + return lines.join('\n'); +} diff --git a/src/mcp/index.ts b/src/mcp/index.ts index 63b8955..703c028 100644 --- a/src/mcp/index.ts +++ b/src/mcp/index.ts @@ -590,6 +590,57 @@ server.tool( } ); +// --- memory_index --- +import { indexProject } from '../core/indexer'; + +server.tool( + 'memory_index', + 'Index a codebase to create component nodes. Scans files, extracts exports/imports, and maps relationships.', + { + path: z.string().optional().describe('Path to index (default: current directory)'), + update: z.boolean().optional().describe('Only update changed files (incremental)'), + language: z.string().optional().describe('Only index specific language (ts, js, py)'), + maxDepth: z.number().optional().describe('Maximum directory depth (default: 10)'), + }, + async ({ path: inputPath, update, language, maxDepth }) => { + const result = await indexProject(inputPath || '.', { + update, + language, + maxDepth, + }); + return { content: [{ type: 'text' as const, text: serialize(result) }] }; + } +); + +// --- memory_components --- +server.tool( + 'memory_components', + 'List indexed components for a project', + { + project: z.string().optional().describe('Project name to filter by'), + limit: z.number().optional().describe('Max results (default: 50)'), + }, + async ({ project, limit }) => { + const tags = project ? [project, 'indexed'] : ['indexed']; + const components = listNodes({ kind: 'component' as NodeKind, tags, limit: limit || 50 }); + return { + content: [{ + type: 'text' as const, + text: serialize({ + count: components.length, + components: components.map(c => ({ + id: c.id, + title: c.title, + filePath: c.metadata?.filePath, + exports: (c.metadata?.exports as string[])?.length || 0, + loc: c.metadata?.loc, + })), + }), + }], + }; + } +); + async function main() { const transport = new StdioServerTransport(); await server.connect(transport);