Add development plan with 13 milestone specifications
- docs/plan.md: Master roadmap with phases and priorities - docs/milestones/01-13: Detailed specs for each feature - Updated CLAUDE.md with plan references and build commands Milestones cover: - Phase 1: Temporal versioning, auto-capture, context injection, codebase indexing - Phase 2: Daily journal, content ingestion, graph visualization, import/export - Phase 3: Multi-graph, smart retrieval, TUI dashboard, browser extension, shell completions
This commit is contained in:
284
docs/milestones/08-import-export.md
Normal file
284
docs/milestones/08-import-export.md
Normal file
@@ -0,0 +1,284 @@
|
||||
# Milestone 8: Import/Export
|
||||
|
||||
## Overview
|
||||
|
||||
Import from and export to popular formats: Obsidian vaults, Markdown folders, JSON-LD, and more. Never be locked into Cortex.
|
||||
|
||||
## Motivation
|
||||
|
||||
- Users have existing knowledge in other tools
|
||||
- Portability prevents lock-in
|
||||
- Enables backup and migration
|
||||
- Standard formats enable interoperability
|
||||
|
||||
## Features
|
||||
|
||||
### 8.1 Obsidian Import
|
||||
|
||||
```bash
|
||||
# Import Obsidian vault
|
||||
cortex import obsidian ~/vaults/notes
|
||||
|
||||
# Import with tag mapping
|
||||
cortex import obsidian ~/vaults/notes --map-tags
|
||||
|
||||
# Preview without importing
|
||||
cortex import obsidian ~/vaults/notes --dry-run
|
||||
```
|
||||
|
||||
Handles:
|
||||
- Wikilinks `[[Page Name]]` → edges
|
||||
- Tags `#tag` → node tags
|
||||
- YAML frontmatter → metadata
|
||||
- Folder structure → hierarchy
|
||||
|
||||
### 8.2 Markdown Folder Import
|
||||
|
||||
```bash
|
||||
# Import markdown folder
|
||||
cortex import markdown ./docs
|
||||
|
||||
# Respect folder hierarchy
|
||||
cortex import markdown ./docs --hierarchy
|
||||
|
||||
# Set kind for all imported nodes
|
||||
cortex import markdown ./docs --kind memory
|
||||
```
|
||||
|
||||
### 8.3 Markdown Export
|
||||
|
||||
```bash
|
||||
# Export to markdown folder
|
||||
cortex export-md ./output
|
||||
|
||||
# Export with frontmatter
|
||||
cortex export-md ./output --frontmatter
|
||||
|
||||
# Export specific nodes
|
||||
cortex export-md ./output --kind component
|
||||
```
|
||||
|
||||
### 8.4 JSON-LD Export
|
||||
|
||||
```bash
|
||||
# Export as JSON-LD (linked data)
|
||||
cortex export --format jsonld --output knowledge.jsonld
|
||||
|
||||
# Compatible with knowledge graph standards
|
||||
```
|
||||
|
||||
### 8.5 Full Backup/Restore
|
||||
|
||||
```bash
|
||||
# Backup entire database
|
||||
cortex backup ./backup-2024-01-15.cortex
|
||||
|
||||
# Restore from backup
|
||||
cortex restore ./backup-2024-01-15.cortex
|
||||
|
||||
# Auto-backup (cron-friendly)
|
||||
cortex backup --auto --keep 7 # Keep last 7 backups
|
||||
```
|
||||
|
||||
### 8.6 Notion Import (Future)
|
||||
|
||||
```bash
|
||||
# Import from Notion export
|
||||
cortex import notion ./notion-export
|
||||
```
|
||||
|
||||
## Implementation
|
||||
|
||||
### Obsidian Importer
|
||||
|
||||
```typescript
|
||||
// src/core/import/obsidian.ts
|
||||
export async function importObsidian(vaultPath: string, options: ImportOptions): Promise<ImportResult> {
|
||||
const files = await glob(`${vaultPath}/**/*.md`);
|
||||
const nodes: Map<string, Node> = new Map();
|
||||
const pendingLinks: Array<{ from: string; to: string; type: EdgeType }> = [];
|
||||
|
||||
// First pass: create nodes
|
||||
for (const file of files) {
|
||||
const content = await fs.readFile(file, 'utf-8');
|
||||
const { frontmatter, body } = parseFrontmatter(content);
|
||||
|
||||
const title = path.basename(file, '.md');
|
||||
const relativePath = path.relative(vaultPath, file);
|
||||
|
||||
const node = await addNode({
|
||||
kind: frontmatter.kind || 'memory',
|
||||
title,
|
||||
content: body,
|
||||
tags: extractTags(content, frontmatter.tags),
|
||||
status: frontmatter.status,
|
||||
metadata: {
|
||||
...frontmatter,
|
||||
importedFrom: 'obsidian',
|
||||
originalPath: relativePath,
|
||||
},
|
||||
});
|
||||
|
||||
nodes.set(title, node);
|
||||
}
|
||||
|
||||
// Second pass: create links
|
||||
for (const file of files) {
|
||||
const content = await fs.readFile(file, 'utf-8');
|
||||
const title = path.basename(file, '.md');
|
||||
const sourceNode = nodes.get(title);
|
||||
|
||||
// Find wikilinks [[Target]]
|
||||
const wikilinks = content.matchAll(/\[\[([^\]]+)\]\]/g);
|
||||
for (const match of wikilinks) {
|
||||
const targetTitle = match[1].split('|')[0]; // Handle aliases [[Page|Alias]]
|
||||
const targetNode = nodes.get(targetTitle);
|
||||
|
||||
if (sourceNode && targetNode) {
|
||||
addEdge(sourceNode.id, targetNode.id, 'relates_to');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Third pass: folder hierarchy
|
||||
if (options.hierarchy) {
|
||||
await createFolderHierarchy(vaultPath, nodes);
|
||||
}
|
||||
|
||||
return { imported: nodes.size };
|
||||
}
|
||||
```
|
||||
|
||||
### Markdown Exporter
|
||||
|
||||
```typescript
|
||||
// src/core/export/markdown.ts
|
||||
export async function exportMarkdown(outputDir: string, options: ExportOptions): Promise<void> {
|
||||
const nodes = await listNodes(options.filter);
|
||||
|
||||
await fs.mkdir(outputDir, { recursive: true });
|
||||
|
||||
for (const node of nodes) {
|
||||
const filename = sanitizeFilename(node.title) + '.md';
|
||||
const filepath = path.join(outputDir, filename);
|
||||
|
||||
let content = '';
|
||||
|
||||
// Add frontmatter
|
||||
if (options.frontmatter) {
|
||||
content += '---\n';
|
||||
content += `id: ${node.id}\n`;
|
||||
content += `kind: ${node.kind}\n`;
|
||||
content += `tags: [${node.tags.join(', ')}]\n`;
|
||||
if (node.status) content += `status: ${node.status}\n`;
|
||||
content += `created: ${new Date(node.createdAt).toISOString()}\n`;
|
||||
content += '---\n\n';
|
||||
}
|
||||
|
||||
// Add title
|
||||
content += `# ${node.title}\n\n`;
|
||||
|
||||
// Add content
|
||||
content += node.content || '';
|
||||
|
||||
// Add linked nodes as wikilinks
|
||||
const connections = getConnections(node.id);
|
||||
if (connections.outgoing.length > 0) {
|
||||
content += '\n\n## Related\n\n';
|
||||
for (const conn of connections.outgoing) {
|
||||
content += `- [[${conn.node.title}]] (${conn.type})\n`;
|
||||
}
|
||||
}
|
||||
|
||||
await fs.writeFile(filepath, content);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### JSON-LD Exporter
|
||||
|
||||
```typescript
|
||||
// src/core/export/jsonld.ts
|
||||
export async function exportJsonLd(options: ExportOptions): Promise<object> {
|
||||
const nodes = await listNodes(options.filter);
|
||||
const edges = await listEdges();
|
||||
|
||||
return {
|
||||
'@context': {
|
||||
'@vocab': 'https://schema.org/',
|
||||
'cortex': 'https://cortex.memory/',
|
||||
'relates_to': 'cortex:relatesTo',
|
||||
'contains': 'cortex:contains',
|
||||
'depends_on': 'cortex:dependsOn',
|
||||
},
|
||||
'@graph': nodes.map(node => ({
|
||||
'@id': `cortex:node/${node.id}`,
|
||||
'@type': kindToSchemaType(node.kind),
|
||||
'name': node.title,
|
||||
'description': node.content,
|
||||
'keywords': node.tags,
|
||||
'dateCreated': new Date(node.createdAt).toISOString(),
|
||||
...edgesToRelations(node.id, edges),
|
||||
})),
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
## CLI Commands
|
||||
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `cortex import obsidian <path>` | Import Obsidian vault |
|
||||
| `cortex import markdown <path>` | Import markdown folder |
|
||||
| `cortex import notion <path>` | Import Notion export |
|
||||
| `cortex export-md <path>` | Export to markdown |
|
||||
| `cortex export --format jsonld` | Export as JSON-LD |
|
||||
| `cortex backup <path>` | Backup database |
|
||||
| `cortex restore <path>` | Restore from backup |
|
||||
|
||||
## MCP Tools
|
||||
|
||||
```typescript
|
||||
memory_import // Import from file/path
|
||||
memory_export // Export to format
|
||||
memory_backup // Create backup
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
- [ ] Obsidian wikilinks become edges
|
||||
- [ ] Tags extracted correctly
|
||||
- [ ] Frontmatter preserved
|
||||
- [ ] Folder hierarchy respected
|
||||
- [ ] Markdown export produces valid files
|
||||
- [ ] JSON-LD validates against schema
|
||||
- [ ] Backup/restore preserves all data
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [ ] Import Obsidian vault with links intact
|
||||
- [ ] Export to Obsidian-compatible markdown
|
||||
- [ ] Round-trip preserves data
|
||||
- [ ] JSON-LD compatible with knowledge graph tools
|
||||
- [ ] Backup is single file, easily portable
|
||||
- [ ] Large vaults (10k+ files) import in <5 minutes
|
||||
|
||||
## Estimated Effort
|
||||
|
||||
- Obsidian importer: 6 hours
|
||||
- Markdown importer: 3 hours
|
||||
- Markdown exporter: 3 hours
|
||||
- JSON-LD exporter: 3 hours
|
||||
- Backup/restore: 2 hours
|
||||
- Testing: 4 hours
|
||||
- **Total: ~21 hours**
|
||||
|
||||
## Dependencies
|
||||
|
||||
- `gray-matter` for frontmatter parsing
|
||||
|
||||
## References
|
||||
|
||||
- [Obsidian file format](https://help.obsidian.md/Files+and+folders/Accepted+file+formats)
|
||||
- [JSON-LD specification](https://json-ld.org/)
|
||||
- [Schema.org](https://schema.org/)
|
||||
Reference in New Issue
Block a user