Initial commit: Cortex — AI project memory & knowledge graph

SQLite-backed knowledge graph with CLI interface. Supports nodes (memory, component, task, decision) connected by typed edges, with hybrid search (BM25 + Ollama embeddings).
2026-02-02 14:53:26 +01:00
commit 21107443a7
21 changed files with 1624 additions and 0 deletions
--- a/src/core/db.ts
+++ b/src/core/db.ts
@@ -0,0 +1,72 @@
+import Database from 'better-sqlite3';
+import path from 'path';
+import fs from 'fs';
+
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS nodes (
+  id TEXT PRIMARY KEY,
+  kind TEXT NOT NULL,
+  title TEXT NOT NULL,
+  content TEXT NOT NULL DEFAULT '',
+  status TEXT,
+  tags TEXT DEFAULT '[]',
+  metadata TEXT DEFAULT '{}',
+  embedding BLOB,
+  created_at INTEGER NOT NULL,
+  updated_at INTEGER NOT NULL,
+  is_stale INTEGER DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS edges (
+  id TEXT PRIMARY KEY,
+  from_id TEXT NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
+  to_id TEXT NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
+  type TEXT NOT NULL,
+  metadata TEXT DEFAULT '{}',
+  created_at INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS node_tags (
+  node_id TEXT NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
+  tag TEXT NOT NULL,
+  PRIMARY KEY (node_id, tag)
+);
+
+CREATE INDEX IF NOT EXISTS idx_nodes_kind ON nodes(kind);
+CREATE INDEX IF NOT EXISTS idx_nodes_status ON nodes(status);
+CREATE INDEX IF NOT EXISTS idx_nodes_created ON nodes(created_at DESC);
+CREATE INDEX IF NOT EXISTS idx_nodes_stale ON nodes(is_stale);
+CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_id);
+CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_id);
+CREATE INDEX IF NOT EXISTS idx_edges_type ON edges(type);
+CREATE INDEX IF NOT EXISTS idx_tags_tag ON node_tags(tag);
+`;
+
+let _db: Database.Database | null = null;
+
+export function getMemoryDir(): string {
+  return path.join(process.cwd(), '.memory');
+}
+
+export function getDb(): Database.Database {
+  if (_db) return _db;
+
+  const dir = getMemoryDir();
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+
+  _db = new Database(path.join(dir, 'cortex.db'));
+  _db.pragma('journal_mode = WAL');
+  _db.pragma('foreign_keys = ON');
+  _db.exec(SCHEMA);
+
+  return _db;
+}
+
+export function closeDb(): void {
+  if (_db) {
+    _db.close();
+    _db = null;
+  }
+}
--- a/src/core/graph.ts
+++ b/src/core/graph.ts
@@ -0,0 +1,131 @@
+import { getDb } from './db';
+import { Node, Edge } from '../types';
+
+interface TreeNode {
+  id: string;
+  title: string;
+  kind: string;
+  edgeType?: string;
+  children: TreeNode[];
+}
+
+function rowToEdge(row: any): Edge {
+  return {
+    id: row.id,
+    fromId: row.from_id,
+    toId: row.to_id,
+    type: row.type,
+    metadata: JSON.parse(row.metadata || '{}'),
+    createdAt: row.created_at,
+  };
+}
+
+export function getConnections(nodeId: string): { incoming: (Edge & { node: Node })[], outgoing: (Edge & { node: Node })[] } {
+  const db = getDb();
+
+  const outRows = db.prepare(`
+    SELECT e.*, n.id as n_id, n.kind, n.title, n.content, n.status, n.tags, n.metadata as n_metadata, n.created_at as n_created, n.updated_at as n_updated, n.is_stale
+    FROM edges e JOIN nodes n ON e.to_id = n.id WHERE e.from_id = ?
+  `).all(nodeId) as any[];
+
+  const inRows = db.prepare(`
+    SELECT e.*, n.id as n_id, n.kind, n.title, n.content, n.status, n.tags, n.metadata as n_metadata, n.created_at as n_created, n.updated_at as n_updated, n.is_stale
+    FROM edges e JOIN nodes n ON e.from_id = n.id WHERE e.to_id = ?
+  `).all(nodeId) as any[];
+
+  const mapRow = (row: any) => ({
+    ...rowToEdge(row),
+    node: {
+      id: row.n_id,
+      kind: row.kind,
+      title: row.title,
+      content: row.content,
+      status: row.status,
+      tags: JSON.parse(row.tags || '[]'),
+      metadata: JSON.parse(row.n_metadata || '{}'),
+      embedding: null,
+      createdAt: row.n_created,
+      updatedAt: row.n_updated,
+      isStale: !!row.is_stale,
+    },
+  });
+
+  return {
+    outgoing: outRows.map(mapRow),
+    incoming: inRows.map(mapRow),
+  };
+}
+
+export function buildTree(rootId?: string): TreeNode[] {
+  const db = getDb();
+  const edges = db.prepare('SELECT * FROM edges').all() as any[];
+  const nodes = db.prepare('SELECT id, title, kind FROM nodes WHERE is_stale = 0').all() as any[];
+
+  const nodeMap = new Map(nodes.map((n: any) => [n.id, n]));
+
+  // Build adjacency: from -> [{toId, type}]
+  const children = new Map<string, { toId: string; type: string }[]>();
+  const hasParent = new Set<string>();
+
+  for (const e of edges) {
+    const list = children.get(e.from_id) || [];
+    list.push({ toId: e.to_id, type: e.type });
+    children.set(e.from_id, list);
+    hasParent.add(e.to_id);
+  }
+
+  function build(id: string, visited: Set<string>): TreeNode | null {
+    if (visited.has(id)) return null;
+    const node = nodeMap.get(id);
+    if (!node) return null;
+
+    visited.add(id);
+    const kids = (children.get(id) || [])
+      .map(c => {
+        const child = build(c.toId, visited);
+        if (child) child.edgeType = c.type;
+        return child;
+      })
+      .filter(Boolean) as TreeNode[];
+
+    return { id: node.id, title: node.title, kind: node.kind, children: kids };
+  }
+
+  if (rootId) {
+    const tree = build(rootId, new Set());
+    return tree ? [tree] : [];
+  }
+
+  // All roots: nodes with no incoming edges
+  const roots = nodes.filter((n: any) => !hasParent.has(n.id));
+  // Also include orphans (no edges at all)
+  const result: TreeNode[] = [];
+  const visited = new Set<string>();
+  for (const r of roots) {
+    const tree = build(r.id, visited);
+    if (tree) result.push(tree);
+  }
+  // Add any remaining unvisited nodes as standalone
+  for (const n of nodes) {
+    if (!visited.has(n.id)) {
+      result.push({ id: n.id, title: n.title, kind: n.kind, children: [] });
+    }
+  }
+  return result;
+}
+
+export function renderTree(trees: TreeNode[], indent: string = ''): string {
+  const lines: string[] = [];
+  for (let i = 0; i < trees.length; i++) {
+    const t = trees[i];
+    const isLast = i === trees.length - 1;
+    const prefix = indent + (indent ? (isLast ? '└── ' : '├── ') : '');
+    const edgeLabel = t.edgeType ? ` -[${t.edgeType}]` : '';
+    lines.push(`${prefix}[${t.kind}] ${t.title} (${t.id.slice(0, 8)})${edgeLabel}`);
+    const childIndent = indent + (indent ? (isLast ? '    ' : '│   ') : '  ');
+    if (t.children.length > 0) {
+      lines.push(renderTree(t.children, childIndent));
+    }
+  }
+  return lines.join('\n');
+}
--- a/src/core/search/bm25.ts
+++ b/src/core/search/bm25.ts
@@ -0,0 +1,57 @@
+import { Node, SearchResult } from '../../types';
+
+const K1 = 1.2;
+const B = 0.75;
+
+function tokenize(text: string): string[] {
+  return text.toLowerCase().replace(/[^\w\s]/g, ' ').split(/\s+/).filter(t => t.length > 1);
+}
+
+export function bm25Search(nodes: Node[], query: string, limit: number = 10): SearchResult[] {
+  const queryTokens = tokenize(query);
+  if (queryTokens.length === 0) return [];
+
+  // Build corpus stats
+  const N = nodes.length;
+  const docs = nodes.map(n => tokenize(`${n.title} ${n.content}`));
+  const avgDl = docs.reduce((s, d) => s + d.length, 0) / (N || 1);
+
+  // Document frequency for each query term
+  const df: Record<string, number> = {};
+  for (const token of queryTokens) {
+    df[token] = 0;
+    for (const doc of docs) {
+      if (doc.includes(token)) df[token]++;
+    }
+  }
+
+  // Score each document
+  const scored: SearchResult[] = [];
+  for (let i = 0; i < N; i++) {
+    const doc = docs[i];
+    const dl = doc.length;
+    let score = 0;
+
+    // Term frequencies in this doc
+    const tf: Record<string, number> = {};
+    for (const token of doc) {
+      tf[token] = (tf[token] || 0) + 1;
+    }
+
+    for (const token of queryTokens) {
+      const termFreq = tf[token] || 0;
+      if (termFreq === 0) continue;
+
+      const idf = Math.log((N - (df[token] || 0) + 0.5) / ((df[token] || 0) + 0.5) + 1);
+      const tfNorm = (termFreq * (K1 + 1)) / (termFreq + K1 * (1 - B + B * dl / avgDl));
+      score += idf * tfNorm;
+    }
+
+    if (score > 0) {
+      scored.push({ node: nodes[i], score });
+    }
+  }
+
+  scored.sort((a, b) => b.score - a.score);
+  return scored.slice(0, limit);
+}
--- a/src/core/search/index.ts
+++ b/src/core/search/index.ts
@@ -0,0 +1,73 @@
+import { Node, SearchResult, QueryOptions } from '../../types';
+import { bm25Search } from './bm25';
+import { cosineSimilarity } from './vector';
+import { getEmbedding, isOllamaAvailable } from './ollama';
+
+const VECTOR_WEIGHT = 0.7;
+const BM25_WEIGHT = 0.3;
+
+function deserializeEmbedding(blob: Buffer | null): number[] | null {
+  if (!blob || blob.length === 0) return null;
+  const float32 = new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
+  return Array.from(float32);
+}
+
+export async function hybridSearch(
+  nodes: Node[],
+  query: string,
+  options: QueryOptions = {}
+): Promise<SearchResult[]> {
+  const limit = options.limit ?? 10;
+
+  // Filter stale unless requested
+  let candidates = options.includeStale ? nodes : nodes.filter(n => !n.isStale);
+  if (options.kind) candidates = candidates.filter(n => n.kind === options.kind);
+  if (options.tags?.length) {
+    candidates = candidates.filter(n => options.tags!.some(t => n.tags.includes(t)));
+  }
+
+  const bm25Results = bm25Search(candidates, query, limit);
+
+  if (!(await isOllamaAvailable())) {
+    return bm25Results;
+  }
+
+  // Try vector search
+  const queryEmbedding = await getEmbedding(query);
+  if (!queryEmbedding) return bm25Results;
+
+  // Score all candidates with embeddings
+  const vectorScored: SearchResult[] = [];
+  for (const node of candidates) {
+    if (!node.embedding) continue;
+    const sim = cosineSimilarity(queryEmbedding, node.embedding);
+    if (sim > 0) vectorScored.push({ node, score: sim });
+  }
+
+  if (vectorScored.length === 0) return bm25Results;
+
+  // Normalize scores
+  const maxBm25 = Math.max(...bm25Results.map(r => r.score), 1e-10);
+  const maxVector = Math.max(...vectorScored.map(r => r.score), 1e-10);
+
+  // Merge by node ID
+  const merged = new Map<string, number>();
+  for (const r of bm25Results) {
+    merged.set(r.node.id, (merged.get(r.node.id) ?? 0) + BM25_WEIGHT * (r.score / maxBm25));
+  }
+  for (const r of vectorScored) {
+    merged.set(r.node.id, (merged.get(r.node.id) ?? 0) + VECTOR_WEIGHT * (r.score / maxVector));
+  }
+
+  const nodeMap = new Map(candidates.map(n => [n.id, n]));
+  const results: SearchResult[] = [];
+  for (const [id, score] of merged) {
+    const node = nodeMap.get(id);
+    if (node) results.push({ node, score });
+  }
+
+  results.sort((a, b) => b.score - a.score);
+  return results.slice(0, limit);
+}
+
+export { deserializeEmbedding };
--- a/src/core/search/ollama.ts
+++ b/src/core/search/ollama.ts
@@ -0,0 +1,38 @@
+const OLLAMA_URL = 'http://localhost:11434';
+const MODEL = 'nomic-embed-text';
+
+let _available: boolean | null = null;
+
+export async function isOllamaAvailable(): Promise<boolean> {
+  if (_available !== null) return _available;
+  try {
+    const res = await fetch(`${OLLAMA_URL}/api/tags`, { signal: AbortSignal.timeout(2000) });
+    _available = res.ok;
+  } catch {
+    _available = false;
+  }
+  return _available;
+}
+
+export async function getEmbedding(text: string): Promise<number[] | null> {
+  if (!(await isOllamaAvailable())) return null;
+
+  try {
+    const res = await fetch(`${OLLAMA_URL}/api/embeddings`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ model: MODEL, prompt: text }),
+      signal: AbortSignal.timeout(30000),
+    });
+
+    if (!res.ok) return null;
+    const data = await res.json() as { embedding?: number[] };
+    return data.embedding ?? null;
+  } catch {
+    return null;
+  }
+}
+
+export async function getEmbeddings(texts: string[]): Promise<(number[] | null)[]> {
+  return Promise.all(texts.map(getEmbedding));
+}
--- a/src/core/search/vector.ts
+++ b/src/core/search/vector.ts
@@ -0,0 +1,13 @@
+export function cosineSimilarity(a: number[], b: number[]): number {
+  if (a.length !== b.length || a.length === 0) return 0;
+
+  let dot = 0, normA = 0, normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  return denom === 0 ? 0 : dot / denom;
+}
--- a/src/core/store.ts
+++ b/src/core/store.ts
@@ -0,0 +1,184 @@
+import { v4 as uuid } from 'uuid';
+import { getDb } from './db';
+import { Node, Edge, AddNodeInput, UpdateNodeInput, ListOptions, QueryOptions, SearchResult, EdgeType } from '../types';
+import { hybridSearch, deserializeEmbedding } from './search/index';
+import { getEmbedding } from './search/ollama';
+
+function rowToNode(row: any): Node {
+  return {
+    id: row.id,
+    kind: row.kind,
+    title: row.title,
+    content: row.content,
+    status: row.status ?? undefined,
+    tags: JSON.parse(row.tags || '[]'),
+    metadata: JSON.parse(row.metadata || '{}'),
+    embedding: row.embedding ? deserializeEmbedding(row.embedding) : null,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at,
+    isStale: !!row.is_stale,
+  };
+}
+
+function serializeEmbedding(embedding: number[]): Buffer {
+  return Buffer.from(new Float32Array(embedding).buffer);
+}
+
+export async function addNode(input: AddNodeInput): Promise<Node> {
+  const db = getDb();
+  const id = uuid();
+  const now = Date.now();
+  const tags = input.tags ?? [];
+  const metadata = input.metadata ?? {};
+  const content = input.content ?? '';
+
+  // Try to get embedding
+  const embedding = await getEmbedding(`${input.title} ${content}`);
+
+  db.prepare(`
+    INSERT INTO nodes (id, kind, title, content, status, tags, metadata, embedding, created_at, updated_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(
+    id, input.kind, input.title, content, input.status ?? null,
+    JSON.stringify(tags), JSON.stringify(metadata),
+    embedding ? serializeEmbedding(embedding) : null,
+    now, now
+  );
+
+  // Insert tags
+  const insertTag = db.prepare('INSERT OR IGNORE INTO node_tags (node_id, tag) VALUES (?, ?)');
+  for (const tag of tags) {
+    insertTag.run(id, tag);
+  }
+
+  return {
+    id, kind: input.kind, title: input.title, content, status: input.status,
+    tags, metadata, embedding, createdAt: now, updatedAt: now, isStale: false,
+  };
+}
+
+export function getNode(id: string): Node | null {
+  const db = getDb();
+  const row = db.prepare('SELECT * FROM nodes WHERE id = ?').get(id) as any;
+  return row ? rowToNode(row) : null;
+}
+
+export function findNodeByPrefix(prefix: string): Node | null {
+  const db = getDb();
+  const row = db.prepare('SELECT * FROM nodes WHERE id LIKE ?').get(`${prefix}%`) as any;
+  return row ? rowToNode(row) : null;
+}
+
+export function listNodes(options: ListOptions = {}): Node[] {
+  const db = getDb();
+  const conditions: string[] = [];
+  const params: any[] = [];
+
+  if (!options.includeStale) {
+    conditions.push('is_stale = 0');
+  }
+  if (options.kind) {
+    conditions.push('kind = ?');
+    params.push(options.kind);
+  }
+  if (options.status) {
+    conditions.push('status = ?');
+    params.push(options.status);
+  }
+
+  let sql = 'SELECT * FROM nodes';
+  if (conditions.length) sql += ' WHERE ' + conditions.join(' AND ');
+  sql += ' ORDER BY created_at DESC';
+  if (options.limit) {
+    sql += ' LIMIT ?';
+    params.push(options.limit);
+  }
+
+  let nodes = (db.prepare(sql).all(...params) as any[]).map(rowToNode);
+
+  if (options.tags?.length) {
+    nodes = nodes.filter(n => options.tags!.some(t => n.tags.includes(t)));
+  }
+
+  return nodes;
+}
+
+export async function updateNode(id: string, input: UpdateNodeInput): Promise<Node | null> {
+  const db = getDb();
+  const existing = getNode(id);
+  if (!existing) return null;
+
+  const now = Date.now();
+  const sets: string[] = ['updated_at = ?'];
+  const params: any[] = [now];
+
+  if (input.title !== undefined) { sets.push('title = ?'); params.push(input.title); }
+  if (input.content !== undefined) { sets.push('content = ?'); params.push(input.content); }
+  if (input.status !== undefined) { sets.push('status = ?'); params.push(input.status); }
+  if (input.isStale !== undefined) { sets.push('is_stale = ?'); params.push(input.isStale ? 1 : 0); }
+  if (input.tags !== undefined) { sets.push('tags = ?'); params.push(JSON.stringify(input.tags)); }
+  if (input.metadata !== undefined) {
+    const merged = { ...existing.metadata, ...input.metadata };
+    sets.push('metadata = ?');
+    params.push(JSON.stringify(merged));
+  }
+
+  // Re-embed if title or content changed
+  if (input.title !== undefined || input.content !== undefined) {
+    const newTitle = input.title ?? existing.title;
+    const newContent = input.content ?? existing.content;
+    const embedding = await getEmbedding(`${newTitle} ${newContent}`);
+    if (embedding) {
+      sets.push('embedding = ?');
+      params.push(serializeEmbedding(embedding));
+    }
+  }
+
+  params.push(id);
+  db.prepare(`UPDATE nodes SET ${sets.join(', ')} WHERE id = ?`).run(...params);
+
+  // Update tags if changed
+  if (input.tags !== undefined) {
+    db.prepare('DELETE FROM node_tags WHERE node_id = ?').run(id);
+    const insertTag = db.prepare('INSERT OR IGNORE INTO node_tags (node_id, tag) VALUES (?, ?)');
+    for (const tag of input.tags) {
+      insertTag.run(id, tag);
+    }
+  }
+
+  return getNode(id);
+}
+
+export function removeNode(id: string, hard: boolean = false): boolean {
+  const db = getDb();
+  if (hard) {
+    const result = db.prepare('DELETE FROM nodes WHERE id = ?').run(id);
+    return result.changes > 0;
+  } else {
+    const result = db.prepare('UPDATE nodes SET is_stale = 1, updated_at = ? WHERE id = ?').run(Date.now(), id);
+    return result.changes > 0;
+  }
+}
+
+export function addEdge(fromId: string, toId: string, type: EdgeType, metadata: Record<string, any> = {}): Edge {
+  const db = getDb();
+  const id = uuid();
+  const now = Date.now();
+
+  db.prepare(`
+    INSERT INTO edges (id, from_id, to_id, type, metadata, created_at) VALUES (?, ?, ?, ?, ?, ?)
+  `).run(id, fromId, toId, type, JSON.stringify(metadata), now);
+
+  return { id, fromId, toId, type, metadata, createdAt: now };
+}
+
+export function removeEdge(id: string): boolean {
+  const db = getDb();
+  const result = db.prepare('DELETE FROM edges WHERE id = ?').run(id);
+  return result.changes > 0;
+}
+
+export async function query(text: string, options: QueryOptions = {}): Promise<SearchResult[]> {
+  const nodes = listNodes({ kind: options.kind, tags: options.tags, includeStale: options.includeStale });
+  return hybridSearch(nodes, text, options);
+}