diff --git a/USAGE.md b/USAGE.md new file mode 100644 index 0000000..02e746e --- /dev/null +++ b/USAGE.md @@ -0,0 +1,224 @@ +# Cortex — AI Project Memory & Knowledge Graph + +## Overview + +Cortex is a CLI tool and web server for storing, linking, and searching project knowledge as a graph of typed nodes and edges. It supports hybrid search (BM25 + vector similarity + freshness), auto-decay of stale nodes, and a web portal for visualization. + +Data is stored in `.memory/cortex.db` (SQLite) in the current working directory. + +--- + +## CLI Reference + +All commands are invoked as `memory `. + +### `memory add ` + +Add a node to the knowledge graph. + +| Argument / Option | Required | Description | +|---|---|---| +| `` | yes | Node kind: `memory`, `component`, `task`, `decision` | +| `-t, --title ` | yes | Node title | +| `-c, --content <content>` | no | Node content/description | +| `--tags <tags>` | no | Comma-separated tags | +| `--status <status>` | no | Status (e.g. `todo`, `doing`, `done`, `active`, `deprecated`) | + +```bash +memory add memory -t "Auth flow design" -c "OAuth2 PKCE flow for SPA" --tags auth,security --status active +memory add task -t "Implement login page" --status todo +memory add decision -t "Use PostgreSQL" -c "Chose Postgres over MySQL for JSON support" +memory add component -t "UserService" -c "Handles user CRUD operations" --tags backend +``` + +### `memory query <text>` + +Search the knowledge graph using natural language. Uses hybrid BM25 + vector + freshness scoring. + +| Argument / Option | Required | Description | +|---|---|---| +| `<text>` | yes | Natural language search query | +| `--kind <kind>` | no | Filter by node kind | +| `--limit <n>` | no | Max results (default: 10) | +| `--format <fmt>` | no | Output format: `text` or `json` (default: `text`) | + +```bash +memory query "authentication" +memory query "database decisions" --kind decision +memory query "user service" --limit 5 --format json +``` + +### `memory show <id>` + +Show a node's full details and its connections. The `<id>` can be a full UUID or a unique prefix. + +Each call to `show` updates the node's `lastAccessedAt` timestamp, which affects freshness scoring. + +| Argument / Option | Required | Description | +|---|---|---| +| `<id>` | yes | Node ID or unique prefix | +| `--format <fmt>` | no | Output format: `text` or `json` (default: `text`) | + +```bash +memory show abc123 +memory show abc123 --format json +``` + +### `memory list` + +List nodes in the knowledge graph. + +| Option | Required | Description | +|---|---|---| +| `--kind <kind>` | no | Filter by kind | +| `--status <status>` | no | Filter by status | +| `--tags <tags>` | no | Comma-separated tags to filter | +| `--limit <n>` | no | Max results | +| `--stale` | no | Include stale/decayed nodes | +| `--format <fmt>` | no | Output format: `text` or `json` (default: `text`) | + +```bash +memory list +memory list --kind task --status todo +memory list --tags auth,security +memory list --stale +memory list --format json +``` + +### `memory update <id>` + +Update an existing node's fields. + +| Argument / Option | Required | Description | +|---|---|---| +| `<id>` | yes | Node ID or unique prefix | +| `-t, --title <title>` | no | New title | +| `-c, --content <content>` | no | New content | +| `--status <status>` | no | New status | +| `--tags <tags>` | no | Replace tags (comma-separated) | +| `--stale` | no | Mark as stale | + +```bash +memory update abc123 --status done +memory update abc123 -t "Updated title" -c "New content" +memory update abc123 --tags newtag1,newtag2 +``` + +### `memory remove <id>` + +Remove a node. Default is soft delete (marks as stale). Use `--hard` for permanent deletion. + +| Argument / Option | Required | Description | +|---|---|---| +| `<id>` | yes | Node ID or unique prefix | +| `--hard` | no | Permanently delete instead of marking stale | + +```bash +memory remove abc123 +memory remove abc123 --hard +``` + +### `memory link <fromId> <toId>` + +Create a directed edge between two nodes. + +| Argument / Option | Required | Description | +|---|---|---| +| `<fromId>` | yes | Source node ID or prefix | +| `<toId>` | yes | Target node ID or prefix | +| `--type <type>` | yes | Edge type: `depends_on`, `contains`, `implements`, `blocked_by`, `subtask_of`, `relates_to`, `supersedes`, `about` | + +```bash +memory link abc123 def456 --type depends_on +memory link abc123 def456 --type contains +``` + +### `memory graph [id]` + +Visualize the knowledge graph as an ASCII tree. Optionally root at a specific node. + +| Argument | Required | Description | +|---|---|---| +| `[id]` | no | Root node ID or prefix. Omit for full graph. | + +```bash +memory graph +memory graph abc123 +``` + +### `memory decay` + +Run auto-decay to mark old untouched nodes as stale. Nodes whose `lastAccessedAt` exceeds the threshold are marked stale and hidden from default listings and search. + +| Option | Required | Description | +|---|---|---| +| `--days <number>` | no | Max age in days before decay (default: 180) | + +```bash +memory decay +memory decay --days 90 +memory decay --days 0 # decay all nodes not accessed today +``` + +### `memory serve` + +Start the Cortex Portal web server. Auto-decay runs on startup and every 24 hours. + +| Option | Required | Description | +|---|---|---| +| `-p, --port <number>` | no | Port number (default: 3100) | + +```bash +memory serve +memory serve --port 8080 +``` + +--- + +## Node Kinds + +| Kind | Purpose | +|---|---| +| `memory` | General knowledge, notes, context | +| `component` | Code components, services, modules | +| `task` | Work items, todos | +| `decision` | Architectural or design decisions | + +## Edge Types + +| Type | Meaning | +|---|---| +| `depends_on` | Source depends on target | +| `contains` | Source contains target | +| `implements` | Source implements target | +| `blocked_by` | Source is blocked by target | +| `subtask_of` | Source is a subtask of target | +| `relates_to` | General relationship | +| `supersedes` | Source supersedes/replaces target | +| `about` | Source is about target | + +## Search Scoring + +Search combines three signals: + +- **BM25** (weight 0.25) — text relevance +- **Vector similarity** (weight 0.60) — semantic similarity via embeddings (requires Ollama) +- **Freshness** (weight 0.15) — exponential decay based on `lastAccessedAt` + +Freshness multiplier: `e^(-0.01 * ageDays)` — half-life ~69 days. Recently accessed nodes are boosted; old untouched nodes are penalized but not zeroed. + +## REST API + +The web server exposes these endpoints under `/api`: + +| Method | Path | Description | +|---|---|---| +| `GET` | `/api/nodes` | List nodes. Query params: `kind`, `status`, `tags`, `limit`, `includeStale` | +| `GET` | `/api/nodes/:id` | Get node + connections | +| `POST` | `/api/nodes` | Create node. Body: `{ kind, title, content?, status?, tags?, metadata? }` | +| `PATCH` | `/api/nodes/:id` | Update node. Body: `{ title?, content?, status?, tags?, metadata?, isStale? }` | +| `DELETE` | `/api/nodes/:id` | Delete node. Query param: `hard=true` for permanent delete | +| `POST` | `/api/edges` | Create edge. Body: `{ fromId, toId, type, metadata? }` | +| `DELETE` | `/api/edges/:id` | Delete edge | +| `GET` | `/api/graph` | Get full graph (nodes + edges) for visualization | +| `POST` | `/api/search` | Search. Body: `{ text, options?: { kind?, tags?, limit?, includeStale? } }` | diff --git a/portal/src/types.ts b/portal/src/types.ts index 6f8b4a0..7d96f70 100644 --- a/portal/src/types.ts +++ b/portal/src/types.ts @@ -14,6 +14,7 @@ export interface CortexNode { metadata: Record<string, any>; createdAt: number; updatedAt: number; + lastAccessedAt?: number; isStale?: boolean; } diff --git a/src/cli/commands/decay.ts b/src/cli/commands/decay.ts new file mode 100644 index 0000000..9cf0b6e --- /dev/null +++ b/src/cli/commands/decay.ts @@ -0,0 +1,12 @@ +import { Command } from 'commander'; +import chalk from 'chalk'; +import { decayStaleNodes } from '../../core/decay'; + +export const decayCommand = new Command('decay') + .description('Mark old untouched nodes as stale') + .option('--days <number>', 'Max age in days before decay', '180') + .action((opts) => { + const days = parseInt(opts.days, 10); + const count = decayStaleNodes(days); + console.log(chalk.green(`✓ Decayed ${count} node(s) older than ${days} days.`)); + }); diff --git a/src/cli/index.ts b/src/cli/index.ts index ce5d5f5..dadfa67 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -9,6 +9,7 @@ import { updateCommand } from './commands/update'; import { removeCommand } from './commands/remove'; import { graphCommand } from './commands/graph'; import { serveCommand } from './commands/serve'; +import { decayCommand } from './commands/decay'; import { closeDb } from '../core/db'; const program = new Command(); @@ -27,6 +28,7 @@ program.addCommand(updateCommand); program.addCommand(removeCommand); program.addCommand(graphCommand); program.addCommand(serveCommand); +program.addCommand(decayCommand); program.hook('postAction', () => { closeDb(); diff --git a/src/core/db.ts b/src/core/db.ts index 849253f..c0e2d49 100644 --- a/src/core/db.ts +++ b/src/core/db.ts @@ -61,6 +61,13 @@ export function getDb(): Database.Database { _db.pragma('foreign_keys = ON'); _db.exec(SCHEMA); + // Migration: add last_accessed_at column + const cols = _db.prepare("PRAGMA table_info(nodes)").all() as any[]; + if (!cols.some((c: any) => c.name === 'last_accessed_at')) { + _db.exec('ALTER TABLE nodes ADD COLUMN last_accessed_at INTEGER'); + _db.exec('UPDATE nodes SET last_accessed_at = updated_at WHERE last_accessed_at IS NULL'); + } + return _db; } diff --git a/src/core/decay.ts b/src/core/decay.ts new file mode 100644 index 0000000..c916f9d --- /dev/null +++ b/src/core/decay.ts @@ -0,0 +1,10 @@ +import { getDb } from './db'; + +export function decayStaleNodes(maxAgeDays: number = 180): number { + const db = getDb(); + const threshold = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000; + const result = db.prepare( + 'UPDATE nodes SET is_stale = 1 WHERE is_stale = 0 AND (last_accessed_at IS NULL OR last_accessed_at < ?)' + ).run(threshold); + return result.changes; +} diff --git a/src/core/graph.ts b/src/core/graph.ts index ce0af5f..b66494e 100644 --- a/src/core/graph.ts +++ b/src/core/graph.ts @@ -46,6 +46,7 @@ export function getConnections(nodeId: string): { incoming: (Edge & { node: Node embedding: null, createdAt: row.n_created, updatedAt: row.n_updated, + lastAccessedAt: row.last_accessed_at ?? row.n_updated, isStale: !!row.is_stale, }, }); diff --git a/src/core/search/freshness.ts b/src/core/search/freshness.ts new file mode 100644 index 0000000..8b13308 --- /dev/null +++ b/src/core/search/freshness.ts @@ -0,0 +1,6 @@ +const DECAY_RATE = 0.01; // ~69 day half-life + +export function freshnessMultiplier(lastAccessedAt: number, now: number = Date.now()): number { + const ageDays = (now - lastAccessedAt) / (1000 * 60 * 60 * 24); + return Math.exp(-DECAY_RATE * Math.max(0, ageDays)); +} diff --git a/src/core/search/index.ts b/src/core/search/index.ts index aeb0271..57f9bf4 100644 --- a/src/core/search/index.ts +++ b/src/core/search/index.ts @@ -2,9 +2,11 @@ import { Node, SearchResult, QueryOptions } from '../../types'; import { bm25Search } from './bm25'; import { cosineSimilarity } from './vector'; import { getEmbedding, isOllamaAvailable } from './ollama'; +import { freshnessMultiplier } from './freshness'; -const VECTOR_WEIGHT = 0.7; -const BM25_WEIGHT = 0.3; +const VECTOR_WEIGHT = 0.6; +const BM25_WEIGHT = 0.25; +const FRESHNESS_WEIGHT = 0.15; function deserializeEmbedding(blob: Buffer | null): number[] | null { if (!blob || blob.length === 0) return null; @@ -66,6 +68,13 @@ export async function hybridSearch( if (node) results.push({ node, score }); } + // Apply freshness multiplier + const now = Date.now(); + for (const r of results) { + const freshness = freshnessMultiplier(r.node.lastAccessedAt, now); + r.score = r.score * (1 - FRESHNESS_WEIGHT + FRESHNESS_WEIGHT * freshness); + } + results.sort((a, b) => b.score - a.score); return results.slice(0, limit); } diff --git a/src/core/store.ts b/src/core/store.ts index 499f1f1..04ebb33 100644 --- a/src/core/store.ts +++ b/src/core/store.ts @@ -16,6 +16,7 @@ function rowToNode(row: any): Node { embedding: row.embedding ? deserializeEmbedding(row.embedding) : null, createdAt: row.created_at, updatedAt: row.updated_at, + lastAccessedAt: row.last_accessed_at ?? row.updated_at, isStale: !!row.is_stale, }; } @@ -36,13 +37,13 @@ export async function addNode(input: AddNodeInput): Promise<Node> { const embedding = await getEmbedding(`${input.title} ${content}`); db.prepare(` - INSERT INTO nodes (id, kind, title, content, status, tags, metadata, embedding, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO nodes (id, kind, title, content, status, tags, metadata, embedding, created_at, updated_at, last_accessed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `).run( id, input.kind, input.title, content, input.status ?? null, JSON.stringify(tags), JSON.stringify(metadata), embedding ? serializeEmbedding(embedding) : null, - now, now + now, now, now ); // Insert tags @@ -53,14 +54,16 @@ export async function addNode(input: AddNodeInput): Promise<Node> { return { id, kind: input.kind, title: input.title, content, status: input.status, - tags, metadata, embedding, createdAt: now, updatedAt: now, isStale: false, + tags, metadata, embedding, createdAt: now, updatedAt: now, lastAccessedAt: now, isStale: false, }; } export function getNode(id: string): Node | null { const db = getDb(); const row = db.prepare('SELECT * FROM nodes WHERE id = ?').get(id) as any; - return row ? rowToNode(row) : null; + if (!row) return null; + db.prepare('UPDATE nodes SET last_accessed_at = ? WHERE id = ?').run(Date.now(), id); + return rowToNode(row); } export function findNodeByPrefix(prefix: string): Node | null { diff --git a/src/server/index.ts b/src/server/index.ts index c5e1b1c..b02c606 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -3,6 +3,7 @@ import cors from 'cors'; import path from 'path'; import routes from './routes'; import { closeDb } from '../core/db'; +import { decayStaleNodes } from '../core/decay'; const app = express(); const PORT = parseInt(process.env.PORT || '3100'); @@ -18,11 +19,17 @@ app.get('{*path}', (_req, res) => { res.sendFile(path.join(portalDist, 'index.html')); }); +// Run auto-decay on startup and daily +decayStaleNodes(); +const DECAY_INTERVAL = 24 * 60 * 60 * 1000; +const decayTimer = setInterval(() => decayStaleNodes(), DECAY_INTERVAL); + const server = app.listen(PORT, () => { console.log(`Cortex Portal running at http://localhost:${PORT}`); }); process.on('SIGINT', () => { + clearInterval(decayTimer); closeDb(); server.close(); process.exit(0); diff --git a/src/types.ts b/src/types.ts index 4706b86..0ae3317 100644 --- a/src/types.ts +++ b/src/types.ts @@ -21,6 +21,7 @@ export interface Node { embedding: number[] | null; createdAt: number; updatedAt: number; + lastAccessedAt: number; isStale: boolean; }