Add freshness scoring, auto-decay, and USAGE.md

Add lastAccessedAt timestamp to nodes with schema migration and backfill. Touch timestamp on read, apply exponential freshness decay (~69-day half-life) to search scoring alongside BM25 and vector weights. Add auto-decay that marks untouched nodes as stale after a configurable threshold, with CLI command and server-side daily interval. Include comprehensive USAGE.md documenting all CLI commands and REST API.
2026-02-02 22:07:06 +01:00
parent d1e3adcb3c
commit f1b59a2d1a
12 changed files with 290 additions and 7 deletions
--- a/USAGE.md
+++ b/USAGE.md
@@ -0,0 +1,224 @@
+# Cortex — AI Project Memory & Knowledge Graph
+
+## Overview
+
+Cortex is a CLI tool and web server for storing, linking, and searching project knowledge as a graph of typed nodes and edges. It supports hybrid search (BM25 + vector similarity + freshness), auto-decay of stale nodes, and a web portal for visualization.
+
+Data is stored in `.memory/cortex.db` (SQLite) in the current working directory.
+
+---
+
+## CLI Reference
+
+All commands are invoked as `memory <command>`.
+
+### `memory add <kind>`
+
+Add a node to the knowledge graph.
+
+| Argument / Option | Required | Description |
+|---|---|---|
+| `<kind>` | yes | Node kind: `memory`, `component`, `task`, `decision` |
+| `-t, --title <title>` | yes | Node title |
+| `-c, --content <content>` | no | Node content/description |
+| `--tags <tags>` | no | Comma-separated tags |
+| `--status <status>` | no | Status (e.g. `todo`, `doing`, `done`, `active`, `deprecated`) |
+
+```bash
+memory add memory -t "Auth flow design" -c "OAuth2 PKCE flow for SPA" --tags auth,security --status active
+memory add task -t "Implement login page" --status todo
+memory add decision -t "Use PostgreSQL" -c "Chose Postgres over MySQL for JSON support"
+memory add component -t "UserService" -c "Handles user CRUD operations" --tags backend
+```
+
+### `memory query <text>`
+
+Search the knowledge graph using natural language. Uses hybrid BM25 + vector + freshness scoring.
+
+| Argument / Option | Required | Description |
+|---|---|---|
+| `<text>` | yes | Natural language search query |
+| `--kind <kind>` | no | Filter by node kind |
+| `--limit <n>` | no | Max results (default: 10) |
+| `--format <fmt>` | no | Output format: `text` or `json` (default: `text`) |
+
+```bash
+memory query "authentication"
+memory query "database decisions" --kind decision
+memory query "user service" --limit 5 --format json
+```
+
+### `memory show <id>`
+
+Show a node's full details and its connections. The `<id>` can be a full UUID or a unique prefix.
+
+Each call to `show` updates the node's `lastAccessedAt` timestamp, which affects freshness scoring.
+
+| Argument / Option | Required | Description |
+|---|---|---|
+| `<id>` | yes | Node ID or unique prefix |
+| `--format <fmt>` | no | Output format: `text` or `json` (default: `text`) |
+
+```bash
+memory show abc123
+memory show abc123 --format json
+```
+
+### `memory list`
+
+List nodes in the knowledge graph.
+
+| Option | Required | Description |
+|---|---|---|
+| `--kind <kind>` | no | Filter by kind |
+| `--status <status>` | no | Filter by status |
+| `--tags <tags>` | no | Comma-separated tags to filter |
+| `--limit <n>` | no | Max results |
+| `--stale` | no | Include stale/decayed nodes |
+| `--format <fmt>` | no | Output format: `text` or `json` (default: `text`) |
+
+```bash
+memory list
+memory list --kind task --status todo
+memory list --tags auth,security
+memory list --stale
+memory list --format json
+```
+
+### `memory update <id>`
+
+Update an existing node's fields.
+
+| Argument / Option | Required | Description |
+|---|---|---|
+| `<id>` | yes | Node ID or unique prefix |
+| `-t, --title <title>` | no | New title |
+| `-c, --content <content>` | no | New content |
+| `--status <status>` | no | New status |
+| `--tags <tags>` | no | Replace tags (comma-separated) |
+| `--stale` | no | Mark as stale |
+
+```bash
+memory update abc123 --status done
+memory update abc123 -t "Updated title" -c "New content"
+memory update abc123 --tags newtag1,newtag2
+```
+
+### `memory remove <id>`
+
+Remove a node. Default is soft delete (marks as stale). Use `--hard` for permanent deletion.
+
+| Argument / Option | Required | Description |
+|---|---|---|
+| `<id>` | yes | Node ID or unique prefix |
+| `--hard` | no | Permanently delete instead of marking stale |
+
+```bash
+memory remove abc123
+memory remove abc123 --hard
+```
+
+### `memory link <fromId> <toId>`
+
+Create a directed edge between two nodes.
+
+| Argument / Option | Required | Description |
+|---|---|---|
+| `<fromId>` | yes | Source node ID or prefix |
+| `<toId>` | yes | Target node ID or prefix |
+| `--type <type>` | yes | Edge type: `depends_on`, `contains`, `implements`, `blocked_by`, `subtask_of`, `relates_to`, `supersedes`, `about` |
+
+```bash
+memory link abc123 def456 --type depends_on
+memory link abc123 def456 --type contains
+```
+
+### `memory graph [id]`
+
+Visualize the knowledge graph as an ASCII tree. Optionally root at a specific node.
+
+| Argument | Required | Description |
+|---|---|---|
+| `[id]` | no | Root node ID or prefix. Omit for full graph. |
+
+```bash
+memory graph
+memory graph abc123
+```
+
+### `memory decay`
+
+Run auto-decay to mark old untouched nodes as stale. Nodes whose `lastAccessedAt` exceeds the threshold are marked stale and hidden from default listings and search.
+
+| Option | Required | Description |
+|---|---|---|
+| `--days <number>` | no | Max age in days before decay (default: 180) |
+
+```bash
+memory decay
+memory decay --days 90
+memory decay --days 0   # decay all nodes not accessed today
+```
+
+### `memory serve`
+
+Start the Cortex Portal web server. Auto-decay runs on startup and every 24 hours.
+
+| Option | Required | Description |
+|---|---|---|
+| `-p, --port <number>` | no | Port number (default: 3100) |
+
+```bash
+memory serve
+memory serve --port 8080
+```
+
+---
+
+## Node Kinds
+
+| Kind | Purpose |
+|---|---|
+| `memory` | General knowledge, notes, context |
+| `component` | Code components, services, modules |
+| `task` | Work items, todos |
+| `decision` | Architectural or design decisions |
+
+## Edge Types
+
+| Type | Meaning |
+|---|---|
+| `depends_on` | Source depends on target |
+| `contains` | Source contains target |
+| `implements` | Source implements target |
+| `blocked_by` | Source is blocked by target |
+| `subtask_of` | Source is a subtask of target |
+| `relates_to` | General relationship |
+| `supersedes` | Source supersedes/replaces target |
+| `about` | Source is about target |
+
+## Search Scoring
+
+Search combines three signals:
+
+- **BM25** (weight 0.25) — text relevance
+- **Vector similarity** (weight 0.60) — semantic similarity via embeddings (requires Ollama)
+- **Freshness** (weight 0.15) — exponential decay based on `lastAccessedAt`
+
+Freshness multiplier: `e^(-0.01 * ageDays)` — half-life ~69 days. Recently accessed nodes are boosted; old untouched nodes are penalized but not zeroed.
+
+## REST API
+
+The web server exposes these endpoints under `/api`:
+
+| Method | Path | Description |
+|---|---|---|
+| `GET` | `/api/nodes` | List nodes. Query params: `kind`, `status`, `tags`, `limit`, `includeStale` |
+| `GET` | `/api/nodes/:id` | Get node + connections |
+| `POST` | `/api/nodes` | Create node. Body: `{ kind, title, content?, status?, tags?, metadata? }` |
+| `PATCH` | `/api/nodes/:id` | Update node. Body: `{ title?, content?, status?, tags?, metadata?, isStale? }` |
+| `DELETE` | `/api/nodes/:id` | Delete node. Query param: `hard=true` for permanent delete |
+| `POST` | `/api/edges` | Create edge. Body: `{ fromId, toId, type, metadata? }` |
+| `DELETE` | `/api/edges/:id` | Delete edge |
+| `GET` | `/api/graph` | Get full graph (nodes + edges) for visualization |
+| `POST` | `/api/search` | Search. Body: `{ text, options?: { kind?, tags?, limit?, includeStale? } }` |
--- a/portal/src/types.ts
+++ b/portal/src/types.ts
@@ -14,6 +14,7 @@ export interface CortexNode {
  metadata: Record<string, any>;
  createdAt: number;
  updatedAt: number;
+  lastAccessedAt?: number;
  isStale?: boolean;
 }

--- a/src/cli/commands/decay.ts
+++ b/src/cli/commands/decay.ts
@@ -0,0 +1,12 @@
+import { Command } from 'commander';
+import chalk from 'chalk';
+import { decayStaleNodes } from '../../core/decay';
+
+export const decayCommand = new Command('decay')
+  .description('Mark old untouched nodes as stale')
+  .option('--days <number>', 'Max age in days before decay', '180')
+  .action((opts) => {
+    const days = parseInt(opts.days, 10);
+    const count = decayStaleNodes(days);
+    console.log(chalk.green(`✓ Decayed ${count} node(s) older than ${days} days.`));
+  });
--- a/src/cli/index.ts
+++ b/src/cli/index.ts
@@ -9,6 +9,7 @@ import { updateCommand } from './commands/update';
 import { removeCommand } from './commands/remove';
 import { graphCommand } from './commands/graph';
 import { serveCommand } from './commands/serve';
+import { decayCommand } from './commands/decay';
 import { closeDb } from '../core/db';

 const program = new Command();
@@ -27,6 +28,7 @@ program.addCommand(updateCommand);
 program.addCommand(removeCommand);
 program.addCommand(graphCommand);
 program.addCommand(serveCommand);
+program.addCommand(decayCommand);

 program.hook('postAction', () => {
  closeDb();
--- a/src/core/db.ts
+++ b/src/core/db.ts
@@ -61,6 +61,13 @@ export function getDb(): Database.Database {
  _db.pragma('foreign_keys = ON');
  _db.exec(SCHEMA);

+  // Migration: add last_accessed_at column
+  const cols = _db.prepare("PRAGMA table_info(nodes)").all() as any[];
+  if (!cols.some((c: any) => c.name === 'last_accessed_at')) {
+    _db.exec('ALTER TABLE nodes ADD COLUMN last_accessed_at INTEGER');
+    _db.exec('UPDATE nodes SET last_accessed_at = updated_at WHERE last_accessed_at IS NULL');
+  }
+
  return _db;
 }

--- a/src/core/decay.ts
+++ b/src/core/decay.ts
@@ -0,0 +1,10 @@
+import { getDb } from './db';
+
+export function decayStaleNodes(maxAgeDays: number = 180): number {
+  const db = getDb();
+  const threshold = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000;
+  const result = db.prepare(
+    'UPDATE nodes SET is_stale = 1 WHERE is_stale = 0 AND (last_accessed_at IS NULL OR last_accessed_at < ?)'
+  ).run(threshold);
+  return result.changes;
+}
--- a/src/core/graph.ts
+++ b/src/core/graph.ts
@@ -46,6 +46,7 @@ export function getConnections(nodeId: string): { incoming: (Edge & { node: Node
      embedding: null,
      createdAt: row.n_created,
      updatedAt: row.n_updated,
+      lastAccessedAt: row.last_accessed_at ?? row.n_updated,
      isStale: !!row.is_stale,
    },
  });
--- a/src/core/search/freshness.ts
+++ b/src/core/search/freshness.ts
@@ -0,0 +1,6 @@
+const DECAY_RATE = 0.01; // ~69 day half-life
+
+export function freshnessMultiplier(lastAccessedAt: number, now: number = Date.now()): number {
+  const ageDays = (now - lastAccessedAt) / (1000 * 60 * 60 * 24);
+  return Math.exp(-DECAY_RATE * Math.max(0, ageDays));
+}
--- a/src/core/search/index.ts
+++ b/src/core/search/index.ts
@@ -2,9 +2,11 @@ import { Node, SearchResult, QueryOptions } from '../../types';
 import { bm25Search } from './bm25';
 import { cosineSimilarity } from './vector';
 import { getEmbedding, isOllamaAvailable } from './ollama';
+import { freshnessMultiplier } from './freshness';

-const VECTOR_WEIGHT = 0.7;
-const BM25_WEIGHT = 0.3;
+const VECTOR_WEIGHT = 0.6;
+const BM25_WEIGHT = 0.25;
+const FRESHNESS_WEIGHT = 0.15;

 function deserializeEmbedding(blob: Buffer | null): number[] | null {
  if (!blob || blob.length === 0) return null;
@@ -66,6 +68,13 @@ export async function hybridSearch(
    if (node) results.push({ node, score });
  }

+  // Apply freshness multiplier
+  const now = Date.now();
+  for (const r of results) {
+    const freshness = freshnessMultiplier(r.node.lastAccessedAt, now);
+    r.score = r.score * (1 - FRESHNESS_WEIGHT + FRESHNESS_WEIGHT * freshness);
+  }
+
  results.sort((a, b) => b.score - a.score);
  return results.slice(0, limit);
 }
--- a/src/core/store.ts
+++ b/src/core/store.ts
@@ -16,6 +16,7 @@ function rowToNode(row: any): Node {
    embedding: row.embedding ? deserializeEmbedding(row.embedding) : null,
    createdAt: row.created_at,
    updatedAt: row.updated_at,
+    lastAccessedAt: row.last_accessed_at ?? row.updated_at,
    isStale: !!row.is_stale,
  };
 }
@@ -36,13 +37,13 @@ export async function addNode(input: AddNodeInput): Promise<Node> {
  const embedding = await getEmbedding(`${input.title} ${content}`);

  db.prepare(`
-    INSERT INTO nodes (id, kind, title, content, status, tags, metadata, embedding, created_at, updated_at)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    INSERT INTO nodes (id, kind, title, content, status, tags, metadata, embedding, created_at, updated_at, last_accessed_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  `).run(
    id, input.kind, input.title, content, input.status ?? null,
    JSON.stringify(tags), JSON.stringify(metadata),
    embedding ? serializeEmbedding(embedding) : null,
-    now, now
+    now, now, now
  );

  // Insert tags
@@ -53,14 +54,16 @@ export async function addNode(input: AddNodeInput): Promise<Node> {

  return {
    id, kind: input.kind, title: input.title, content, status: input.status,
-    tags, metadata, embedding, createdAt: now, updatedAt: now, isStale: false,
+    tags, metadata, embedding, createdAt: now, updatedAt: now, lastAccessedAt: now, isStale: false,
  };
 }

 export function getNode(id: string): Node | null {
  const db = getDb();
  const row = db.prepare('SELECT * FROM nodes WHERE id = ?').get(id) as any;
-  return row ? rowToNode(row) : null;
+  if (!row) return null;
+  db.prepare('UPDATE nodes SET last_accessed_at = ? WHERE id = ?').run(Date.now(), id);
+  return rowToNode(row);
 }

 export function findNodeByPrefix(prefix: string): Node | null {
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -3,6 +3,7 @@ import cors from 'cors';
 import path from 'path';
 import routes from './routes';
 import { closeDb } from '../core/db';
+import { decayStaleNodes } from '../core/decay';

 const app = express();
 const PORT = parseInt(process.env.PORT || '3100');
@@ -18,11 +19,17 @@ app.get('{*path}', (_req, res) => {
  res.sendFile(path.join(portalDist, 'index.html'));
 });

+// Run auto-decay on startup and daily
+decayStaleNodes();
+const DECAY_INTERVAL = 24 * 60 * 60 * 1000;
+const decayTimer = setInterval(() => decayStaleNodes(), DECAY_INTERVAL);
+
 const server = app.listen(PORT, () => {
  console.log(`Cortex Portal running at http://localhost:${PORT}`);
 });

 process.on('SIGINT', () => {
+  clearInterval(decayTimer);
  closeDb();
  server.close();
  process.exit(0);
--- a/src/types.ts
+++ b/src/types.ts
@@ -21,6 +21,7 @@ export interface Node {
  embedding: number[] | null;
  createdAt: number;
  updatedAt: number;
+  lastAccessedAt: number;
  isStale: boolean;
 }