Add query bar, maintenance panel, and heartbeat system
- Query bar with organized/grouped search results in portal - Maintenance panel UI for triggering and viewing maintenance status - Heartbeat service with periodic maintenance and dirty-tracking - Query organizer for grouping search results by tag/kind/parent - Slide-up animation for query panel
This commit is contained in:
@@ -4,8 +4,10 @@ import GraphView from './components/GraphView';
|
||||
import Sidebar from './components/Sidebar';
|
||||
import NodePanel from './components/NodePanel';
|
||||
import AddNodeModal from './components/AddNodeModal';
|
||||
import QueryBar from './components/QueryBar';
|
||||
import LinkModal from './components/LinkModal';
|
||||
import Toast from './components/Toast';
|
||||
import MaintenancePanel from './components/MaintenancePanel';
|
||||
|
||||
export default function App() {
|
||||
const [selectedId, setSelectedId] = useState<string | null>(null);
|
||||
@@ -13,6 +15,8 @@ export default function App() {
|
||||
const [linkFromId, setLinkFromId] = useState<string | null>(null);
|
||||
const [toast, setToast] = useState<string | null>(null);
|
||||
const [drawerOpen, setDrawerOpen] = useState(false);
|
||||
const [showQuery, setShowQuery] = useState(false);
|
||||
const [showMaintenance, setShowMaintenance] = useState(false);
|
||||
const qc = useQueryClient();
|
||||
|
||||
const refresh = useCallback(() => {
|
||||
@@ -34,12 +38,13 @@ export default function App() {
|
||||
const handler = (e: KeyboardEvent) => {
|
||||
if (e.key === 'Escape') {
|
||||
if (selectedId) setSelectedId(null);
|
||||
else if (showQuery) setShowQuery(false);
|
||||
else if (drawerOpen) setDrawerOpen(false);
|
||||
}
|
||||
};
|
||||
window.addEventListener('keydown', handler);
|
||||
return () => window.removeEventListener('keydown', handler);
|
||||
}, [selectedId, drawerOpen]);
|
||||
}, [selectedId, drawerOpen, showQuery]);
|
||||
|
||||
return (
|
||||
<div className="h-screen w-screen overflow-hidden relative">
|
||||
@@ -64,6 +69,20 @@ export default function App() {
|
||||
>
|
||||
+
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setShowQuery(!showQuery)}
|
||||
className="w-12 h-12 rounded-full bg-purple-600/90 backdrop-blur border border-purple-500 text-white hover:bg-purple-500 shadow-lg flex items-center justify-center text-lg font-bold"
|
||||
title="Query memory"
|
||||
>
|
||||
?
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setShowMaintenance(!showMaintenance)}
|
||||
className="w-12 h-12 rounded-full bg-emerald-600/90 backdrop-blur border border-emerald-500 text-white hover:bg-emerald-500 shadow-lg flex items-center justify-center text-lg"
|
||||
title="Maintenance"
|
||||
>
|
||||
⚙
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Sidebar drawer — slides in from left */}
|
||||
@@ -106,6 +125,22 @@ export default function App() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Query panel — slides up from bottom */}
|
||||
{showQuery && (
|
||||
<div className="fixed inset-0 z-40" onClick={() => setShowQuery(false)}>
|
||||
<div className="absolute inset-0 bg-black/40" />
|
||||
<div
|
||||
className="absolute bottom-0 left-0 right-0 h-[70vh] animate-slide-in-up"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
>
|
||||
<QueryBar
|
||||
onSelectNode={(id) => { setShowQuery(false); selectNode(id); }}
|
||||
onClose={() => setShowQuery(false)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{showAddNode && (
|
||||
<AddNodeModal
|
||||
onClose={() => setShowAddNode(false)}
|
||||
@@ -119,6 +154,11 @@ export default function App() {
|
||||
onCreated={() => { refresh(); notify('Edge created'); }}
|
||||
/>
|
||||
)}
|
||||
{showMaintenance && (
|
||||
<div className="fixed bottom-20 left-5 z-50">
|
||||
<MaintenancePanel onClose={() => setShowMaintenance(false)} onNotify={(msg) => { notify(msg); refresh(); }} />
|
||||
</div>
|
||||
)}
|
||||
{toast && <Toast message={toast} />}
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { CortexNode, CortexEdge, GraphData, NodeWithConnections, SearchResult, NodeKind, EdgeType } from './types';
|
||||
import type { CortexNode, CortexEdge, GraphData, NodeWithConnections, SearchResult, NodeKind, EdgeType, GroupedQueryResult } from './types';
|
||||
|
||||
const BASE = '/api';
|
||||
|
||||
@@ -41,4 +41,13 @@ export const api = {
|
||||
|
||||
search: (text: string, options?: Record<string, any>) =>
|
||||
request<SearchResult[]>('/search', { method: 'POST', body: JSON.stringify({ text, options }) }),
|
||||
|
||||
queryOrganized: (text: string) =>
|
||||
request<GroupedQueryResult>('/query/organize', { method: 'POST', body: JSON.stringify({ text }) }),
|
||||
|
||||
getMaintenanceStatus: () =>
|
||||
request<Record<string, any>>('/maintenance/status'),
|
||||
|
||||
runMaintenance: () =>
|
||||
request<Record<string, any>>('/maintenance/run', { method: 'POST' }),
|
||||
};
|
||||
|
||||
@@ -18,3 +18,12 @@
|
||||
.animate-slide-in-right {
|
||||
animation: slide-in-right 0.2s ease-out;
|
||||
}
|
||||
|
||||
@keyframes slide-in-up {
|
||||
from { transform: translateY(100%); }
|
||||
to { transform: translateY(0); }
|
||||
}
|
||||
|
||||
.animate-slide-in-up {
|
||||
animation: slide-in-up 0.2s ease-out;
|
||||
}
|
||||
|
||||
103
portal/src/components/MaintenancePanel.tsx
Normal file
103
portal/src/components/MaintenancePanel.tsx
Normal file
@@ -0,0 +1,103 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { api } from '../api';
|
||||
|
||||
interface Report {
|
||||
ranAt: number;
|
||||
deduped: number;
|
||||
autoTagged: number;
|
||||
autoOrganized: number;
|
||||
pruned: number;
|
||||
summarized?: number;
|
||||
merged?: number;
|
||||
split?: number;
|
||||
archived?: number;
|
||||
aiAvailable?: boolean;
|
||||
skipped: boolean;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export default function MaintenancePanel({ onClose, onNotify }: { onClose: () => void; onNotify: (msg: string) => void }) {
|
||||
const [report, setReport] = useState<Report | null>(null);
|
||||
const [running, setRunning] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
api.getMaintenanceStatus().then(r => setReport(r as any)).catch(() => {});
|
||||
}, []);
|
||||
|
||||
const run = async () => {
|
||||
setRunning(true);
|
||||
setError(null);
|
||||
try {
|
||||
const r = await api.runMaintenance();
|
||||
setReport(r as Report);
|
||||
onNotify('Maintenance complete');
|
||||
} catch (e: any) {
|
||||
setError(e.message);
|
||||
} finally {
|
||||
setRunning(false);
|
||||
}
|
||||
};
|
||||
|
||||
const stats: { label: string; key: keyof Report }[] = [
|
||||
{ label: 'Deduped', key: 'deduped' },
|
||||
{ label: 'Auto-tagged', key: 'autoTagged' },
|
||||
{ label: 'Organized', key: 'autoOrganized' },
|
||||
{ label: 'Pruned', key: 'pruned' },
|
||||
{ label: 'Summarized', key: 'summarized' },
|
||||
{ label: 'Merged', key: 'merged' },
|
||||
{ label: 'Split', key: 'split' },
|
||||
{ label: 'Archived', key: 'archived' },
|
||||
];
|
||||
|
||||
return (
|
||||
<div className="bg-gray-900 border border-gray-700 rounded-xl p-5 w-80 max-w-[90vw] shadow-2xl text-gray-200">
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<h2 className="text-sm font-semibold uppercase tracking-wide text-gray-400">Maintenance</h2>
|
||||
<button onClick={onClose} className="text-gray-500 hover:text-white text-lg leading-none">×</button>
|
||||
</div>
|
||||
|
||||
{report && !report.message && (
|
||||
<div className="mb-4 space-y-1 text-xs">
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<span className={`inline-block w-2 h-2 rounded-full ${report.aiAvailable ? 'bg-green-400' : 'bg-yellow-500'}`} />
|
||||
<span className="text-gray-400">{report.aiAvailable ? 'AI available (Ollama)' : 'AI unavailable — cosine fallback'}</span>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-x-4 gap-y-1">
|
||||
{stats.map(s => {
|
||||
const val = report[s.key];
|
||||
if (val === undefined) return null;
|
||||
return (
|
||||
<div key={s.key} className="flex justify-between">
|
||||
<span className="text-gray-400">{s.label}</span>
|
||||
<span className="font-mono text-gray-200">{val as number}</span>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
<div className="text-gray-500 mt-2">
|
||||
{report.skipped ? 'Skipped (no changes)' : `Ran ${new Date(report.ranAt).toLocaleTimeString()}`}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{report?.message && (
|
||||
<p className="text-xs text-gray-500 mb-4">{report.message}</p>
|
||||
)}
|
||||
|
||||
{!report && !error && (
|
||||
<p className="text-xs text-gray-500 mb-4">No maintenance has run yet.</p>
|
||||
)}
|
||||
|
||||
{error && <p className="text-xs text-red-400 mb-4">{error}</p>}
|
||||
|
||||
<button
|
||||
onClick={run}
|
||||
disabled={running}
|
||||
className="w-full py-2 rounded-lg text-sm font-medium bg-indigo-600 hover:bg-indigo-500 disabled:opacity-50 disabled:cursor-wait transition"
|
||||
>
|
||||
{running ? 'Running...' : 'Run Maintenance'}
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
109
portal/src/components/QueryBar.tsx
Normal file
109
portal/src/components/QueryBar.tsx
Normal file
@@ -0,0 +1,109 @@
|
||||
import { useState } from 'react';
|
||||
import { api } from '../api';
|
||||
import type { GroupedQueryResult } from '../types';
|
||||
|
||||
interface Props {
|
||||
onSelectNode: (id: string) => void;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
export default function QueryBar({ onSelectNode, onClose }: Props) {
|
||||
const [text, setText] = useState('');
|
||||
const [result, setResult] = useState<GroupedQueryResult | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [expanded, setExpanded] = useState<Set<string>>(new Set());
|
||||
|
||||
const submit = async () => {
|
||||
if (!text.trim()) return;
|
||||
setLoading(true);
|
||||
try {
|
||||
const data = await api.queryOrganized(text);
|
||||
setResult(data);
|
||||
setExpanded(new Set(data.groups.map(g => g.label)));
|
||||
} catch {
|
||||
setResult(null);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const toggle = (label: string) => {
|
||||
setExpanded(prev => {
|
||||
const next = new Set(prev);
|
||||
next.has(label) ? next.delete(label) : next.add(label);
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="h-full flex flex-col bg-gray-900/95 backdrop-blur border-t border-purple-500/30 rounded-t-xl">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between px-4 py-3 border-b border-gray-700/50">
|
||||
<span className="text-purple-300 font-medium text-sm">Query Memory</span>
|
||||
<button onClick={onClose} className="text-gray-500 hover:text-gray-300 text-lg">×</button>
|
||||
</div>
|
||||
|
||||
{/* Input */}
|
||||
<div className="px-4 py-3 flex gap-2">
|
||||
<input
|
||||
value={text}
|
||||
onChange={e => setText(e.target.value)}
|
||||
onKeyDown={e => e.key === 'Enter' && submit()}
|
||||
placeholder='e.g. "group all git commands by tag"'
|
||||
className="flex-1 bg-gray-800 border border-gray-600 rounded-lg px-3 py-2 text-sm text-gray-200 placeholder-gray-500 focus:outline-none focus:border-purple-500"
|
||||
autoFocus
|
||||
/>
|
||||
<button
|
||||
onClick={submit}
|
||||
disabled={loading}
|
||||
className="px-4 py-2 bg-purple-600 hover:bg-purple-500 disabled:opacity-50 rounded-lg text-sm text-white font-medium"
|
||||
>
|
||||
{loading ? '...' : 'Search'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Results */}
|
||||
<div className="flex-1 overflow-y-auto px-4 pb-4">
|
||||
{!result && !loading && (
|
||||
<div className="text-gray-500 text-sm mt-4 space-y-1">
|
||||
<p>Example queries:</p>
|
||||
<p className="text-gray-600 ml-2">"all decisions about architecture"</p>
|
||||
<p className="text-gray-600 ml-2">"group tasks by tag"</p>
|
||||
<p className="text-gray-600 ml-2">"show components under parent"</p>
|
||||
</div>
|
||||
)}
|
||||
{result && (
|
||||
<div className="space-y-2">
|
||||
<p className="text-xs text-gray-500">{result.totalResults} results · grouped by {result.strategy}</p>
|
||||
{result.groups.map(group => (
|
||||
<div key={group.label} className="border border-gray-700/50 rounded-lg overflow-hidden">
|
||||
<button
|
||||
onClick={() => toggle(group.label)}
|
||||
className="w-full flex items-center justify-between px-3 py-2 bg-gray-800/50 hover:bg-gray-800 text-sm text-gray-300"
|
||||
>
|
||||
<span className="font-medium">{group.label} ({group.items.length})</span>
|
||||
<span className="text-xs">{expanded.has(group.label) ? '▼' : '▶'}</span>
|
||||
</button>
|
||||
{expanded.has(group.label) && (
|
||||
<div className="divide-y divide-gray-800">
|
||||
{group.items.map(item => (
|
||||
<button
|
||||
key={item.node.id}
|
||||
onClick={() => onSelectNode(item.node.id)}
|
||||
className="w-full text-left px-3 py-2 hover:bg-gray-800/50 flex items-center gap-2"
|
||||
>
|
||||
<span className="text-xs px-1.5 py-0.5 rounded bg-gray-700 text-gray-400">{item.node.kind}</span>
|
||||
<span className="text-sm text-gray-200 truncate">{item.node.title}</span>
|
||||
<span className="text-xs text-gray-600 ml-auto">{(item.score * 100).toFixed(0)}%</span>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -43,3 +43,14 @@ export interface SearchResult {
|
||||
node: CortexNode;
|
||||
score: number;
|
||||
}
|
||||
|
||||
export interface ResultGroup {
|
||||
label: string;
|
||||
items: { node: { id: string; kind: string; title: string; content: string; status?: string; tags: string[] }; score: number }[];
|
||||
}
|
||||
|
||||
export interface GroupedQueryResult {
|
||||
strategy: 'tag' | 'kind' | 'parent' | 'flat';
|
||||
groups: ResultGroup[];
|
||||
totalResults: number;
|
||||
}
|
||||
|
||||
@@ -25,6 +25,13 @@ function serializeEmbedding(embedding: number[]): Buffer {
|
||||
return Buffer.from(new Float32Array(embedding).buffer);
|
||||
}
|
||||
|
||||
function notifyDirty(): void {
|
||||
try {
|
||||
const { markDirty } = require('../server/heartbeat');
|
||||
markDirty();
|
||||
} catch {}
|
||||
}
|
||||
|
||||
export async function addNode(input: AddNodeInput): Promise<Node> {
|
||||
const db = getDb();
|
||||
const id = uuid();
|
||||
@@ -52,6 +59,7 @@ export async function addNode(input: AddNodeInput): Promise<Node> {
|
||||
insertTag.run(id, tag);
|
||||
}
|
||||
|
||||
notifyDirty();
|
||||
return {
|
||||
id, kind: input.kind, title: input.title, content, status: input.status,
|
||||
tags, metadata, embedding, createdAt: now, updatedAt: now, lastAccessedAt: now, isStale: false,
|
||||
@@ -149,6 +157,7 @@ export async function updateNode(id: string, input: UpdateNodeInput): Promise<No
|
||||
}
|
||||
}
|
||||
|
||||
notifyDirty();
|
||||
return getNode(id);
|
||||
}
|
||||
|
||||
@@ -159,6 +168,7 @@ export function removeNode(id: string, hard: boolean = false): boolean {
|
||||
return result.changes > 0;
|
||||
} else {
|
||||
const result = db.prepare('UPDATE nodes SET is_stale = 1, updated_at = ? WHERE id = ?').run(Date.now(), id);
|
||||
if (result.changes > 0) notifyDirty();
|
||||
return result.changes > 0;
|
||||
}
|
||||
}
|
||||
|
||||
345
src/server/heartbeat.ts
Normal file
345
src/server/heartbeat.ts
Normal file
@@ -0,0 +1,345 @@
|
||||
import { getDb } from '../core/db';
|
||||
import { deserializeEmbedding } from '../core/search/index';
|
||||
import { cosineSimilarity } from '../core/search/vector';
|
||||
import { isGenAvailable, generate } from '../core/search/ollamaGen';
|
||||
|
||||
let dirty = false;
|
||||
|
||||
export function markDirty(): void {
|
||||
dirty = true;
|
||||
}
|
||||
|
||||
export interface HeartbeatReport {
|
||||
ranAt: number;
|
||||
deduped: number;
|
||||
autoTagged: number;
|
||||
autoOrganized: number;
|
||||
pruned: number;
|
||||
summarized: number;
|
||||
merged: number;
|
||||
split: number;
|
||||
archived: number;
|
||||
aiAvailable: boolean;
|
||||
skipped: boolean;
|
||||
}
|
||||
|
||||
let lastReport: HeartbeatReport | null = null;
|
||||
|
||||
export function getLastReport(): HeartbeatReport | null {
|
||||
return lastReport;
|
||||
}
|
||||
|
||||
export async function runMaintenance(): Promise<HeartbeatReport> {
|
||||
if (!dirty) {
|
||||
const report: HeartbeatReport = {
|
||||
ranAt: Date.now(), deduped: 0, autoTagged: 0, autoOrganized: 0,
|
||||
pruned: 0, summarized: 0, merged: 0, split: 0, archived: 0,
|
||||
aiAvailable: false, skipped: true,
|
||||
};
|
||||
lastReport = report;
|
||||
return report;
|
||||
}
|
||||
|
||||
dirty = false;
|
||||
const db = getDb();
|
||||
const now = Date.now();
|
||||
let deduped = 0;
|
||||
let autoTagged = 0;
|
||||
let autoOrganized = 0;
|
||||
let pruned = 0;
|
||||
let summarized = 0;
|
||||
let merged = 0;
|
||||
let splitCount = 0;
|
||||
let archived = 0;
|
||||
|
||||
const aiAvailable = await isGenAvailable();
|
||||
|
||||
// Load all active nodes with embeddings
|
||||
const rows = db.prepare('SELECT * FROM nodes WHERE is_stale = 0').all() as any[];
|
||||
const nodes = rows.map(r => ({
|
||||
id: r.id as string,
|
||||
kind: r.kind as string,
|
||||
title: r.title as string,
|
||||
content: (r.content || '') as string,
|
||||
tags: JSON.parse(r.tags || '[]') as string[],
|
||||
metadata: JSON.parse(r.metadata || '{}') as Record<string, any>,
|
||||
embedding: deserializeEmbedding(r.embedding),
|
||||
updatedAt: r.updated_at as number,
|
||||
lastAccessedAt: (r.last_accessed_at ?? r.updated_at) as number,
|
||||
}));
|
||||
|
||||
const withEmb = nodes.filter(n => n.embedding !== null);
|
||||
|
||||
// --- Auto-dedupe: cosine >= 0.92 ---
|
||||
const staleIds = new Set<string>();
|
||||
const dupePairs: { survivor: typeof nodes[0]; victim: typeof nodes[0] }[] = [];
|
||||
|
||||
for (let i = 0; i < withEmb.length; i++) {
|
||||
if (staleIds.has(withEmb[i].id)) continue;
|
||||
for (let j = i + 1; j < withEmb.length; j++) {
|
||||
if (staleIds.has(withEmb[j].id)) continue;
|
||||
const sim = cosineSimilarity(withEmb[i].embedding!, withEmb[j].embedding!);
|
||||
if (sim >= 0.92) {
|
||||
const victim = withEmb[i].updatedAt > withEmb[j].updatedAt ? withEmb[i] : withEmb[j];
|
||||
const survivor = victim === withEmb[i] ? withEmb[j] : withEmb[i];
|
||||
staleIds.add(victim.id);
|
||||
dupePairs.push({ survivor, victim });
|
||||
deduped++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AI merge duplicates or just mark stale
|
||||
if (aiAvailable && dupePairs.length > 0) {
|
||||
const updateContent = db.prepare('UPDATE nodes SET content = ?, updated_at = ? WHERE id = ?');
|
||||
const markStale = db.prepare('UPDATE nodes SET is_stale = 1, updated_at = ? WHERE id = ?');
|
||||
const moveEdgesFrom = db.prepare('UPDATE edges SET from_id = ? WHERE from_id = ?');
|
||||
const moveEdgesTo = db.prepare('UPDATE edges SET to_id = ? WHERE to_id = ?');
|
||||
|
||||
for (const { survivor, victim } of dupePairs) {
|
||||
const prompt = `Merge these two related memory nodes into one coherent piece of content. Keep all unique information, remove redundancy. Output ONLY the merged content, no explanation.
|
||||
|
||||
Node 1 (${survivor.title}):
|
||||
${survivor.content}
|
||||
|
||||
Node 2 (${victim.title}):
|
||||
${victim.content}`;
|
||||
|
||||
const mergedContent = await generate(prompt);
|
||||
if (mergedContent) {
|
||||
updateContent.run(mergedContent, now, survivor.id);
|
||||
survivor.content = mergedContent;
|
||||
merged++;
|
||||
}
|
||||
markStale.run(now, victim.id);
|
||||
moveEdgesFrom.run(survivor.id, victim.id);
|
||||
moveEdgesTo.run(survivor.id, victim.id);
|
||||
}
|
||||
} else if (staleIds.size > 0) {
|
||||
const markStale = db.prepare('UPDATE nodes SET is_stale = 1, updated_at = ? WHERE id = ?');
|
||||
for (const id of staleIds) {
|
||||
markStale.run(now, id);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Auto-tag ---
|
||||
const allTags = new Set<string>();
|
||||
for (const n of nodes) {
|
||||
if (!staleIds.has(n.id)) n.tags.forEach(t => allTags.add(t));
|
||||
}
|
||||
const tagVocab = [...allTags];
|
||||
|
||||
const untagged = nodes.filter(n => n.tags.length === 0 && !staleIds.has(n.id));
|
||||
|
||||
if (aiAvailable && untagged.length > 0 && tagVocab.length > 0) {
|
||||
const updateTags = db.prepare('UPDATE nodes SET tags = ?, updated_at = ? WHERE id = ?');
|
||||
const insertTag = db.prepare('INSERT OR IGNORE INTO node_tags (node_id, tag) VALUES (?, ?)');
|
||||
|
||||
for (const node of untagged) {
|
||||
const prompt = `Given this memory node, pick 1-3 tags from the existing vocabulary. If none fit, suggest one new short tag. Output ONLY comma-separated tags, nothing else.
|
||||
|
||||
Title: ${node.title}
|
||||
Content: ${node.content.slice(0, 500)}
|
||||
|
||||
Existing tags: ${tagVocab.join(', ')}`;
|
||||
|
||||
const resp = await generate(prompt);
|
||||
if (resp) {
|
||||
const tags = resp.split(',').map(t => t.trim().toLowerCase().replace(/[^a-z0-9_-]/g, '')).filter(Boolean).slice(0, 3);
|
||||
if (tags.length > 0) {
|
||||
updateTags.run(JSON.stringify(tags), now, node.id);
|
||||
for (const tag of tags) {
|
||||
insertTag.run(node.id, tag);
|
||||
allTags.add(tag);
|
||||
}
|
||||
autoTagged++;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (!aiAvailable) {
|
||||
// Fallback: cosine-based auto-tag
|
||||
const untaggedEmb = withEmb.filter(n => n.tags.length === 0 && !staleIds.has(n.id));
|
||||
const tagged = withEmb.filter(n => n.tags.length > 0 && !staleIds.has(n.id));
|
||||
if (untaggedEmb.length > 0 && tagged.length > 0) {
|
||||
const updateTags = db.prepare('UPDATE nodes SET tags = ?, updated_at = ? WHERE id = ?');
|
||||
const insertTag = db.prepare('INSERT OR IGNORE INTO node_tags (node_id, tag) VALUES (?, ?)');
|
||||
for (const node of untaggedEmb) {
|
||||
let bestSim = 0;
|
||||
let bestTag = '';
|
||||
for (const candidate of tagged) {
|
||||
const sim = cosineSimilarity(node.embedding!, candidate.embedding!);
|
||||
if (sim >= 0.75 && sim > bestSim) {
|
||||
bestSim = sim;
|
||||
bestTag = candidate.tags[0];
|
||||
}
|
||||
}
|
||||
if (bestTag) {
|
||||
updateTags.run(JSON.stringify([bestTag]), now, node.id);
|
||||
insertTag.run(node.id, bestTag);
|
||||
autoTagged++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Auto-organize orphans ---
|
||||
const edgeRows = db.prepare('SELECT from_id, to_id FROM edges').all() as any[];
|
||||
const hasEdge = new Set<string>();
|
||||
for (const e of edgeRows) {
|
||||
hasEdge.add(e.from_id);
|
||||
hasEdge.add(e.to_id);
|
||||
}
|
||||
const orphans = nodes.filter(n => !hasEdge.has(n.id) && !staleIds.has(n.id));
|
||||
const nonOrphans = nodes.filter(n => hasEdge.has(n.id) && !staleIds.has(n.id));
|
||||
|
||||
if (orphans.length > 0 && nonOrphans.length > 0) {
|
||||
const insertEdge = db.prepare('INSERT INTO edges (id, from_id, to_id, type, metadata, created_at) VALUES (?, ?, ?, ?, ?, ?)');
|
||||
const { v4: uuidv4 } = require('uuid');
|
||||
|
||||
if (aiAvailable) {
|
||||
for (const orphan of orphans) {
|
||||
const candidates = nonOrphans.slice(0, 20).map(n => `- ${n.id.slice(0, 8)}: ${n.title}`).join('\n');
|
||||
const prompt = `This orphan memory node needs a parent. Pick the best parent and relationship type.
|
||||
Output ONLY in format: <parent_id_prefix> <type>
|
||||
Where type is one of: contains, relates_to, about
|
||||
|
||||
Orphan: "${orphan.title}" — ${orphan.content.slice(0, 200)}
|
||||
|
||||
Potential parents:
|
||||
${candidates}`;
|
||||
|
||||
const resp = await generate(prompt);
|
||||
if (resp) {
|
||||
const parts = resp.trim().split(/\s+/);
|
||||
const prefix = parts[0]?.replace(/[^a-f0-9]/gi, '');
|
||||
const edgeType = ['contains', 'relates_to', 'about'].includes(parts[1]) ? parts[1] : 'relates_to';
|
||||
const parent = nonOrphans.find(n => n.id.startsWith(prefix));
|
||||
if (parent) {
|
||||
insertEdge.run(uuidv4(), parent.id, orphan.id, edgeType, '{}', now);
|
||||
hasEdge.add(orphan.id);
|
||||
autoOrganized++;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Fallback: cosine-based
|
||||
const orphansEmb = orphans.filter(n => n.embedding !== null);
|
||||
const nonOrphansEmb = nonOrphans.filter(n => n.embedding !== null);
|
||||
for (const orphan of orphansEmb) {
|
||||
let bestSim = 0;
|
||||
let bestParent = '';
|
||||
for (const candidate of nonOrphansEmb) {
|
||||
const sim = cosineSimilarity(orphan.embedding!, candidate.embedding!);
|
||||
if (sim >= 0.70 && sim > bestSim) {
|
||||
bestSim = sim;
|
||||
bestParent = candidate.id;
|
||||
}
|
||||
}
|
||||
if (bestParent) {
|
||||
insertEdge.run(uuidv4(), bestParent, orphan.id, 'relates_to', '{}', now);
|
||||
autoOrganized++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- AI-only operations (require Ollama) ---
|
||||
if (aiAvailable) {
|
||||
// Auto-summarize: content > 500 chars, no existing summary
|
||||
const longNodes = nodes.filter(n => n.content.length > 500 && !n.metadata.summary && !staleIds.has(n.id));
|
||||
const updateMeta = db.prepare('UPDATE nodes SET metadata = ?, updated_at = ? WHERE id = ?');
|
||||
|
||||
for (const node of longNodes) {
|
||||
const prompt = `Summarize this memory node in 1-2 sentences. Output ONLY the summary, nothing else.
|
||||
|
||||
Title: ${node.title}
|
||||
Content: ${node.content.slice(0, 2000)}`;
|
||||
|
||||
const summary = await generate(prompt);
|
||||
if (summary) {
|
||||
const newMeta = { ...node.metadata, summary };
|
||||
updateMeta.run(JSON.stringify(newMeta), now, node.id);
|
||||
summarized++;
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-split: content > 2000 chars
|
||||
const hugeNodes = nodes.filter(n => n.content.length > 2000 && !staleIds.has(n.id));
|
||||
const { v4: uuidv4 } = require('uuid');
|
||||
const insertNode = db.prepare(`INSERT INTO nodes (id, kind, title, content, status, tags, metadata, embedding, created_at, updated_at, last_accessed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
||||
const insertEdge2 = db.prepare('INSERT INTO edges (id, from_id, to_id, type, metadata, created_at) VALUES (?, ?, ?, ?, ?, ?)');
|
||||
|
||||
for (const node of hugeNodes) {
|
||||
const prompt = `Split this large memory node into 2-4 logical sections. Output in this exact format (one section per block):
|
||||
---SECTION---
|
||||
Title: <section title>
|
||||
Content: <section content>
|
||||
|
||||
Do this for each section. No other text.
|
||||
|
||||
Original title: ${node.title}
|
||||
Content: ${node.content.slice(0, 3000)}`;
|
||||
|
||||
const resp = await generate(prompt);
|
||||
if (resp && resp.includes('---SECTION---')) {
|
||||
const sections = resp.split('---SECTION---').filter(s => s.trim());
|
||||
if (sections.length >= 2) {
|
||||
const childIds: string[] = [];
|
||||
for (const section of sections) {
|
||||
const titleMatch = section.match(/Title:\s*(.+)/);
|
||||
const contentMatch = section.match(/Content:\s*([\s\S]+)/);
|
||||
if (titleMatch && contentMatch) {
|
||||
const childId = uuidv4();
|
||||
insertNode.run(childId, node.kind, titleMatch[1].trim(), contentMatch[1].trim(),
|
||||
null, JSON.stringify(node.tags), JSON.stringify({}), null, now, now, now);
|
||||
insertEdge2.run(uuidv4(), node.id, childId, 'contains', '{}', now);
|
||||
childIds.push(childId);
|
||||
}
|
||||
}
|
||||
if (childIds.length >= 2) {
|
||||
// Update parent to summary
|
||||
const summaryPrompt = `Summarize this in 1-2 sentences as a parent overview. Output ONLY the summary.\n\n${node.content.slice(0, 2000)}`;
|
||||
const parentSummary = await generate(summaryPrompt);
|
||||
if (parentSummary) {
|
||||
db.prepare('UPDATE nodes SET content = ?, updated_at = ? WHERE id = ?').run(parentSummary, now, node.id);
|
||||
}
|
||||
splitCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-archive: not accessed in 90+ days
|
||||
const ninetyDaysAgo = now - 90 * 24 * 60 * 60 * 1000;
|
||||
const staleCandidate = nodes.filter(n => n.lastAccessedAt < ninetyDaysAgo && !staleIds.has(n.id));
|
||||
|
||||
for (const node of staleCandidate) {
|
||||
const edgeCount = db.prepare('SELECT COUNT(*) as c FROM edges WHERE from_id = ? OR to_id = ?').get(node.id, node.id) as any;
|
||||
const prompt = `This memory node hasn't been accessed in over 90 days. It has ${edgeCount.c} connections. Should it be archived (marked stale)?
|
||||
Answer ONLY "yes" or "no".
|
||||
|
||||
Title: ${node.title}
|
||||
Content: ${node.content.slice(0, 300)}`;
|
||||
|
||||
const resp = await generate(prompt);
|
||||
if (resp && resp.toLowerCase().includes('yes')) {
|
||||
db.prepare('UPDATE nodes SET is_stale = 1, updated_at = ? WHERE id = ?').run(now, node.id);
|
||||
archived++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Prune: hard-delete stale nodes > 30 days old
|
||||
const thirtyDaysAgo = now - 30 * 24 * 60 * 60 * 1000;
|
||||
const pruneResult = db.prepare('DELETE FROM nodes WHERE is_stale = 1 AND updated_at < ?').run(thirtyDaysAgo);
|
||||
pruned = pruneResult.changes;
|
||||
|
||||
const report: HeartbeatReport = {
|
||||
ranAt: now, deduped, autoTagged, autoOrganized, pruned,
|
||||
summarized, merged, split: splitCount, archived,
|
||||
aiAvailable, skipped: false,
|
||||
};
|
||||
lastReport = report;
|
||||
console.log(`[Heartbeat] deduped=${deduped} autoTagged=${autoTagged} autoOrganized=${autoOrganized} pruned=${pruned} summarized=${summarized} merged=${merged} split=${splitCount} archived=${archived} ai=${aiAvailable}`);
|
||||
return report;
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import path from 'path';
|
||||
import routes from './routes';
|
||||
import { closeDb } from '../core/db';
|
||||
import { decayStaleNodes } from '../core/decay';
|
||||
import { runMaintenance, markDirty } from './heartbeat';
|
||||
|
||||
const app = express();
|
||||
const PORT = parseInt(process.env.PORT || '3100');
|
||||
@@ -24,12 +25,18 @@ decayStaleNodes();
|
||||
const DECAY_INTERVAL = 24 * 60 * 60 * 1000;
|
||||
const decayTimer = setInterval(() => decayStaleNodes(), DECAY_INTERVAL);
|
||||
|
||||
// Heartbeat maintenance every 5 minutes
|
||||
markDirty(); // run on first heartbeat
|
||||
const HEARTBEAT_INTERVAL = 5 * 60 * 1000;
|
||||
const heartbeatTimer = setInterval(() => runMaintenance(), HEARTBEAT_INTERVAL);
|
||||
|
||||
const server = app.listen(PORT, () => {
|
||||
console.log(`Cortex Portal running at http://localhost:${PORT}`);
|
||||
});
|
||||
|
||||
process.on('SIGINT', () => {
|
||||
clearInterval(decayTimer);
|
||||
clearInterval(heartbeatTimer);
|
||||
closeDb();
|
||||
server.close();
|
||||
process.exit(0);
|
||||
|
||||
107
src/server/queryOrganizer.ts
Normal file
107
src/server/queryOrganizer.ts
Normal file
@@ -0,0 +1,107 @@
|
||||
import { SearchResult } from '../types';
|
||||
import { getDb } from '../core/db';
|
||||
|
||||
export type GroupingStrategy = 'tag' | 'kind' | 'parent' | 'flat';
|
||||
|
||||
export interface ResultGroup {
|
||||
label: string;
|
||||
items: { node: { id: string; kind: string; title: string; content: string; status?: string; tags: string[] }; score: number }[];
|
||||
}
|
||||
|
||||
export interface GroupedQueryResult {
|
||||
strategy: GroupingStrategy;
|
||||
groups: ResultGroup[];
|
||||
totalResults: number;
|
||||
}
|
||||
|
||||
const KIND_KEYWORDS = ['type', 'kind', 'category', 'categories', 'types'];
|
||||
const TAG_KEYWORDS = ['tag', 'tagged', 'label', 'topic'];
|
||||
const PARENT_KEYWORDS = ['parent', 'group', 'tree', 'hierarchy', 'under', 'children'];
|
||||
|
||||
export function determineGroupingStrategy(text: string): GroupingStrategy {
|
||||
const lower = text.toLowerCase();
|
||||
if (KIND_KEYWORDS.some(k => lower.includes(k))) return 'kind';
|
||||
if (TAG_KEYWORDS.some(k => lower.includes(k))) return 'tag';
|
||||
if (PARENT_KEYWORDS.some(k => lower.includes(k))) return 'parent';
|
||||
return 'flat';
|
||||
}
|
||||
|
||||
export function groupResults(results: SearchResult[], strategy: GroupingStrategy): GroupedQueryResult {
|
||||
const strip = (r: SearchResult) => ({
|
||||
node: {
|
||||
id: r.node.id,
|
||||
kind: r.node.kind,
|
||||
title: r.node.title,
|
||||
content: r.node.content,
|
||||
status: r.node.status,
|
||||
tags: r.node.tags,
|
||||
},
|
||||
score: r.score,
|
||||
});
|
||||
|
||||
if (strategy === 'flat') {
|
||||
return {
|
||||
strategy,
|
||||
groups: [{ label: 'Results', items: results.map(strip) }],
|
||||
totalResults: results.length,
|
||||
};
|
||||
}
|
||||
|
||||
if (strategy === 'kind') {
|
||||
const map = new Map<string, ResultGroup['items']>();
|
||||
for (const r of results) {
|
||||
const key = r.node.kind;
|
||||
if (!map.has(key)) map.set(key, []);
|
||||
map.get(key)!.push(strip(r));
|
||||
}
|
||||
return {
|
||||
strategy,
|
||||
groups: [...map.entries()].map(([label, items]) => ({ label, items })),
|
||||
totalResults: results.length,
|
||||
};
|
||||
}
|
||||
|
||||
if (strategy === 'tag') {
|
||||
const map = new Map<string, ResultGroup['items']>();
|
||||
for (const r of results) {
|
||||
const tag = r.node.tags[0] || 'untagged';
|
||||
if (!map.has(tag)) map.set(tag, []);
|
||||
map.get(tag)!.push(strip(r));
|
||||
}
|
||||
return {
|
||||
strategy,
|
||||
groups: [...map.entries()].map(([label, items]) => ({ label, items })),
|
||||
totalResults: results.length,
|
||||
};
|
||||
}
|
||||
|
||||
// parent — group by parent via 'contains' edges
|
||||
const db = getDb();
|
||||
const nodeIds = results.map(r => r.node.id);
|
||||
const parentMap = new Map<string, string>();
|
||||
|
||||
if (nodeIds.length > 0) {
|
||||
const placeholders = nodeIds.map(() => '?').join(',');
|
||||
const rows = db.prepare(`
|
||||
SELECT e.to_id, n.title as parent_title
|
||||
FROM edges e JOIN nodes n ON e.from_id = n.id
|
||||
WHERE e.type = 'contains' AND e.to_id IN (${placeholders})
|
||||
`).all(...nodeIds) as any[];
|
||||
for (const row of rows) {
|
||||
parentMap.set(row.to_id, row.parent_title);
|
||||
}
|
||||
}
|
||||
|
||||
const map = new Map<string, ResultGroup['items']>();
|
||||
for (const r of results) {
|
||||
const parent = parentMap.get(r.node.id) || 'Orphan';
|
||||
if (!map.has(parent)) map.set(parent, []);
|
||||
map.get(parent)!.push(strip(r));
|
||||
}
|
||||
|
||||
return {
|
||||
strategy,
|
||||
groups: [...map.entries()].map(([label, items]) => ({ label, items })),
|
||||
totalResults: results.length,
|
||||
};
|
||||
}
|
||||
@@ -2,6 +2,8 @@ import { Router, Request, Response } from 'express';
|
||||
import { addNode, getNode, listNodes, updateNode, removeNode, addEdge, removeEdge, query } from '../core/store';
|
||||
import { getConnections, buildTree } from '../core/graph';
|
||||
import { getDb } from '../core/db';
|
||||
import { determineGroupingStrategy, groupResults } from './queryOrganizer';
|
||||
import { getLastReport, markDirty, runMaintenance } from './heartbeat';
|
||||
|
||||
const router = Router();
|
||||
|
||||
@@ -133,4 +135,35 @@ router.post('/search', async (req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
// Organized query
|
||||
router.post('/query/organize', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { text } = req.body;
|
||||
if (!text) return res.status(400).json({ error: 'text is required' });
|
||||
const results = await query(text, { limit: 30 });
|
||||
const strategy = determineGroupingStrategy(text);
|
||||
const grouped = groupResults(results, strategy);
|
||||
res.json(grouped);
|
||||
} catch (err: any) {
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Maintenance status
|
||||
router.get('/maintenance/status', (_req: Request, res: Response) => {
|
||||
const report = getLastReport();
|
||||
res.json(report || { message: 'No heartbeat has run yet' });
|
||||
});
|
||||
|
||||
// Trigger maintenance manually
|
||||
router.post('/maintenance/run', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
markDirty();
|
||||
const report = await runMaintenance();
|
||||
res.json(report);
|
||||
} catch (err: any) {
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
||||
Reference in New Issue
Block a user