From 416c447ad8a5b82296053eeacc91236201802716 Mon Sep 17 00:00:00 2001 From: omigamedev Date: Sun, 18 Jan 2026 18:52:08 +0100 Subject: [PATCH] finalize Go + SQLite stack decisions for Synology NAS deployment --- DEV_PORTAL_M02_WEB_STACK.md | 44 +++++- DEV_PORTAL_M03_DATABASE.md | 279 +++++++++++++++++++++++++----------- 2 files changed, 233 insertions(+), 90 deletions(-) diff --git a/DEV_PORTAL_M02_WEB_STACK.md b/DEV_PORTAL_M02_WEB_STACK.md index 3404392..19d13b9 100644 --- a/DEV_PORTAL_M02_WEB_STACK.md +++ b/DEV_PORTAL_M02_WEB_STACK.md @@ -1,8 +1,44 @@ # Milestone 2: Web Stack Selection -**Status**: Planning +**Status**: Decided **Goal**: Choose backend technologies for the developer portal and app store API. +## Decision + +**Go** with the following stack: + +``` +Language: Go 1.22+ +Framework: Chi (lightweight, idiomatic) +Database: SQLite via modernc.org/sqlite (pure Go, no CGO) +Migrations: golang-migrate +Validation: go-playground/validator +Auth: Custom JWT + OAuth2 +Deployment: Single Docker container on Synology NAS +``` + +### Rationale + +1. **NAS-friendly** - Tiny Docker image (~15MB), low RAM (~30-50MB) +2. **Cross-compilation** - Easy build for ARM64 or AMD64 Synology models +3. **Pure Go SQLite** - `modernc.org/sqlite` requires no CGO, cross-compiles easily +4. **Single container** - Go binary + SQLite + Litestream in one image +5. **Standard library** - HTTP, JSON, crypto all built-in + +### Target Deployment + +``` +Synology NAS (Docker) +├── mosis-portal container (~15MB image) +│ ├── Go binary +│ ├── SQLite database (WAL mode) +│ └── Litestream backup +└── Volumes + ├── /volume1/mosis/data/portal.db + ├── /volume1/mosis/backups/ + └── /volume1/mosis/packages/ +``` + --- ## Overview @@ -308,12 +344,10 @@ Based on: ## Deliverables -- [ ] Prototype API in Go -- [ ] Prototype API in Node.js (if needed) -- [ ] Benchmark comparison document -- [ ] Final selection with rationale +- [x] Final selection with rationale - [ ] Project structure template - [ ] Development environment setup guide +- [ ] Base Go project scaffolding --- diff --git a/DEV_PORTAL_M03_DATABASE.md b/DEV_PORTAL_M03_DATABASE.md index c441c0b..fb18c03 100644 --- a/DEV_PORTAL_M03_DATABASE.md +++ b/DEV_PORTAL_M03_DATABASE.md @@ -1,8 +1,61 @@ # Milestone 3: Database Selection -**Status**: Planning +**Status**: Decided **Goal**: Choose database for developer accounts, app metadata, and analytics. +## Decision + +**SQLite + Litestream** for self-hosted deployment on Synology NAS. + +``` +Database: SQLite 3.x (WAL mode) +Driver: modernc.org/sqlite (pure Go, no CGO) +Backup: Litestream continuous replication +Storage: Synology volume (/volume1/mosis/) +``` + +### Rationale + +1. **Single container** - No separate database service needed +2. **Minimal resources** - ~50MB RAM, perfect for NAS +3. **Zero ops** - No connection pooling, no tuning +4. **Continuous backup** - Litestream replicates to local storage +5. **Point-in-time recovery** - Restore to any moment +6. **Sufficient scale** - Handles 1000s of developers easily + +### Architecture + +``` +┌─────────────────────────────────────────┐ +│ Synology NAS │ +│ ┌─────────────────────────────────┐ │ +│ │ mosis-portal container │ │ +│ │ ├── Go binary │ │ +│ │ ├── SQLite (portal.db) │ │ +│ │ └── Litestream │ │ +│ └──────────────┬──────────────────┘ │ +│ │ │ +│ ┌──────────────▼──────────────────┐ │ +│ │ /volume1/mosis/ │ │ +│ │ ├── data/portal.db │ │ +│ │ ├── data/portal.db-wal │ │ +│ │ ├── backups/ (litestream) │ │ +│ │ └── packages/ (app uploads) │ │ +│ └─────────────────────────────────┘ │ +└─────────────────────────────────────────┘ +``` + +### Litestream Configuration + +```yaml +dbs: + - path: /data/portal.db + replicas: + - type: file + path: /backups/portal + retention: 720h # 30 days +``` + --- ## Overview @@ -161,127 +214,140 @@ Redis → Caching, Sessions --- -## Schema Design +## Schema Design (SQLite) ### Core Tables ```sql -- Developers CREATE TABLE developers ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - email VARCHAR(255) UNIQUE NOT NULL, - name VARCHAR(100) NOT NULL, - password_hash VARCHAR(255), - oauth_provider VARCHAR(50), - oauth_id VARCHAR(255), - verified BOOLEAN DEFAULT FALSE, - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW() + id TEXT PRIMARY KEY, -- UUID as text + email TEXT UNIQUE NOT NULL, + name TEXT NOT NULL, + password_hash TEXT, + oauth_provider TEXT, + oauth_id TEXT, + verified INTEGER DEFAULT 0, + created_at TEXT DEFAULT (datetime('now')), + updated_at TEXT DEFAULT (datetime('now')) ); -- API Keys CREATE TABLE api_keys ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - developer_id UUID REFERENCES developers(id) ON DELETE CASCADE, - name VARCHAR(100) NOT NULL, - key_hash VARCHAR(255) NOT NULL, - key_prefix VARCHAR(10) NOT NULL, -- For display: "mk_abc..." - permissions JSONB DEFAULT '[]', - last_used_at TIMESTAMPTZ, - expires_at TIMESTAMPTZ, - created_at TIMESTAMPTZ DEFAULT NOW() + id TEXT PRIMARY KEY, + developer_id TEXT NOT NULL REFERENCES developers(id) ON DELETE CASCADE, + name TEXT NOT NULL, + key_hash TEXT NOT NULL, + key_prefix TEXT NOT NULL, -- For display: "mk_abc..." + permissions TEXT DEFAULT '[]', -- JSON array + last_used_at TEXT, + expires_at TEXT, + created_at TEXT DEFAULT (datetime('now')) ); -- Apps CREATE TABLE apps ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - developer_id UUID REFERENCES developers(id) ON DELETE CASCADE, - package_id VARCHAR(255) UNIQUE NOT NULL, -- com.dev.app - name VARCHAR(100) NOT NULL, + id TEXT PRIMARY KEY, + developer_id TEXT NOT NULL REFERENCES developers(id) ON DELETE CASCADE, + package_id TEXT UNIQUE NOT NULL, -- com.dev.app + name TEXT NOT NULL, description TEXT, - category VARCHAR(50), - tags VARCHAR(50)[] DEFAULT '{}', - status VARCHAR(20) DEFAULT 'draft', -- draft, published, suspended - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW() + category TEXT, + tags TEXT DEFAULT '[]', -- JSON array + status TEXT DEFAULT 'draft', -- draft, published, suspended + created_at TEXT DEFAULT (datetime('now')), + updated_at TEXT DEFAULT (datetime('now')) ); -- App Versions CREATE TABLE app_versions ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - app_id UUID REFERENCES apps(id) ON DELETE CASCADE, + id TEXT PRIMARY KEY, + app_id TEXT NOT NULL REFERENCES apps(id) ON DELETE CASCADE, version_code INTEGER NOT NULL, - version_name VARCHAR(20) NOT NULL, + version_name TEXT NOT NULL, package_url TEXT NOT NULL, - package_size BIGINT NOT NULL, - signature VARCHAR(512) NOT NULL, - permissions JSONB DEFAULT '[]', - min_mosis_version VARCHAR(20), + package_size INTEGER NOT NULL, + signature TEXT NOT NULL, + permissions TEXT DEFAULT '[]', -- JSON array + min_mosis_version TEXT, release_notes TEXT, - status VARCHAR(20) DEFAULT 'draft', -- draft, review, approved, published, rejected + status TEXT DEFAULT 'draft', -- draft, review, approved, published, rejected review_notes TEXT, - published_at TIMESTAMPTZ, - created_at TIMESTAMPTZ DEFAULT NOW(), + published_at TEXT, + created_at TEXT DEFAULT (datetime('now')), UNIQUE(app_id, version_code) ); -- Developer Signing Keys CREATE TABLE signing_keys ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - developer_id UUID REFERENCES developers(id) ON DELETE CASCADE, - name VARCHAR(100) NOT NULL, + id TEXT PRIMARY KEY, + developer_id TEXT NOT NULL REFERENCES developers(id) ON DELETE CASCADE, + name TEXT NOT NULL, public_key TEXT NOT NULL, - fingerprint VARCHAR(64) NOT NULL, - is_active BOOLEAN DEFAULT TRUE, - created_at TIMESTAMPTZ DEFAULT NOW() + fingerprint TEXT NOT NULL, + is_active INTEGER DEFAULT 1, + created_at TEXT DEFAULT (datetime('now')) ); ``` -### Telemetry Tables (if using PostgreSQL) +### Telemetry Tables ```sql --- Telemetry Events (consider partitioning by time) +-- Telemetry Events (append-only, partition by month via separate tables) CREATE TABLE telemetry_events ( - id BIGSERIAL, - app_id UUID NOT NULL, - device_id VARCHAR(64) NOT NULL, -- Hashed - event_type VARCHAR(50) NOT NULL, - event_data JSONB, - mosis_version VARCHAR(20), - timestamp TIMESTAMPTZ NOT NULL, - PRIMARY KEY (timestamp, id) -) PARTITION BY RANGE (timestamp); - --- Create monthly partitions -CREATE TABLE telemetry_events_2024_01 PARTITION OF telemetry_events - FOR VALUES FROM ('2024-01-01') TO ('2024-02-01'); + id INTEGER PRIMARY KEY AUTOINCREMENT, + app_id TEXT NOT NULL, + device_id TEXT NOT NULL, -- Hashed for privacy + event_type TEXT NOT NULL, + event_data TEXT, -- JSON string + mosis_version TEXT, + timestamp TEXT NOT NULL -- ISO8601 format +); -- Crash Reports CREATE TABLE crash_reports ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - app_id UUID NOT NULL, - app_version VARCHAR(20) NOT NULL, - device_id VARCHAR(64) NOT NULL, - crash_type VARCHAR(50) NOT NULL, + id TEXT PRIMARY KEY, + app_id TEXT NOT NULL, + app_version TEXT NOT NULL, + device_id TEXT NOT NULL, + crash_type TEXT NOT NULL, message TEXT, stack_trace TEXT, - context JSONB, - mosis_version VARCHAR(20), - timestamp TIMESTAMPTZ NOT NULL, - created_at TIMESTAMPTZ DEFAULT NOW() + context TEXT, -- JSON string + mosis_version TEXT, + timestamp TEXT NOT NULL, + created_at TEXT DEFAULT (datetime('now')) ); --- Daily aggregates (materialized or computed) +-- Daily aggregates (computed by background job) CREATE TABLE telemetry_daily ( - app_id UUID NOT NULL, - date DATE NOT NULL, - event_type VARCHAR(50) NOT NULL, - count BIGINT NOT NULL, - unique_devices BIGINT NOT NULL, + app_id TEXT NOT NULL, + date TEXT NOT NULL, -- YYYY-MM-DD + event_type TEXT NOT NULL, + count INTEGER NOT NULL, + unique_devices INTEGER NOT NULL, PRIMARY KEY (app_id, date, event_type) ); + +-- Audit Logs +CREATE TABLE audit_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + developer_id TEXT, + action TEXT NOT NULL, + resource_type TEXT, + resource_id TEXT, + details TEXT, -- JSON string + ip_address TEXT, + user_agent TEXT, + created_at TEXT DEFAULT (datetime('now')) +); ``` +**Note**: For high-volume telemetry, consider: +- Separate SQLite database file for telemetry (isolates write load) +- Monthly table rotation with application-level partitioning +- Aggressive data retention (delete events older than 90 days) + ### Indexes ```sql @@ -289,16 +355,24 @@ CREATE TABLE telemetry_daily ( CREATE INDEX idx_developers_email ON developers(email); CREATE INDEX idx_developers_oauth ON developers(oauth_provider, oauth_id); +-- API Keys +CREATE INDEX idx_api_keys_developer ON api_keys(developer_id); +CREATE INDEX idx_api_keys_prefix ON api_keys(key_prefix); + -- Apps CREATE INDEX idx_apps_developer ON apps(developer_id); CREATE INDEX idx_apps_package ON apps(package_id); CREATE INDEX idx_apps_status ON apps(status); -CREATE INDEX idx_apps_search ON apps USING gin(to_tsvector('english', name || ' ' || COALESCE(description, ''))); +CREATE INDEX idx_apps_name ON apps(name); -- For LIKE searches -- Versions CREATE INDEX idx_versions_app ON app_versions(app_id); CREATE INDEX idx_versions_status ON app_versions(status); +-- Signing Keys +CREATE INDEX idx_signing_keys_developer ON signing_keys(developer_id); +CREATE INDEX idx_signing_keys_fingerprint ON signing_keys(fingerprint); + -- Telemetry CREATE INDEX idx_telemetry_app ON telemetry_events(app_id, timestamp); CREATE INDEX idx_telemetry_type ON telemetry_events(event_type, timestamp); @@ -306,6 +380,41 @@ CREATE INDEX idx_telemetry_type ON telemetry_events(event_type, timestamp); -- Crashes CREATE INDEX idx_crashes_app ON crash_reports(app_id, timestamp); CREATE INDEX idx_crashes_type ON crash_reports(crash_type); + +-- Audit Logs +CREATE INDEX idx_audit_developer ON audit_logs(developer_id); +CREATE INDEX idx_audit_created ON audit_logs(created_at); +``` + +**Full-text Search**: For app search, use SQLite FTS5: + +```sql +-- Create FTS5 virtual table for app search +CREATE VIRTUAL TABLE apps_fts USING fts5( + name, + description, + tags, + content='apps', + content_rowid='rowid' +); + +-- Triggers to keep FTS in sync +CREATE TRIGGER apps_ai AFTER INSERT ON apps BEGIN + INSERT INTO apps_fts(rowid, name, description, tags) + VALUES (NEW.rowid, NEW.name, NEW.description, NEW.tags); +END; + +CREATE TRIGGER apps_ad AFTER DELETE ON apps BEGIN + INSERT INTO apps_fts(apps_fts, rowid, name, description, tags) + VALUES ('delete', OLD.rowid, OLD.name, OLD.description, OLD.tags); +END; + +CREATE TRIGGER apps_au AFTER UPDATE ON apps BEGIN + INSERT INTO apps_fts(apps_fts, rowid, name, description, tags) + VALUES ('delete', OLD.rowid, OLD.name, OLD.description, OLD.tags); + INSERT INTO apps_fts(rowid, name, description, tags) + VALUES (NEW.rowid, NEW.name, NEW.description, NEW.tags); +END; ``` --- @@ -392,21 +501,21 @@ Redis → Caching, rate limiting ## Deliverables -- [ ] Final database selection -- [ ] Complete schema design -- [ ] Migration scripts -- [ ] Backup/restore procedures -- [ ] Connection pooling setup (if PostgreSQL) +- [x] Final database selection (SQLite + Litestream) +- [x] Complete schema design (core + telemetry + FTS5) +- [ ] Migration scripts (golang-migrate) +- [x] Backup/restore procedures (Litestream to local storage) +- [x] ~~Connection pooling setup~~ (not needed for SQLite) - [ ] Monitoring queries --- ## Open Questions -1. Expected telemetry volume per day? -2. How long to retain raw telemetry? -3. Need for real-time analytics vs batch? -4. Multi-region requirements? +1. ~~Expected telemetry volume per day?~~ → Start simple, optimize if needed +2. ~~How long to retain raw telemetry?~~ → 90 days raw, daily aggregates indefinitely +3. ~~Need for real-time analytics vs batch?~~ → Batch is sufficient for MVP +4. ~~Multi-region requirements?~~ → Single NAS deployment for now ---