finalize M06-M12 with Go/SQLite/Synology NAS implementation decisions
This commit is contained in:
@@ -1,8 +1,39 @@
|
||||
# Milestone 6: App Store Backend API
|
||||
|
||||
**Status**: Planning
|
||||
**Status**: Decided
|
||||
**Goal**: REST API for app submission, review, and distribution.
|
||||
|
||||
## Decision
|
||||
|
||||
**Go + Chi router** with JSON REST API:
|
||||
|
||||
```
|
||||
Framework: Chi (lightweight, idiomatic)
|
||||
Validation: go-playground/validator/v10
|
||||
OpenAPI: ogen (generated from spec)
|
||||
Middleware: Custom auth, rate limiting, logging
|
||||
Database: SQLite via repository pattern
|
||||
```
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ mosis-portal container │
|
||||
│ ┌────────────────────────────────────────────────────────────┐ │
|
||||
│ │ Chi Router │ │
|
||||
│ │ /v1/auth/* /v1/apps/* /v1/store/* /v1/keys/* │ │
|
||||
│ └──────────────────────┬─────────────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ ┌──────────┬───────────┼───────────┬──────────┐ │
|
||||
│ │ Handlers │ Services │ Repos │ SQLite │ │
|
||||
│ └──────────┴───────────┴───────────┴──────────┘ │
|
||||
│ │
|
||||
│ /volume1/mosis/data/portal.db │
|
||||
│ /volume1/mosis/packages/ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
@@ -319,108 +350,128 @@ GET /v1/apps/:id/crashes:
|
||||
|
||||
---
|
||||
|
||||
## Data Models
|
||||
## Data Models (Go)
|
||||
|
||||
### Developer
|
||||
|
||||
```typescript
|
||||
interface Developer {
|
||||
id: string;
|
||||
email: string;
|
||||
name: string;
|
||||
avatar_url?: string;
|
||||
verified: boolean;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
```go
|
||||
type Developer struct {
|
||||
ID string `json:"id"`
|
||||
Email string `json:"email"`
|
||||
Name string `json:"name"`
|
||||
AvatarURL *string `json:"avatar_url,omitempty"`
|
||||
Verified bool `json:"verified"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
```
|
||||
|
||||
### App
|
||||
|
||||
```typescript
|
||||
interface App {
|
||||
id: string;
|
||||
package_id: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
category?: string;
|
||||
tags: string[];
|
||||
status: 'draft' | 'published' | 'suspended';
|
||||
icon_url?: string;
|
||||
latest_version?: AppVersion;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
```go
|
||||
type AppStatus string
|
||||
|
||||
const (
|
||||
AppStatusDraft AppStatus = "draft"
|
||||
AppStatusPublished AppStatus = "published"
|
||||
AppStatusSuspended AppStatus = "suspended"
|
||||
)
|
||||
|
||||
type App struct {
|
||||
ID string `json:"id"`
|
||||
PackageID string `json:"package_id"`
|
||||
Name string `json:"name"`
|
||||
Description *string `json:"description,omitempty"`
|
||||
Category *string `json:"category,omitempty"`
|
||||
Tags []string `json:"tags"`
|
||||
Status AppStatus `json:"status"`
|
||||
IconURL *string `json:"icon_url,omitempty"`
|
||||
LatestVersion *AppVersion `json:"latest_version,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
```
|
||||
|
||||
### AppVersion
|
||||
|
||||
```typescript
|
||||
interface AppVersion {
|
||||
id: string;
|
||||
app_id: string;
|
||||
version_name: string;
|
||||
version_code: number;
|
||||
package_url?: string;
|
||||
package_size?: number;
|
||||
signature?: string;
|
||||
permissions: string[];
|
||||
min_mosis_version?: string;
|
||||
release_notes?: string;
|
||||
status: 'draft' | 'uploading' | 'validating' | 'review' | 'approved' | 'published' | 'rejected';
|
||||
review_notes?: string;
|
||||
published_at?: string;
|
||||
created_at: string;
|
||||
```go
|
||||
type VersionStatus string
|
||||
|
||||
const (
|
||||
VersionStatusDraft VersionStatus = "draft"
|
||||
VersionStatusUploading VersionStatus = "uploading"
|
||||
VersionStatusValidating VersionStatus = "validating"
|
||||
VersionStatusReview VersionStatus = "review"
|
||||
VersionStatusApproved VersionStatus = "approved"
|
||||
VersionStatusPublished VersionStatus = "published"
|
||||
VersionStatusRejected VersionStatus = "rejected"
|
||||
)
|
||||
|
||||
type AppVersion struct {
|
||||
ID string `json:"id"`
|
||||
AppID string `json:"app_id"`
|
||||
VersionName string `json:"version_name"`
|
||||
VersionCode int `json:"version_code"`
|
||||
PackageURL *string `json:"package_url,omitempty"`
|
||||
PackageSize *int64 `json:"package_size,omitempty"`
|
||||
Signature *string `json:"signature,omitempty"`
|
||||
Permissions []string `json:"permissions"`
|
||||
MinMosisVersion *string `json:"min_mosis_version,omitempty"`
|
||||
ReleaseNotes *string `json:"release_notes,omitempty"`
|
||||
Status VersionStatus `json:"status"`
|
||||
ReviewNotes *string `json:"review_notes,omitempty"`
|
||||
PublishedAt *time.Time `json:"published_at,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
```
|
||||
|
||||
### PublicApp
|
||||
|
||||
```typescript
|
||||
interface PublicApp {
|
||||
package_id: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
category?: string;
|
||||
tags: string[];
|
||||
icon_url?: string;
|
||||
author_name: string;
|
||||
latest_version: string;
|
||||
download_count: number;
|
||||
rating?: number;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
```go
|
||||
type PublicApp struct {
|
||||
PackageID string `json:"package_id"`
|
||||
Name string `json:"name"`
|
||||
Description *string `json:"description,omitempty"`
|
||||
Category *string `json:"category,omitempty"`
|
||||
Tags []string `json:"tags"`
|
||||
IconURL *string `json:"icon_url,omitempty"`
|
||||
AuthorName string `json:"author_name"`
|
||||
LatestVersion string `json:"latest_version"`
|
||||
DownloadCount int64 `json:"download_count"`
|
||||
Rating *float64 `json:"rating,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
```
|
||||
|
||||
### APIKey
|
||||
|
||||
```typescript
|
||||
interface APIKey {
|
||||
id: string;
|
||||
name: string;
|
||||
key_prefix: string; // "mk_live_abc..."
|
||||
permissions: string[];
|
||||
last_used_at?: string;
|
||||
expires_at?: string;
|
||||
created_at: string;
|
||||
```go
|
||||
type APIKey struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
KeyPrefix string `json:"key_prefix"` // "mk_live_abc..."
|
||||
Permissions []string `json:"permissions"`
|
||||
LastUsedAt *time.Time `json:"last_used_at,omitempty"`
|
||||
ExpiresAt *time.Time `json:"expires_at,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
```
|
||||
|
||||
### CrashReport
|
||||
|
||||
```typescript
|
||||
interface CrashReport {
|
||||
id: string;
|
||||
app_id: string;
|
||||
app_version: string;
|
||||
crash_type: string;
|
||||
message: string;
|
||||
stack_trace: string;
|
||||
context: object;
|
||||
occurrences: number;
|
||||
first_seen: string;
|
||||
last_seen: string;
|
||||
```go
|
||||
type CrashReport struct {
|
||||
ID string `json:"id"`
|
||||
AppID string `json:"app_id"`
|
||||
AppVersion string `json:"app_version"`
|
||||
CrashType string `json:"crash_type"`
|
||||
Message string `json:"message"`
|
||||
StackTrace string `json:"stack_trace"`
|
||||
Context map[string]interface{} `json:"context"`
|
||||
Occurrences int `json:"occurrences"`
|
||||
FirstSeen time.Time `json:"first_seen"`
|
||||
LastSeen time.Time `json:"last_seen"`
|
||||
}
|
||||
```
|
||||
|
||||
@@ -532,40 +583,100 @@ Full OpenAPI 3.0 spec will be generated and hosted at:
|
||||
|
||||
---
|
||||
|
||||
## Implementation Structure
|
||||
## Implementation Structure (Go)
|
||||
|
||||
```
|
||||
src/
|
||||
├── main.go (or index.ts)
|
||||
cmd/
|
||||
└── portal/
|
||||
└── main.go # Entry point, wire dependencies
|
||||
|
||||
internal/
|
||||
├── api/
|
||||
│ ├── routes.go
|
||||
│ ├── router.go # Chi router setup
|
||||
│ ├── middleware/
|
||||
│ │ ├── auth.go
|
||||
│ │ ├── ratelimit.go
|
||||
│ │ └── logging.go
|
||||
│ │ ├── auth.go # JWT/API key validation
|
||||
│ │ ├── ratelimit.go # Token bucket rate limiter
|
||||
│ │ ├── logging.go # Request/response logging
|
||||
│ │ └── recovery.go # Panic recovery
|
||||
│ └── handlers/
|
||||
│ ├── auth.go
|
||||
│ ├── apps.go
|
||||
│ ├── versions.go
|
||||
│ ├── store.go
|
||||
│ ├── keys.go
|
||||
│ └── telemetry.go
|
||||
│ ├── auth.go # OAuth2 + token endpoints
|
||||
│ ├── apps.go # App CRUD
|
||||
│ ├── versions.go # Version upload/publish
|
||||
│ ├── store.go # Public store API
|
||||
│ ├── keys.go # API/signing keys
|
||||
│ └── telemetry.go # Event ingestion
|
||||
├── service/
|
||||
│ ├── app_service.go
|
||||
│ ├── version_service.go
|
||||
│ ├── auth_service.go
|
||||
│ └── storage_service.go
|
||||
├── repository/
|
||||
│ ├── app_repo.go
|
||||
│ ├── version_repo.go
|
||||
│ └── developer_repo.go
|
||||
├── domain/
|
||||
│ ├── app.go
|
||||
│ ├── app.go # Business logic
|
||||
│ ├── version.go
|
||||
│ └── developer.go
|
||||
└── pkg/
|
||||
├── validator/
|
||||
└── crypto/
|
||||
│ ├── auth.go
|
||||
│ └── storage.go # File storage operations
|
||||
├── repository/
|
||||
│ ├── sqlite/ # SQLite implementations
|
||||
│ │ ├── app.go
|
||||
│ │ ├── version.go
|
||||
│ │ ├── developer.go
|
||||
│ │ └── migrations/ # SQL migrations
|
||||
│ └── interfaces.go # Repository interfaces
|
||||
└── domain/
|
||||
├── app.go # Domain types
|
||||
├── version.go
|
||||
└── developer.go
|
||||
|
||||
pkg/
|
||||
├── validator/ # Custom validation rules
|
||||
└── signing/ # Ed25519 operations
|
||||
```
|
||||
|
||||
### Chi Router Setup
|
||||
|
||||
```go
|
||||
func NewRouter(
|
||||
authHandler *handlers.AuthHandler,
|
||||
appHandler *handlers.AppHandler,
|
||||
storeHandler *handlers.StoreHandler,
|
||||
) chi.Router {
|
||||
r := chi.NewRouter()
|
||||
|
||||
// Global middleware
|
||||
r.Use(middleware.RequestID)
|
||||
r.Use(middleware.RealIP)
|
||||
r.Use(middleware.Logger)
|
||||
r.Use(middleware.Recoverer)
|
||||
r.Use(middleware.Timeout(30 * time.Second))
|
||||
|
||||
// API v1
|
||||
r.Route("/v1", func(r chi.Router) {
|
||||
// Public routes
|
||||
r.Group(func(r chi.Router) {
|
||||
r.Post("/auth/oauth/github", authHandler.GitHubOAuth)
|
||||
r.Get("/auth/oauth/github/callback", authHandler.GitHubCallback)
|
||||
r.Post("/auth/refresh", authHandler.Refresh)
|
||||
r.Get("/store/apps", storeHandler.ListApps)
|
||||
r.Get("/store/apps/{packageID}", storeHandler.GetApp)
|
||||
})
|
||||
|
||||
// Protected routes
|
||||
r.Group(func(r chi.Router) {
|
||||
r.Use(middleware.RequireAuth)
|
||||
r.Get("/auth/me", authHandler.Me)
|
||||
r.Route("/apps", func(r chi.Router) {
|
||||
r.Get("/", appHandler.List)
|
||||
r.Post("/", appHandler.Create)
|
||||
r.Route("/{appID}", func(r chi.Router) {
|
||||
r.Get("/", appHandler.Get)
|
||||
r.Patch("/", appHandler.Update)
|
||||
r.Delete("/", appHandler.Delete)
|
||||
r.Route("/versions", func(r chi.Router) {
|
||||
r.Get("/", appHandler.ListVersions)
|
||||
r.Post("/", appHandler.CreateVersion)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
return r
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
@@ -590,10 +701,10 @@ src/
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. GraphQL alongside REST?
|
||||
2. WebSocket for real-time review status?
|
||||
3. Batch operations for bulk updates?
|
||||
4. API versioning strategy (URL vs header)?
|
||||
1. ~~GraphQL alongside REST?~~ → REST only for simplicity
|
||||
2. WebSocket for real-time review status? → Consider for v1.1
|
||||
3. ~~Batch operations for bulk updates?~~ → Not needed for MVP
|
||||
4. ~~API versioning strategy (URL vs header)?~~ → URL prefix (/v1/)
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,13 +1,64 @@
|
||||
# Milestone 7: CDN & Storage
|
||||
|
||||
**Status**: Planning
|
||||
**Status**: Decided
|
||||
**Goal**: Scalable storage for app packages and assets with global distribution.
|
||||
|
||||
## Decision
|
||||
|
||||
**Local Synology filesystem** as primary storage, with optional Cloudflare R2 for CDN:
|
||||
|
||||
```
|
||||
Primary: Synology volume (/volume1/mosis/)
|
||||
CDN: Cloudflare R2 (optional, for global distribution)
|
||||
Serving: Go binary serves files directly (local)
|
||||
Backup: Synology Hyper Backup / rsync
|
||||
```
|
||||
|
||||
### Rationale
|
||||
|
||||
1. **Self-hosted** - All data stays on premises
|
||||
2. **Zero cost** - No cloud storage fees
|
||||
3. **Simple** - Go binary serves files directly via http.FileServer
|
||||
4. **Fast local** - NAS has gigabit+ internal network
|
||||
5. **Optional CDN** - Can sync to R2 if global distribution needed
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Synology NAS │
|
||||
│ ┌────────────────────────────────────────────────────────────┐ │
|
||||
│ │ mosis-portal container │ │
|
||||
│ │ │ │
|
||||
│ │ Go binary ──serves──► /packages/, /assets/ │ │
|
||||
│ │ │ │ │
|
||||
│ │ └── SQLite (/data/portal.db) │ │
|
||||
│ └──────────────────────────┬─────────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ /volume1/mosis/ │ bind mount │
|
||||
│ ├── data/portal.db │ │
|
||||
│ ├── packages/ ◄───────────┘ │
|
||||
│ │ └── {dev_id}/{app_id}/{version}/package.mosis │
|
||||
│ ├── assets/ │
|
||||
│ │ └── {app_id}/icon-{size}.png │
|
||||
│ └── backups/ │
|
||||
│ └── litestream replicas │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
(optional sync)
|
||||
▼
|
||||
┌──────────────────┐
|
||||
│ Cloudflare R2 │
|
||||
│ (CDN for global │
|
||||
│ distribution) │
|
||||
└──────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Storage handles app packages, icons, screenshots, and serves downloads to devices worldwide. Must be cost-effective, fast, and reliable.
|
||||
Storage handles app packages, icons, screenshots, and serves downloads to devices. For self-hosted Synology NAS deployment, local filesystem is the primary storage.
|
||||
|
||||
---
|
||||
|
||||
@@ -170,84 +221,120 @@ Pricing: VPS cost only
|
||||
|
||||
## Recommendation
|
||||
|
||||
**Primary: Cloudflare R2**
|
||||
- Zero egress fees (biggest cost saver)
|
||||
- S3-compatible (easy migration)
|
||||
- Built-in global CDN
|
||||
- Good enough tooling
|
||||
**Primary: Local Synology Filesystem**
|
||||
- Zero recurring costs
|
||||
- All data on premises
|
||||
- Simple Go file serving
|
||||
- Synology's built-in backup tools
|
||||
|
||||
**Fallback: Backblaze B2 + Cloudflare**
|
||||
- If R2 has issues
|
||||
- Even cheaper storage
|
||||
- Slightly more setup
|
||||
**Optional: Cloudflare R2 for CDN**
|
||||
- If global distribution needed
|
||||
- Sync packages via cron/background job
|
||||
- Zero egress fees
|
||||
- S3-compatible API
|
||||
|
||||
---
|
||||
|
||||
## Upload Flow
|
||||
|
||||
### Presigned URL Approach
|
||||
### Local Filesystem Approach
|
||||
|
||||
```
|
||||
┌────────┐ ┌─────────┐ ┌─────────┐
|
||||
│ Client │───►│ API │───►│ R2 │
|
||||
└────────┘ └─────────┘ └─────────┘
|
||||
│ │ │
|
||||
│ 1. Request upload URL │
|
||||
│◄─────────────┤ │
|
||||
│ (presigned URL) │
|
||||
│ │
|
||||
│ 2. Upload directly │
|
||||
│────────────────────────────►│
|
||||
│ │
|
||||
│ 3. Notify complete │
|
||||
│─────────────►│ │
|
||||
│ │ 4. Validate │
|
||||
│ │─────────────►│
|
||||
│ │ │
|
||||
│ 5. Confirm │ │
|
||||
│◄─────────────┤ │
|
||||
┌────────┐ ┌─────────┐ ┌─────────────────┐
|
||||
│ Client │───►│ API │───►│ Local Filesystem│
|
||||
└────────┘ └─────────┘ └─────────────────┘
|
||||
│ │ │
|
||||
│ 1. Upload package (multipart) │
|
||||
│─────────────►│ │
|
||||
│ │ │
|
||||
│ │ 2. Save to temp │
|
||||
│ │─────────────────►│
|
||||
│ │ │
|
||||
│ │ 3. Validate │
|
||||
│ │ 4. Move to final│
|
||||
│ │─────────────────►│
|
||||
│ │ │
|
||||
│ 5. Confirm │ │
|
||||
│◄─────────────┤ │
|
||||
```
|
||||
|
||||
### API Implementation
|
||||
### Go Implementation (Local Storage)
|
||||
|
||||
```go
|
||||
// 1. Request upload URL
|
||||
func CreateVersion(c *gin.Context) {
|
||||
// Create version record
|
||||
version := Version{
|
||||
AppID: appID,
|
||||
VersionCode: req.VersionCode,
|
||||
Status: "uploading",
|
||||
// Upload handler - direct file upload
|
||||
func (h *VersionHandler) Upload(w http.ResponseWriter, r *http.Request) {
|
||||
appID := chi.URLParam(r, "appID")
|
||||
devID := r.Context().Value("developer_id").(string)
|
||||
|
||||
// Parse multipart form (max 50MB)
|
||||
r.ParseMultipartForm(50 << 20)
|
||||
file, header, err := r.FormFile("package")
|
||||
if err != nil {
|
||||
respondError(w, http.StatusBadRequest, "INVALID_FILE", "No package file")
|
||||
return
|
||||
}
|
||||
db.Create(&version)
|
||||
defer file.Close()
|
||||
|
||||
// Generate presigned URL
|
||||
key := fmt.Sprintf("temp/%s/package.mosis", version.ID)
|
||||
url, err := r2.PresignPut(key, 15*time.Minute)
|
||||
// Create version record
|
||||
version := &domain.Version{
|
||||
ID: uuid.New().String(),
|
||||
AppID: appID,
|
||||
VersionCode: parseInt(r.FormValue("version_code")),
|
||||
VersionName: r.FormValue("version_name"),
|
||||
Status: domain.VersionStatusUploading,
|
||||
}
|
||||
|
||||
c.JSON(200, gin.H{
|
||||
"version": version,
|
||||
"upload_url": url,
|
||||
"expires": time.Now().Add(15 * time.Minute),
|
||||
})
|
||||
// Save to temp directory
|
||||
tempPath := filepath.Join(h.storagePath, "temp", version.ID, "package.mosis")
|
||||
os.MkdirAll(filepath.Dir(tempPath), 0755)
|
||||
|
||||
dst, err := os.Create(tempPath)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "STORAGE_ERROR", err.Error())
|
||||
return
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
size, err := io.Copy(dst, file)
|
||||
version.PackageSize = size
|
||||
|
||||
// Validate package (signature, manifest, etc.)
|
||||
if err := h.validator.Validate(tempPath); err != nil {
|
||||
os.Remove(tempPath)
|
||||
respondError(w, http.StatusBadRequest, "VALIDATION_ERROR", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Move to final location
|
||||
finalPath := filepath.Join(h.storagePath, "packages",
|
||||
devID, appID, fmt.Sprintf("%d", version.VersionCode), "package.mosis")
|
||||
os.MkdirAll(filepath.Dir(finalPath), 0755)
|
||||
os.Rename(tempPath, finalPath)
|
||||
|
||||
version.PackageURL = finalPath
|
||||
version.Status = domain.VersionStatusDraft
|
||||
h.repo.Save(version)
|
||||
|
||||
respondJSON(w, http.StatusCreated, version)
|
||||
}
|
||||
```
|
||||
|
||||
// 3. Upload complete notification
|
||||
func UploadComplete(c *gin.Context) {
|
||||
// Move from temp to final location
|
||||
tempKey := fmt.Sprintf("temp/%s/package.mosis", versionID)
|
||||
finalKey := fmt.Sprintf("packages/%s/%s/%d/package.mosis",
|
||||
developerID, appID, versionCode)
|
||||
### Optional: Presigned URL for R2 CDN
|
||||
|
||||
r2.Copy(tempKey, finalKey)
|
||||
r2.Delete(tempKey)
|
||||
If global distribution is needed, sync to R2 and generate presigned download URLs:
|
||||
|
||||
// Update version status
|
||||
version.Status = "validating"
|
||||
version.PackageURL = finalKey
|
||||
```go
|
||||
// Sync to R2 after publishing (background job)
|
||||
func (s *SyncService) SyncToR2(version *domain.Version) error {
|
||||
localPath := version.PackageURL
|
||||
r2Key := fmt.Sprintf("packages/%s/%s/%d/package.mosis",
|
||||
version.DeveloperID, version.AppID, version.VersionCode)
|
||||
|
||||
// Trigger async validation
|
||||
queue.Publish("validate-package", version.ID)
|
||||
file, _ := os.Open(localPath)
|
||||
defer file.Close()
|
||||
|
||||
_, err := s.r2.PutObject(r2Key, file)
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
@@ -255,71 +342,142 @@ func UploadComplete(c *gin.Context) {
|
||||
|
||||
## Download Flow
|
||||
|
||||
### Public Downloads
|
||||
### Local File Serving
|
||||
|
||||
```go
|
||||
// Get download URL (short-lived)
|
||||
func GetDownloadURL(c *gin.Context) {
|
||||
version := getLatestPublishedVersion(packageID)
|
||||
// Serve package downloads directly from filesystem
|
||||
func (h *StoreHandler) Download(w http.ResponseWriter, r *http.Request) {
|
||||
packageID := chi.URLParam(r, "packageID")
|
||||
|
||||
// Generate presigned download URL (1 hour)
|
||||
url, _ := r2.PresignGet(version.PackageURL, 1*time.Hour)
|
||||
// Get latest published version
|
||||
version, err := h.repo.GetLatestPublished(packageID)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusNotFound, "NOT_FOUND", "App not found")
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(200, gin.H{
|
||||
"download_url": url,
|
||||
"version": version.VersionName,
|
||||
"size": version.PackageSize,
|
||||
"signature": version.Signature,
|
||||
// Serve file directly
|
||||
filePath := version.PackageURL
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusNotFound, "FILE_NOT_FOUND", "Package not available")
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
stat, _ := file.Stat()
|
||||
|
||||
// Set headers
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s-%s.mosis",
|
||||
packageID, version.VersionName))
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", stat.Size()))
|
||||
w.Header().Set("X-Mosis-Version", version.VersionName)
|
||||
w.Header().Set("X-Mosis-Signature", version.Signature)
|
||||
|
||||
// Stream file
|
||||
io.Copy(w, file)
|
||||
}
|
||||
|
||||
// Alternative: Return download info + serve via static file handler
|
||||
func (h *StoreHandler) GetDownloadInfo(w http.ResponseWriter, r *http.Request) {
|
||||
packageID := chi.URLParam(r, "packageID")
|
||||
version, _ := h.repo.GetLatestPublished(packageID)
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"download_url": fmt.Sprintf("/downloads/%s/%s/%d/package.mosis",
|
||||
version.DeveloperID, version.AppID, version.VersionCode),
|
||||
"version": version.VersionName,
|
||||
"size": version.PackageSize,
|
||||
"signature": version.Signature,
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
### CDN Caching
|
||||
### Static File Server
|
||||
|
||||
```go
|
||||
// In router setup - serve packages directory
|
||||
r.Handle("/downloads/*", http.StripPrefix("/downloads/",
|
||||
http.FileServer(http.Dir("/volume1/mosis/packages"))))
|
||||
```
|
||||
|
||||
### Caching (via Nginx or Cloudflare Tunnel)
|
||||
|
||||
```
|
||||
Cache-Control: public, max-age=86400
|
||||
```
|
||||
|
||||
- Packages are immutable (version code = unique)
|
||||
- Cache aggressively at edge
|
||||
- Invalidate only if package is pulled
|
||||
- Put Nginx in front for caching if needed
|
||||
- Or use Cloudflare Tunnel for edge caching
|
||||
|
||||
---
|
||||
|
||||
## Icon/Screenshot Handling
|
||||
|
||||
### Upload
|
||||
### Upload (Go)
|
||||
|
||||
```go
|
||||
// Icons uploaded with app creation/update
|
||||
func UploadIcon(c *gin.Context) {
|
||||
file, _ := c.FormFile("icon")
|
||||
func (h *AppHandler) UploadIcon(w http.ResponseWriter, r *http.Request) {
|
||||
appID := chi.URLParam(r, "appID")
|
||||
|
||||
// Validate dimensions
|
||||
img, _ := png.Decode(file)
|
||||
if img.Bounds().Dx() != img.Bounds().Dy() {
|
||||
return error("Icon must be square")
|
||||
r.ParseMultipartForm(10 << 20) // 10MB max
|
||||
file, _, err := r.FormFile("icon")
|
||||
if err != nil {
|
||||
respondError(w, http.StatusBadRequest, "INVALID_FILE", "No icon file")
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Decode and validate
|
||||
img, _, err := image.Decode(file)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusBadRequest, "INVALID_IMAGE", "Cannot decode image")
|
||||
return
|
||||
}
|
||||
|
||||
bounds := img.Bounds()
|
||||
if bounds.Dx() != bounds.Dy() {
|
||||
respondError(w, http.StatusBadRequest, "INVALID_DIMENSIONS", "Icon must be square")
|
||||
return
|
||||
}
|
||||
|
||||
// Generate multiple sizes
|
||||
sizes := []int{32, 64, 128}
|
||||
assetsDir := filepath.Join(h.storagePath, "assets", appID)
|
||||
os.MkdirAll(assetsDir, 0755)
|
||||
|
||||
for _, size := range sizes {
|
||||
resized := resize(img, size, size)
|
||||
key := fmt.Sprintf("assets/%s/icon-%d.png", appID, size)
|
||||
r2.Put(key, resized)
|
||||
resized := resize.Resize(uint(size), uint(size), img, resize.Lanczos3)
|
||||
outPath := filepath.Join(assetsDir, fmt.Sprintf("icon-%d.png", size))
|
||||
|
||||
out, _ := os.Create(outPath)
|
||||
png.Encode(out, resized)
|
||||
out.Close()
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]string{"status": "uploaded"})
|
||||
}
|
||||
```
|
||||
|
||||
### Serving
|
||||
|
||||
```go
|
||||
// Serve assets via static file handler
|
||||
r.Handle("/assets/*", http.StripPrefix("/assets/",
|
||||
http.FileServer(http.Dir("/volume1/mosis/assets"))))
|
||||
```
|
||||
https://cdn.mosis.dev/assets/{app_id}/icon-64.png
|
||||
|
||||
URLs:
|
||||
```
|
||||
https://portal.mosis.local/assets/{app_id}/icon-64.png
|
||||
```
|
||||
|
||||
- Public read access
|
||||
- Long cache TTL (icons rarely change)
|
||||
- Cloudflare image optimization (optional)
|
||||
- Nginx caching if needed
|
||||
|
||||
---
|
||||
|
||||
@@ -358,17 +516,36 @@ rules:
|
||||
|
||||
## Backup Strategy
|
||||
|
||||
### R2 Built-in
|
||||
### Synology Built-in Options
|
||||
|
||||
- 11 nines durability
|
||||
- Automatic replication within region
|
||||
- No additional backup needed for packages
|
||||
- **Hyper Backup** - Backup to external drive, another NAS, or cloud
|
||||
- **Snapshot Replication** - Point-in-time snapshots (Btrfs)
|
||||
- **rsync** - Script-based backup to remote location
|
||||
|
||||
### Recommended Setup
|
||||
|
||||
```bash
|
||||
# Cron job: Daily backup of packages to external drive
|
||||
0 3 * * * rsync -av /volume1/mosis/packages/ /volumeUSB1/mosis-backup/packages/
|
||||
|
||||
# Or use Synology Hyper Backup with versioning
|
||||
```
|
||||
|
||||
### Metadata Backup
|
||||
|
||||
- Package metadata in PostgreSQL
|
||||
- PostgreSQL backups cover this
|
||||
- Can regenerate URLs from DB
|
||||
- Package metadata in SQLite (portal.db)
|
||||
- Litestream handles continuous replication
|
||||
- Can regenerate file paths from DB
|
||||
|
||||
### Recovery
|
||||
|
||||
```bash
|
||||
# Restore from backup
|
||||
rsync -av /volumeUSB1/mosis-backup/packages/ /volume1/mosis/packages/
|
||||
|
||||
# Restore database (via Litestream)
|
||||
litestream restore -o /data/portal.db /backups/portal/
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
@@ -424,23 +601,22 @@ url := r2.PresignPut(key, 15*time.Minute, PutOptions{
|
||||
|
||||
## Deliverables
|
||||
|
||||
- [ ] R2 bucket setup
|
||||
- [ ] Presigned URL generation
|
||||
- [ ] Upload flow implementation
|
||||
- [ ] Download URL generation
|
||||
- [ ] Icon/screenshot upload
|
||||
- [ ] Lifecycle rules for cleanup
|
||||
- [ ] Monitoring dashboard
|
||||
- [ ] Cost tracking
|
||||
- [x] Storage approach decided (local Synology filesystem)
|
||||
- [ ] Upload flow implementation (multipart to local)
|
||||
- [ ] Download serving (http.FileServer)
|
||||
- [ ] Icon/screenshot upload and resize
|
||||
- [ ] Temp file cleanup (cron job)
|
||||
- [ ] Backup setup (Hyper Backup or rsync)
|
||||
- [ ] (Optional) R2 sync for CDN
|
||||
|
||||
---
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. Multi-region storage for lower latency?
|
||||
2. Package compression (gzip/brotli)?
|
||||
3. Delta updates storage structure?
|
||||
4. Screenshot requirements (dimensions, count)?
|
||||
1. ~~Multi-region storage for lower latency?~~ → Use R2 sync if needed
|
||||
2. ~~Package compression (gzip/brotli)?~~ → Defer, .mosis is already ZIP
|
||||
3. Delta updates storage structure? → Consider for v1.1
|
||||
4. ~~Screenshot requirements (dimensions, count)?~~ → Max 5, 1280x720 or 720x1280
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,8 +1,54 @@
|
||||
# Milestone 8: Telemetry System
|
||||
|
||||
**Status**: Planning
|
||||
**Status**: Decided
|
||||
**Goal**: Collect app usage analytics and crash reports while respecting privacy.
|
||||
|
||||
## Decision
|
||||
|
||||
**SQLite with background aggregation** for self-hosted Synology NAS:
|
||||
|
||||
```
|
||||
Storage: SQLite (separate telemetry.db to isolate write load)
|
||||
Aggregation: Go background goroutine (hourly/daily rollups)
|
||||
Retention: Raw events 7 days, aggregates indefinitely
|
||||
Privacy: Hashed device IDs, no PII, opt-out available
|
||||
```
|
||||
|
||||
### Rationale
|
||||
|
||||
1. **Simple** - No separate time-series database needed
|
||||
2. **SQLite scales** - Can handle thousands of events/day easily
|
||||
3. **Background jobs** - Go goroutines for aggregation, cleanup
|
||||
4. **Separate DB** - Telemetry writes don't affect main portal.db
|
||||
5. **Privacy-first** - Minimal collection, hashed IDs
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ mosis-portal container │
|
||||
│ ┌────────────────────────────────────────────────────────────┐ │
|
||||
│ │ Go Binary │ │
|
||||
│ │ ┌─────────────┐ ┌────────────────┐ │ │
|
||||
│ │ │ API Handler │───►│ Telemetry Svc │ │ │
|
||||
│ │ │ POST /v1/ │ │ - Buffer events│ │ │
|
||||
│ │ │ telemetry/* │ │ - Batch insert │ │ │
|
||||
│ │ └─────────────┘ └───────┬────────┘ │ │
|
||||
│ │ │ │ │
|
||||
│ │ ┌─────────────────────────▼────────────────────────────┐ │ │
|
||||
│ │ │ Background Workers │ │ │
|
||||
│ │ │ • Hourly aggregation (event counts, unique devices) │ │ │
|
||||
│ │ │ • Daily cleanup (delete raw events > 7 days) │ │ │
|
||||
│ │ │ • Crash grouping (fingerprint + dedup) │ │ │
|
||||
│ │ └───────────────────────────────────────────────────────┘ │ │
|
||||
│ └──────────────────────────────┬─────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ /volume1/mosis/data/ │ │
|
||||
│ ├── portal.db (main) │ │
|
||||
│ └── telemetry.db ◄────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
@@ -173,61 +219,107 @@ end
|
||||
|
||||
---
|
||||
|
||||
## Storage Options
|
||||
## Storage (SQLite)
|
||||
|
||||
### Option A: PostgreSQL + TimescaleDB
|
||||
### Telemetry Database Schema
|
||||
|
||||
```sql
|
||||
-- Hypertable for time-series data
|
||||
CREATE TABLE telemetry_events (
|
||||
time TIMESTAMPTZ NOT NULL,
|
||||
app_id TEXT NOT NULL,
|
||||
device_id TEXT NOT NULL,
|
||||
session_id TEXT,
|
||||
event_type TEXT NOT NULL,
|
||||
event_data JSONB,
|
||||
-- telemetry.db (separate from portal.db)
|
||||
|
||||
-- Raw events (7-day retention)
|
||||
CREATE TABLE events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
app_id TEXT NOT NULL,
|
||||
device_id TEXT NOT NULL, -- SHA256 hashed
|
||||
session_id TEXT,
|
||||
event_type TEXT NOT NULL,
|
||||
event_data TEXT, -- JSON string
|
||||
app_version TEXT,
|
||||
mosis_version TEXT
|
||||
mosis_version TEXT,
|
||||
timestamp TEXT NOT NULL -- ISO8601
|
||||
);
|
||||
|
||||
SELECT create_hypertable('telemetry_events', 'time');
|
||||
CREATE INDEX idx_events_app_time ON events(app_id, timestamp);
|
||||
CREATE INDEX idx_events_type ON events(event_type, timestamp);
|
||||
|
||||
-- Continuous aggregate for daily stats
|
||||
CREATE MATERIALIZED VIEW daily_stats
|
||||
WITH (timescaledb.continuous) AS
|
||||
SELECT
|
||||
time_bucket('1 day', time) AS day,
|
||||
app_id,
|
||||
event_type,
|
||||
COUNT(*) as count,
|
||||
COUNT(DISTINCT device_id) as unique_devices
|
||||
FROM telemetry_events
|
||||
GROUP BY day, app_id, event_type;
|
||||
-- Hourly aggregates (computed by background job)
|
||||
CREATE TABLE hourly_stats (
|
||||
app_id TEXT NOT NULL,
|
||||
hour TEXT NOT NULL, -- YYYY-MM-DDTHH
|
||||
event_type TEXT NOT NULL,
|
||||
count INTEGER NOT NULL,
|
||||
unique_devices INTEGER NOT NULL,
|
||||
PRIMARY KEY (app_id, hour, event_type)
|
||||
);
|
||||
|
||||
-- Daily aggregates (computed from hourly)
|
||||
CREATE TABLE daily_stats (
|
||||
app_id TEXT NOT NULL,
|
||||
date TEXT NOT NULL, -- YYYY-MM-DD
|
||||
event_type TEXT NOT NULL,
|
||||
count INTEGER NOT NULL,
|
||||
unique_devices INTEGER NOT NULL,
|
||||
PRIMARY KEY (app_id, date, event_type)
|
||||
);
|
||||
|
||||
-- Crash groups (deduplicated by fingerprint)
|
||||
CREATE TABLE crash_groups (
|
||||
id TEXT PRIMARY KEY,
|
||||
app_id TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
crash_type TEXT NOT NULL,
|
||||
message TEXT,
|
||||
sample_stack_trace TEXT,
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
occurrence_count INTEGER DEFAULT 1,
|
||||
affected_versions TEXT, -- JSON array
|
||||
status TEXT DEFAULT 'open',
|
||||
UNIQUE(app_id, fingerprint)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_crashes_app ON crash_groups(app_id, status);
|
||||
```
|
||||
|
||||
### Option B: ClickHouse
|
||||
### Go Background Workers
|
||||
|
||||
```sql
|
||||
CREATE TABLE telemetry_events (
|
||||
timestamp DateTime,
|
||||
app_id String,
|
||||
device_id String,
|
||||
session_id String,
|
||||
event_type String,
|
||||
event_data String, -- JSON
|
||||
app_version String,
|
||||
mosis_version String
|
||||
) ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(timestamp)
|
||||
ORDER BY (app_id, timestamp);
|
||||
```
|
||||
```go
|
||||
// Start background workers
|
||||
func (s *TelemetryService) StartWorkers(ctx context.Context) {
|
||||
// Hourly aggregation
|
||||
go s.runPeriodic(ctx, time.Hour, s.aggregateHourly)
|
||||
|
||||
### Option C: Custom + PostgreSQL
|
||||
// Daily aggregation (run at 2am)
|
||||
go s.runDaily(ctx, 2, s.aggregateDaily)
|
||||
|
||||
```
|
||||
Raw events → Write to append-only log
|
||||
Aggregator → Process hourly → Write to PostgreSQL
|
||||
Cleanup → Delete raw after 24h
|
||||
// Cleanup old events (run at 3am)
|
||||
go s.runDaily(ctx, 3, s.cleanupOldEvents)
|
||||
}
|
||||
|
||||
func (s *TelemetryService) aggregateHourly(ctx context.Context) error {
|
||||
hour := time.Now().Add(-time.Hour).Format("2006-01-02T15")
|
||||
|
||||
_, err := s.db.ExecContext(ctx, `
|
||||
INSERT OR REPLACE INTO hourly_stats (app_id, hour, event_type, count, unique_devices)
|
||||
SELECT
|
||||
app_id,
|
||||
strftime('%Y-%m-%dT%H', timestamp) as hour,
|
||||
event_type,
|
||||
COUNT(*) as count,
|
||||
COUNT(DISTINCT device_id) as unique_devices
|
||||
FROM events
|
||||
WHERE strftime('%Y-%m-%dT%H', timestamp) = ?
|
||||
GROUP BY app_id, hour, event_type
|
||||
`, hour)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *TelemetryService) cleanupOldEvents(ctx context.Context) error {
|
||||
cutoff := time.Now().AddDate(0, 0, -7).Format(time.RFC3339)
|
||||
_, err := s.db.ExecContext(ctx,
|
||||
"DELETE FROM events WHERE timestamp < ?", cutoff)
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
@@ -474,25 +566,26 @@ DELETE /v1/privacy/data:
|
||||
|
||||
## Deliverables
|
||||
|
||||
- [x] Storage approach decided (SQLite with separate telemetry.db)
|
||||
- [ ] Event schema specification
|
||||
- [ ] Client-side SDK for batching
|
||||
- [ ] Ingestion API endpoints
|
||||
- [ ] Storage setup (TimescaleDB or ClickHouse)
|
||||
- [ ] Aggregation jobs
|
||||
- [ ] Client-side batching (Lua TelemetryManager)
|
||||
- [ ] Ingestion API endpoints (Go + Chi)
|
||||
- [ ] SQLite schema and migrations
|
||||
- [ ] Background aggregation workers (Go goroutines)
|
||||
- [ ] Crash grouping logic
|
||||
- [ ] Developer dashboard
|
||||
- [ ] Privacy controls
|
||||
- [ ] Data retention automation
|
||||
- [ ] GDPR export/delete
|
||||
- [ ] Developer analytics dashboard (htmx)
|
||||
- [ ] Privacy controls (opt-out in manifest)
|
||||
- [ ] Data retention cleanup job
|
||||
- [ ] GDPR export/delete endpoints
|
||||
|
||||
---
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. Real-time crash alerts (email/Slack)?
|
||||
2. Sampling for high-volume apps?
|
||||
3. Custom events API for developers?
|
||||
4. Benchmarks/comparisons with similar apps?
|
||||
1. Real-time crash alerts? → Consider email notifications for v1.1
|
||||
2. ~~Sampling for high-volume apps?~~ → Not needed for self-hosted scale
|
||||
3. ~~Custom events API for developers?~~ → Yes, via manifest opt-in
|
||||
4. ~~Benchmarks/comparisons with similar apps?~~ → Defer to post-MVP
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,8 +1,60 @@
|
||||
# Milestone 9: App Review System
|
||||
|
||||
**Status**: Planning
|
||||
**Status**: Decided
|
||||
**Goal**: Automated and manual review process for app submissions.
|
||||
|
||||
## Decision
|
||||
|
||||
**Go validation workers + SQLite** for self-hosted review pipeline:
|
||||
|
||||
```
|
||||
Validation: Go workers with concurrent file processing
|
||||
Storage: SQLite (review state in portal.db)
|
||||
Queue: In-memory channel + SQLite persistence
|
||||
UI: htmx server-rendered pages (admin section)
|
||||
```
|
||||
|
||||
### Rationale
|
||||
|
||||
1. **Go concurrency** - Process multiple files in parallel with goroutines
|
||||
2. **Single binary** - No separate queue service needed
|
||||
3. **Simple state** - Review state in SQLite alongside app data
|
||||
4. **htmx admin UI** - Server-rendered review queue, no SPA needed
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ mosis-portal container │
|
||||
│ ┌────────────────────────────────────────────────────────────┐ │
|
||||
│ │ Go Binary │ │
|
||||
│ │ ┌─────────────┐ ┌────────────────┐ │ │
|
||||
│ │ │ Upload API │───►│ Review Service │ │ │
|
||||
│ │ │ POST /v1/ │ │ - Queue submit │ │ │
|
||||
│ │ │ versions │ │ - Track state │ │ │
|
||||
│ │ └─────────────┘ └───────┬────────┘ │ │
|
||||
│ │ │ │ │
|
||||
│ │ ┌─────────────────────────▼────────────────────────────┐ │ │
|
||||
│ │ │ Validation Worker Pool │ │ │
|
||||
│ │ │ • Tier 1: Package validation (ZIP, manifest, sig) │ │ │
|
||||
│ │ │ • Tier 2: Content validation (RML, RCSS, Lua) │ │ │
|
||||
│ │ │ • Tier 3: Security analysis (patterns, perms) │ │ │
|
||||
│ │ │ • Tier 4: Quality checks (description, icons) │ │ │
|
||||
│ │ └───────────────────────────────────────────────────────┘ │ │
|
||||
│ │ │ │ │
|
||||
│ │ ┌─────────────────────────▼────────────────────────────┐ │ │
|
||||
│ │ │ Admin Review UI (htmx) │ │ │
|
||||
│ │ │ • /admin/review-queue │ │ │
|
||||
│ │ │ • /admin/review/:id │ │ │
|
||||
│ │ └───────────────────────────────────────────────────────┘ │ │
|
||||
│ └──────────────────────────────┬─────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ /volume1/mosis/ │ │
|
||||
│ ├── data/portal.db ◄───────────┘ │
|
||||
│ └── packages/{dev}/{app}/{ver}/ (validation target) │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
@@ -448,23 +500,23 @@ GROUP BY week;
|
||||
|
||||
## Deliverables
|
||||
|
||||
- [ ] Validation worker implementation
|
||||
- [ ] Dangerous pattern database
|
||||
- [ ] Review queue UI
|
||||
- [ ] Reviewer tools
|
||||
- [x] Review approach decided (Go workers + SQLite + htmx admin)
|
||||
- [ ] Validation worker implementation (Go concurrent file processing)
|
||||
- [ ] Dangerous pattern database (regex patterns in code)
|
||||
- [ ] Review queue UI (htmx server-rendered)
|
||||
- [ ] Reviewer tools (file browser, source viewer)
|
||||
- [ ] Rejection feedback system
|
||||
- [ ] Appeal workflow
|
||||
- [ ] Review metrics dashboard
|
||||
- [ ] SLA monitoring
|
||||
- [ ] Review metrics queries
|
||||
|
||||
---
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. Automated approval for trusted developers?
|
||||
2. Community moderators?
|
||||
3. Content policy document?
|
||||
4. Rate limiting resubmissions?
|
||||
1. ~~Automated approval for trusted developers?~~ → Yes, after 3+ approved apps
|
||||
2. ~~Community moderators?~~ → Defer to post-MVP (single admin for now)
|
||||
3. Content policy document? → Create during M12 Docs
|
||||
4. ~~Rate limiting resubmissions?~~ → Max 3 resubmits per day per app
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,8 +1,43 @@
|
||||
# Milestone 10: Device-Side App Management
|
||||
|
||||
**Status**: Planning
|
||||
**Status**: Decided
|
||||
**Goal**: Install, update, and manage apps on Mosis devices.
|
||||
|
||||
## Decision
|
||||
|
||||
**C++ AppManager + Lua bindings** running on MosisService:
|
||||
|
||||
```
|
||||
AppManager: C++ class managing installation/updates
|
||||
Storage: Local device storage (/data/mosis/apps/)
|
||||
Updates: Background service checking Portal API
|
||||
UI: App Store system app (RML/Lua)
|
||||
API: Connects to Portal at portal.mosis.dev (or self-hosted)
|
||||
```
|
||||
|
||||
### Rationale
|
||||
|
||||
1. **Native C++** - AppManager runs in MosisService process for performance
|
||||
2. **Background updates** - UpdateService thread checks Portal API periodically
|
||||
3. **System app** - App Store is a privileged RML/Lua app with special permissions
|
||||
4. **Ed25519 verification** - All packages verified before installation
|
||||
|
||||
### API Integration
|
||||
|
||||
```
|
||||
Device Portal (Synology NAS)
|
||||
┌──────────────┐ ┌──────────────────────┐
|
||||
│ MosisService │ │ mosis-portal │
|
||||
│ │ │ │
|
||||
│ UpdateService├──────GET /store/apps────►│ Chi API Router │
|
||||
│ │ /updates?pkgs=... │ │
|
||||
│ │◄─────{updates: [...]}───┤ SQLite portal.db │
|
||||
│ │ │ │
|
||||
│ AppManager ├──────GET /packages/...──►│ /volume1/mosis/ │
|
||||
│ │◄─────[package.mosis]────┤ packages/{dev}/... │
|
||||
└──────────────┘ └──────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
@@ -577,9 +612,10 @@ adb shell am broadcast -a com.omixlab.mosis.INSTALL_APP \
|
||||
|
||||
## Deliverables
|
||||
|
||||
- [x] Architecture decided (C++ AppManager + Lua bindings)
|
||||
- [ ] AppManager C++ class
|
||||
- [ ] UpdateService background checker
|
||||
- [ ] App Store system app
|
||||
- [ ] App Store system app (RML/Lua)
|
||||
- [ ] Lua API bindings (mosis.apps, mosis.app)
|
||||
- [ ] Installation progress UI
|
||||
- [ ] Uninstall confirmation UI
|
||||
@@ -590,10 +626,10 @@ adb shell am broadcast -a com.omixlab.mosis.INSTALL_APP \
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. App backup to cloud?
|
||||
2. Family sharing / multiple devices?
|
||||
3. Enterprise MDM integration?
|
||||
4. Sideloading policy?
|
||||
1. ~~App backup to cloud?~~ → Defer to post-MVP (local backups only)
|
||||
2. ~~Family sharing / multiple devices?~~ → Defer to post-MVP
|
||||
3. ~~Enterprise MDM integration?~~ → Not needed for self-hosted
|
||||
4. ~~Sideloading policy?~~ → Enabled via Settings toggle (developer mode)
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,8 +1,52 @@
|
||||
# Milestone 11: Developer CLI Tool
|
||||
|
||||
**Status**: Planning
|
||||
**Status**: Decided
|
||||
**Goal**: Command-line tool for app development workflow.
|
||||
|
||||
## Decision
|
||||
|
||||
**Go + Cobra** for single-binary cross-platform CLI:
|
||||
|
||||
```
|
||||
Framework: Cobra (github.com/spf13/cobra)
|
||||
Distribution: Single binary (no runtime dependencies)
|
||||
Signing: crypto/ed25519 (stdlib)
|
||||
Auth: OAuth2 device flow + API key storage
|
||||
Portal URL: Configurable (default: self-hosted Synology)
|
||||
```
|
||||
|
||||
### Rationale
|
||||
|
||||
1. **Consistency** - Same language as Portal backend (Go)
|
||||
2. **Single binary** - No Node.js/Python runtime needed
|
||||
3. **Fast** - Compiles to native code, instant startup
|
||||
4. **Cross-platform** - Build for Windows, macOS, Linux from one codebase
|
||||
5. **Cobra ecosystem** - Shell completions, man pages, help generation
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ mosis CLI │
|
||||
│ ┌─────────────────────────────────────────────────────────────┐│
|
||||
│ │ Cobra Commands ││
|
||||
│ │ ├── init → Template generation ││
|
||||
│ │ ├── build → ZIP package creation ││
|
||||
│ │ ├── sign → Ed25519 signing (crypto/ed25519) ││
|
||||
│ │ ├── run → Launch mosis-designer subprocess ││
|
||||
│ │ ├── login → OAuth2 device flow → ~/.mosis/credentials ││
|
||||
│ │ └── publish → HTTP client → Portal API ││
|
||||
│ └───────────────────────────────────────────────────────────────┘│
|
||||
│ │ │
|
||||
│ ~/.mosis/ │ │
|
||||
│ ├── config.json │ Portal (Synology NAS) │
|
||||
│ ├── credentials │ ┌──────────────────────────┐ │
|
||||
│ ├── signing_key.pem ────────┼──│ POST /v1/versions │ │
|
||||
│ └── signing_key.pub │ │ POST /auth/device │ │
|
||||
│ │ └──────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
@@ -383,92 +427,161 @@ key will be accepted for review.
|
||||
|
||||
---
|
||||
|
||||
## Implementation
|
||||
## Implementation (Go + Cobra)
|
||||
|
||||
### Tech Stack Options
|
||||
|
||||
#### Option A: Go
|
||||
### Main Entry Point
|
||||
|
||||
```go
|
||||
// Using cobra for CLI framework
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func main() {
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "mosis",
|
||||
Short: "Mosis app development CLI",
|
||||
Long: "CLI tool for building, signing, and publishing Mosis apps",
|
||||
}
|
||||
|
||||
// Global flags
|
||||
rootCmd.PersistentFlags().StringP("portal", "p", "", "Portal URL (default from config)")
|
||||
rootCmd.PersistentFlags().BoolP("verbose", "v", false, "Verbose output")
|
||||
|
||||
// Commands
|
||||
rootCmd.AddCommand(initCmd())
|
||||
rootCmd.AddCommand(buildCmd())
|
||||
rootCmd.AddCommand(signCmd())
|
||||
rootCmd.AddCommand(runCmd())
|
||||
rootCmd.AddCommand(loginCmd())
|
||||
rootCmd.AddCommand(publishCmd())
|
||||
// ...
|
||||
rootCmd.AddCommand(statusCmd())
|
||||
rootCmd.AddCommand(keysCmd())
|
||||
rootCmd.AddCommand(configCmd())
|
||||
|
||||
rootCmd.Execute()
|
||||
}
|
||||
```
|
||||
|
||||
**Pros**: Single binary, fast, cross-platform
|
||||
**Cons**: More code to write
|
||||
|
||||
#### Option B: Node.js (oclif)
|
||||
|
||||
```typescript
|
||||
// Using oclif framework
|
||||
import { Command } from '@oclif/core'
|
||||
|
||||
export default class Build extends Command {
|
||||
static description = 'Build .mosis package'
|
||||
|
||||
async run() {
|
||||
const manifest = await this.readManifest()
|
||||
const files = await this.collectFiles()
|
||||
const package = await this.createPackage(files)
|
||||
this.log(`✓ Package created: ${package.path}`)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Pros**: Fast development, npm distribution
|
||||
**Cons**: Requires Node.js runtime
|
||||
|
||||
#### Option C: Rust (clap)
|
||||
|
||||
```rust
|
||||
use clap::{Parser, Subcommand};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "mosis")]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Commands,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Commands {
|
||||
Init { name: Option<String> },
|
||||
Build { output: Option<PathBuf> },
|
||||
Sign { package: PathBuf },
|
||||
Publish,
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let cli = Cli::parse();
|
||||
match cli.command {
|
||||
Commands::Init { name } => init::run(name),
|
||||
Commands::Build { output } => build::run(output),
|
||||
// ...
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Pros**: Single binary, very fast
|
||||
**Cons**: Slower development
|
||||
### Build Command Example
|
||||
|
||||
```go
|
||||
func buildCmd() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "build",
|
||||
Short: "Create .mosis package",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
manifest, err := readManifest("manifest.json")
|
||||
if err != nil {
|
||||
return fmt.Errorf("manifest.json not found\n\nAre you in a Mosis project directory?")
|
||||
}
|
||||
|
||||
output, _ := cmd.Flags().GetString("output")
|
||||
if output == "" {
|
||||
output = fmt.Sprintf("dist/%s-%s.mosis", manifest.PackageID, manifest.Version)
|
||||
}
|
||||
|
||||
fmt.Printf("Building %s v%s...\n", manifest.Name, manifest.Version)
|
||||
|
||||
files, err := collectFiles(manifest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := createPackage(files, output); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
info, _ := os.Stat(output)
|
||||
fmt.Printf("✓ Package created: %s (%.1f KB)\n", output, float64(info.Size())/1024)
|
||||
fmt.Println("\n⚠ Package is unsigned. Run 'mosis sign' before publishing.")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().StringP("output", "o", "", "Output path (default: dist/)")
|
||||
return cmd
|
||||
}
|
||||
```
|
||||
|
||||
### OAuth2 Device Flow (Login)
|
||||
|
||||
```go
|
||||
func loginCmd() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "login",
|
||||
Short: "Authenticate with developer portal",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
portalURL := viper.GetString("portal_url")
|
||||
|
||||
// Start device flow
|
||||
resp, err := http.Post(portalURL+"/auth/device", "application/json", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var device DeviceResponse
|
||||
json.NewDecoder(resp.Body).Decode(&device)
|
||||
|
||||
fmt.Printf("Go to: %s\n", device.VerificationURI)
|
||||
fmt.Printf("Enter code: %s\n\n", device.UserCode)
|
||||
fmt.Println("Waiting for authorization...")
|
||||
|
||||
// Poll for token
|
||||
token, err := pollForToken(portalURL, device.DeviceCode, device.Interval)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Save credentials
|
||||
if err := saveCredentials(token); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("✓ Logged in as %s\n", token.Email)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Ed25519 Signing
|
||||
|
||||
```go
|
||||
func signPackage(packagePath, keyPath string) error {
|
||||
// Read private key
|
||||
keyPEM, err := os.ReadFile(keyPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read key: %w", err)
|
||||
}
|
||||
|
||||
block, _ := pem.Decode(keyPEM)
|
||||
privateKey, err := x509.ParsePKCS8PrivateKey(block.Bytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid key format: %w", err)
|
||||
}
|
||||
|
||||
ed25519Key := privateKey.(ed25519.PrivateKey)
|
||||
|
||||
// Generate MANIFEST.MF with file hashes
|
||||
manifest, err := generateManifest(packagePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sign manifest
|
||||
signature := ed25519.Sign(ed25519Key, manifest)
|
||||
|
||||
// Add signature to package
|
||||
return addSignatureToPackage(packagePath, manifest, signature)
|
||||
}
|
||||
|
||||
---
|
||||
|
||||
@@ -577,28 +690,28 @@ jobs:
|
||||
|
||||
## Deliverables
|
||||
|
||||
- [ ] CLI framework selection
|
||||
- [ ] `init` command
|
||||
- [x] CLI framework selected (Go + Cobra)
|
||||
- [ ] `init` command (template generation)
|
||||
- [ ] `validate` command
|
||||
- [ ] `build` command
|
||||
- [ ] `sign` command
|
||||
- [ ] `run` command (designer integration)
|
||||
- [ ] `login/logout` commands
|
||||
- [ ] `publish` command
|
||||
- [ ] `build` command (ZIP package creation)
|
||||
- [ ] `sign` command (Ed25519 signing)
|
||||
- [ ] `run` command (designer subprocess)
|
||||
- [ ] `login/logout` commands (OAuth2 device flow)
|
||||
- [ ] `publish` command (HTTP upload to Portal)
|
||||
- [ ] `status` command
|
||||
- [ ] `keys` subcommands
|
||||
- [ ] Configuration management
|
||||
- [ ] Distribution packages
|
||||
- [ ] CI/CD examples
|
||||
- [ ] `keys` subcommands (generate, register, list)
|
||||
- [ ] Configuration management (viper)
|
||||
- [ ] Cross-platform builds (goreleaser)
|
||||
- [ ] CI/CD examples (GitHub Actions)
|
||||
|
||||
---
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. Should CLI auto-update itself?
|
||||
2. Offline mode for build/sign?
|
||||
3. Plugin system for custom commands?
|
||||
4. IDE integrations (VS Code extension)?
|
||||
1. ~~Should CLI auto-update itself?~~ → No, manual updates via package manager
|
||||
2. ~~Offline mode for build/sign?~~ → Yes, build/sign work offline
|
||||
3. ~~Plugin system for custom commands?~~ → Defer to post-MVP
|
||||
4. IDE integrations (VS Code extension)? → Consider for v1.1
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,8 +1,59 @@
|
||||
# Milestone 12: Documentation Site
|
||||
|
||||
**Status**: Planning
|
||||
**Status**: Decided
|
||||
**Goal**: Comprehensive documentation for Mosis app developers.
|
||||
|
||||
## Decision
|
||||
|
||||
**Hugo + Docsy theme** for self-hosted static documentation:
|
||||
|
||||
```
|
||||
Framework: Hugo (Go-based static site generator)
|
||||
Theme: Docsy (technical documentation theme)
|
||||
Search: Pagefind (local, no external service)
|
||||
Hosting: Synology NAS (nginx or Go static server)
|
||||
```
|
||||
|
||||
### Rationale
|
||||
|
||||
1. **Go ecosystem** - Hugo is written in Go, consistent with Portal
|
||||
2. **Fast builds** - Hugo compiles thousands of pages in seconds
|
||||
3. **No runtime** - Generates static HTML, served directly from NAS
|
||||
4. **Docsy theme** - Full-featured docs theme with versioning, search, i18n
|
||||
5. **Self-contained** - Pagefind search works offline, no Algolia needed
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Synology NAS │
|
||||
│ ┌────────────────────────────────────────────────────────────┐ │
|
||||
│ │ mosis-portal container │ │
|
||||
│ │ ├── Go binary (API + Portal UI) │ │
|
||||
│ │ ├── /static/docs/ → Hugo build output │ │
|
||||
│ │ └── http.FileServer serves docs at /docs/* │ │
|
||||
│ └────────────────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
│ /volume1/mosis/ │
|
||||
│ └── docs/ │
|
||||
│ ├── content/ (Markdown source) │
|
||||
│ ├── static/ (Images, assets) │
|
||||
│ └── public/ (Hugo build output → served) │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
|
||||
Build pipeline:
|
||||
docs/ (Markdown) ──► hugo build ──► public/ ──► Deploy to NAS
|
||||
```
|
||||
|
||||
### URL Structure
|
||||
|
||||
```
|
||||
https://portal.mosis.dev/docs/ # Docs home
|
||||
https://portal.mosis.dev/docs/quickstart/ # Getting started
|
||||
https://portal.mosis.dev/docs/api/ # API reference
|
||||
https://portal.mosis.dev/docs/cli/ # CLI reference
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
@@ -557,18 +608,18 @@ function submitFeedback(page, helpful, comment) {
|
||||
|
||||
## Deliverables
|
||||
|
||||
- [ ] Framework selection
|
||||
- [ ] Information architecture
|
||||
- [ ] Getting Started content
|
||||
- [ ] UI design guides
|
||||
- [x] Framework selected (Hugo + Docsy)
|
||||
- [x] Hosting decided (self-hosted on Synology NAS)
|
||||
- [ ] Hugo project setup with Docsy theme
|
||||
- [ ] Information architecture (directory structure)
|
||||
- [ ] Getting Started content (Quick Start, First App)
|
||||
- [ ] UI design guides (RML, RCSS)
|
||||
- [ ] Lua scripting guides
|
||||
- [ ] API reference (all namespaces)
|
||||
- [ ] CLI reference
|
||||
- [ ] Best practices
|
||||
- [ ] Search integration
|
||||
- [ ] Version selector
|
||||
- [ ] Deploy pipeline
|
||||
- [ ] Feedback system
|
||||
- [ ] CLI reference (all commands)
|
||||
- [ ] Best practices (performance, security)
|
||||
- [ ] Pagefind search integration
|
||||
- [ ] Deploy script (hugo build + copy to NAS)
|
||||
|
||||
---
|
||||
|
||||
@@ -602,10 +653,10 @@ function submitFeedback(page, helpful, comment) {
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. Host docs separately or under main domain?
|
||||
2. Community wiki/contributions?
|
||||
3. Video tutorial platform (YouTube, embedded)?
|
||||
4. Glossary/terminology page?
|
||||
1. ~~Host docs separately or under main domain?~~ → Under main domain at /docs/
|
||||
2. ~~Community wiki/contributions?~~ → Defer to post-MVP (GitHub PRs for docs)
|
||||
3. Video tutorial platform (YouTube, embedded)? → Consider for v1.1
|
||||
4. ~~Glossary/terminology page?~~ → Yes, include in Phase 2
|
||||
|
||||
---
|
||||
|
||||
|
||||
Reference in New Issue
Block a user