From 2e097e4e5449aa273222ed7cd37ce087873f965f Mon Sep 17 00:00:00 2001 From: omigamedev Date: Fri, 16 Jan 2026 19:50:59 +0100 Subject: [PATCH] add testing and plan milestones --- MILESTONE-2.md | 186 +++++++++ MILESTONE-3.md | 333 ++++++++++++++++ MILESTONE-4.md | 403 +++++++++++++++++++ MILESTONE-5.md | 407 +++++++++++++++++++ MILESTONE-6.md | 500 ++++++++++++++++++++++++ MILESTONE-7.md | 496 +++++++++++++++++++++++ designer/src/main.cpp | 94 ++++- designer/src/testing/visual_capture.cpp | 149 +++++-- 8 files changed, 2540 insertions(+), 28 deletions(-) create mode 100644 MILESTONE-2.md create mode 100644 MILESTONE-3.md create mode 100644 MILESTONE-4.md create mode 100644 MILESTONE-5.md create mode 100644 MILESTONE-6.md create mode 100644 MILESTONE-7.md diff --git a/MILESTONE-2.md b/MILESTONE-2.md new file mode 100644 index 0000000..20afc61 --- /dev/null +++ b/MILESTONE-2.md @@ -0,0 +1,186 @@ +# Milestone 2: Testing Framework + +**Status**: 95% Complete +**Goal**: Automated UI testing for rapid iteration and AI agent verification. + +--- + +## Overview + +The testing framework enables automated validation of UI behavior through: +- UI hierarchy inspection (JSON dump) +- Input simulation and recording +- Log verification for navigation events +- Visual regression testing via screenshot diff +- JSON test results compatible with CI/CD + +--- + +## Current State + +### Completed Components + +| Component | File | Status | +|-----------|------|--------| +| Action Types | `designer/src/testing/action_types.h` | Complete | +| Action Recorder | `designer/src/testing/action_recorder.h/.cpp` | Complete | +| Action Player | `designer/src/testing/action_player.h/.cpp` | Complete | +| UI Inspector | `designer/src/testing/ui_inspector.cpp` | Complete | +| Visual Capture | `designer/src/testing/visual_capture.cpp` | Complete | +| Test Runner | `designer-test/src/test_runner.cpp` | Complete | +| CLI Integration | `designer/src/main.cpp` | Complete | + +### Already Working + +1. **Action Types** (`action_types.h`) + - TapAction, SwipeAction, LongPressAction + - ButtonAction, WaitAction, KeyAction + - ActionSequence container with metadata + +2. **Action Recorder** (`action_recorder.h/.cpp`) + - Mouse down/up/move tracking + - Automatic gesture classification (tap vs swipe vs long press) + - JSON serialization/deserialization + - Configurable thresholds + +3. **Action Player** (`action_player.h/.cpp`) + - Timestamp-based playback + - RmlUi event injection + - Pause/resume/stop controls + - Progress tracking + +4. **UI Inspector** (`ui_inspector.cpp`) + - Full element tree traversal + - JSON export with bounds, IDs, classes + - Continuous hierarchy dumping mode + +5. **Test Runner** (`designer-test/`) + - WindowController (Windows SendInput) + - HierarchyReader (element lookup) + - LogParser (navigation verification) + - All 5 navigation tests passing + +--- + +## Recently Completed + +### Screenshot Diff Implementation (Complete) + +**File**: `designer/src/testing/visual_capture.cpp` + +Implemented real PNG pixel-by-pixel comparison: +- `LoadPNG()` helper function loads PNG files using libpng +- `PixelsMatch()` compares pixels with configurable tolerance (default: 2) +- `CompareImages()` returns difference ratio (0.0 = identical, 1.0 = completely different) + +### Recording/Playback CLI (Complete) + +**File**: `designer/src/main.cpp` + +Added CLI options and keyboard controls: +- `--record ` - Enable recording mode +- `--playback ` - Play back recorded actions +- F5 - Start/stop recording (saves to specified file) +- F6 - Pause/resume playback + +**Usage**: +```bash +# Record interactions +mosis-designer.exe home.rml --record my-test.json +# Press F5 to start recording, interact, press F5 again to save + +# Play back +mosis-designer.exe home.rml --playback my-test.json +``` + +--- + +## Remaining Task + +### Task 2.3: GLFW Input Hooks for Recording + +**Status**: Partially Complete +**Effort**: Medium +**Limitation**: Requires RmlUi Backend modification + +**Current State**: +- Recording infrastructure is complete (ActionRecorder) +- Playback works fully (ActionPlayer calls RmlUi context directly) +- CLI and keyboard controls are wired up +- **Missing**: Direct GLFW callback access for mouse recording + +**The Problem**: +The RmlUi Backend abstraction handles all GLFW callbacks internally and doesn't expose them for interception. To record actual mouse events, we would need to either: + +1. **Modify RmlUi Backend** (third-party code) + - Add callback hooks to `RmlUi_Backend_GLFW_GL3.cpp` + - Expose GLFW window handle for custom callbacks + +2. **Fork RmlUi Backends** (more maintainable) + - Copy Backend files into designer project + - Add recording hooks + +3. **Alternative: Element-Based Recording** + - Listen to RmlUi events after processing + - Record element clicks by ID rather than coordinates + - Less precise but avoids backend modification + +**Workaround for Now**: +Tests can be created manually by: +1. Using the UI hierarchy to find element coordinates +2. Writing JSON test files directly +3. Using the external designer-test framework (Windows SendInput) + +**Future Work**: +Consider option 2 (fork backends) when recording becomes a priority. + +--- + +## File Changes Summary + +| File | Changes | Status | +|------|---------|--------| +| `designer/src/testing/visual_capture.cpp` | Real PNG pixel comparison | Done | +| `designer/src/main.cpp` | --record/--playback CLI, F5/F6 keys | Done | +| `designer/src/RmlUi_Backend.h` | Expose GLFW window for recording | Future | + +--- + +## Test Recording Format + +```json +{ + "name": "Navigate to contacts and back", + "description": "Test navigation flow", + "screen_width": 540, + "screen_height": 960, + "initial_screen": "apps/home/home.rml", + "actions": [ + {"type": "tap", "x": 413, "y": 1174, "timestamp": 0}, + {"type": "wait", "duration": 1000, "timestamp": 100}, + {"type": "tap", "x": 40, "y": 28, "timestamp": 1100}, + {"type": "swipe", "x1": 100, "y1": 500, "x2": 100, "y2": 200, "duration": 300, "timestamp": 2000} + ] +} +``` + +--- + +## Acceptance Criteria + +- [x] `CompareImages()` returns accurate pixel difference ratio +- [ ] `--record ` captures all mouse/key events to JSON (needs GLFW hooks) +- [x] `--playback ` replays recorded actions with correct timing +- [x] Recording stops gracefully on F5 or window close +- [x] Playback shows progress in console +- [x] Screenshot diff with 2-pixel tolerance per channel + +--- + +## Future Enhancements (Not This Milestone) + +- Visual diff output (highlight changed pixels) +- Parallel test execution +- Android action recording/playback +- Cross-platform test runner +- Coverage reporting diff --git a/MILESTONE-3.md b/MILESTONE-3.md new file mode 100644 index 0000000..33d9848 --- /dev/null +++ b/MILESTONE-3.md @@ -0,0 +1,333 @@ +# Milestone 3: Virtual Hardware + +**Status**: Not Started +**Goal**: Hardware-like APIs backed by game engine or real devices. + +--- + +## Overview + +Mosis needs to expose smartphone-like hardware interfaces that can be: +- Provided by game engines (Unity/Unreal) in VR mode +- Mocked for desktop testing +- Connected to real device hardware on Android + +--- + +## Components + +### 3.1 Camera Interface + +**Header**: `src/main/kernel/include/camera.h` + +```cpp +namespace mosis { + +struct CameraFrame { + int width; + int height; + std::vector rgba_data; // RGBA8 format + int64_t timestamp_ms; +}; + +using CameraCallback = std::function; + +class ICamera { +public: + virtual ~ICamera() = default; + + // Start/stop frame capture + virtual void StartCapture(CameraCallback callback) = 0; + virtual void StopCapture() = 0; + + // Configuration + virtual void SetResolution(int width, int height) = 0; + virtual bool IsAvailable() const = 0; + + // Get current state + virtual int GetWidth() const = 0; + virtual int GetHeight() const = 0; +}; + +} // namespace mosis +``` + +**Implementations**: + +| Implementation | Description | File | +|----------------|-------------|------| +| `GameCamera` | Receives texture from Unity/Unreal | `game_camera.cpp` | +| `DesktopCamera` | System webcam via OpenCV (optional) | `desktop_camera.cpp` | +| `AndroidCamera` | Camera2 API integration | `android_camera.cpp` | +| `MockCamera` | Test patterns (checkerboard, gradient) | `mock_camera.cpp` | + +### 3.2 Microphone Interface + +**Header**: `src/main/kernel/include/microphone.h` + +```cpp +namespace mosis { + +struct AudioBuffer { + std::vector samples; // PCM 16-bit + int sample_rate; // Typically 44100 or 48000 + int channels; // 1 = mono, 2 = stereo + int64_t timestamp_ms; +}; + +using AudioCallback = std::function; + +class IMicrophone { +public: + virtual ~IMicrophone() = default; + + virtual void StartCapture(AudioCallback callback) = 0; + virtual void StopCapture() = 0; + + virtual void SetSampleRate(int rate) = 0; + virtual bool IsAvailable() const = 0; + virtual bool IsCapturing() const = 0; +}; + +} // namespace mosis +``` + +**Implementations**: + +| Implementation | Description | +|----------------|-------------| +| `GameMicrophone` | Audio from Unity/Unreal AudioSource | +| `DesktopMicrophone` | System mic via PortAudio | +| `AndroidMicrophone` | AudioRecord API | +| `MockMicrophone` | Silence or test tones | + +### 3.3 Speaker Interface + +**Header**: `src/main/kernel/include/speaker.h` + +```cpp +namespace mosis { + +class ISpeaker { +public: + virtual ~ISpeaker() = default; + + virtual void PlayAudio(const AudioBuffer& buffer) = 0; + virtual void SetVolume(float volume) = 0; // 0.0 - 1.0 + virtual float GetVolume() const = 0; + + virtual bool IsAvailable() const = 0; +}; + +} // namespace mosis +``` + +### 3.4 Filesystem Interface + +**Header**: `src/main/kernel/include/filesystem.h` + +```cpp +namespace mosis { + +enum class FileMode { Read, Write, Append }; + +struct FileInfo { + std::string name; + bool is_directory; + size_t size; + int64_t modified_time; +}; + +class IFileSystem { +public: + virtual ~IFileSystem() = default; + + // File operations + virtual std::vector ReadFile(const std::string& path) = 0; + virtual bool WriteFile(const std::string& path, const std::vector& data) = 0; + virtual bool DeleteFile(const std::string& path) = 0; + virtual bool FileExists(const std::string& path) = 0; + + // Directory operations + virtual std::vector ListDirectory(const std::string& path) = 0; + virtual bool CreateDirectory(const std::string& path) = 0; + virtual bool DeleteDirectory(const std::string& path) = 0; + + // Sandboxed paths + virtual std::string GetAppDataPath(const std::string& app_id) = 0; + virtual std::string GetSharedMediaPath() = 0; +}; + +} // namespace mosis +``` + +**App Storage Structure**: +``` +/data/ +├── apps/ +│ ├── com.example.app1/ +│ │ ├── files/ +│ │ ├── cache/ +│ │ └── databases/ +│ └── com.example.app2/ +├── shared/ +│ ├── photos/ +│ ├── downloads/ +│ └── music/ +``` + +### 3.5 Network Interface + +**Header**: `src/main/kernel/include/network.h` + +```cpp +namespace mosis { + +struct HttpRequest { + std::string method; // GET, POST, PUT, DELETE + std::string url; + std::map headers; + std::vector body; +}; + +struct HttpResponse { + int status_code; + std::map headers; + std::vector body; +}; + +class INetwork { +public: + virtual ~INetwork() = default; + + // HTTP + using HttpCallback = std::function; + virtual void Fetch(const HttpRequest& request, HttpCallback callback) = 0; + + // Connectivity + virtual bool IsOnline() const = 0; + virtual std::string GetConnectionType() const = 0; // "wifi", "cellular", "none" +}; + +} // namespace mosis +``` + +--- + +## Platform Integration + +### Platform Interface Extension + +**File**: `src/main/kernel/include/platform.h` + +```cpp +class IPlatform { +public: + // ... existing methods ... + + // Hardware providers + virtual ICamera* GetCamera() = 0; + virtual IMicrophone* GetMicrophone() = 0; + virtual ISpeaker* GetSpeaker() = 0; + virtual IFileSystem* GetFileSystem() = 0; + virtual INetwork* GetNetwork() = 0; +}; +``` + +### Desktop Implementation + +**File**: `designer/src/desktop_platform.cpp` + +```cpp +class DesktopPlatform : public IPlatform { + std::unique_ptr m_camera; + std::unique_ptr m_microphone; + std::unique_ptr m_speaker; + std::unique_ptr m_filesystem; + std::unique_ptr m_network; + +public: + ICamera* GetCamera() override { return m_camera.get(); } + // ... +}; +``` + +### Android Implementation + +**File**: `src/main/cpp/android_platform.cpp` + +```cpp +class AndroidPlatform : public IPlatform { + // Use JNI to access Android APIs + std::unique_ptr m_camera; + // ... +}; +``` + +### Game Engine Implementation + +**File**: Unity plugin or Unreal plugin + +```cpp +class GamePlatform : public IPlatform { + // Receives textures/audio from game engine + std::unique_ptr m_camera; + // ... +}; +``` + +--- + +## Implementation Plan + +### Phase 1: Interfaces Only +- [ ] Define all interface headers +- [ ] Add to platform abstraction +- [ ] Create mock implementations for testing + +### Phase 2: Desktop Implementation +- [ ] MockCamera (test patterns) +- [ ] PortAudio for speaker output +- [ ] Standard filesystem access +- [ ] libcurl for HTTP + +### Phase 3: Android Implementation +- [ ] Camera2 API wrapper (JNI) +- [ ] AudioRecord/AudioTrack wrappers +- [ ] Android filesystem with proper sandboxing +- [ ] OkHttp or native networking + +### Phase 4: Game Engine Integration +- [ ] Unity RenderTexture → ICamera +- [ ] Unity AudioSource → IMicrophone +- [ ] Unity AudioListener → ISpeaker +- [ ] Unreal equivalents + +--- + +## Dependencies + +| Dependency | Purpose | vcpkg Package | +|------------|---------|---------------| +| PortAudio | Desktop audio I/O | `portaudio` | +| OpenCV | Desktop webcam (optional) | `opencv4` | +| libcurl | HTTP client | `curl` | + +--- + +## Test Cases + +1. **MockCamera**: Renders test pattern, verify frame callback +2. **FileSystem**: Create, read, write, delete operations +3. **Network**: Mock HTTP responses, verify request/response +4. **Audio**: Verify sample rates, buffer formats + +--- + +## Acceptance Criteria + +- [ ] All interfaces defined in kernel/include/ +- [ ] Mock implementations work on desktop +- [ ] Camera app can display camera frames +- [ ] Browser app can make HTTP requests +- [ ] Apps can persist data to filesystem diff --git a/MILESTONE-4.md b/MILESTONE-4.md new file mode 100644 index 0000000..15b4fbb --- /dev/null +++ b/MILESTONE-4.md @@ -0,0 +1,403 @@ +# Milestone 4: App Sandboxing + +**Status**: Not Started +**Goal**: Secure app runtime with defined package format and permission system. + +--- + +## Overview + +Apps in Mosis need: +- Isolation from each other and the system +- Defined package format for distribution +- Permission model for hardware/data access +- Lifecycle management + +--- + +## Runtime Architecture + +### Recommendation: Hybrid Approach + +| Component | Runtime | Reason | +|-----------|---------|--------| +| UI Scripts | Lua | Native RmlUi integration, simple | +| App Logic | Lua (now), WASM (future) | Start simple, add WASM for isolation | +| System Services | C++ | Performance, direct hardware access | + +### Lua Sandbox + +RmlUi already uses Lua for UI scripting. We enhance it with: + +```lua +-- Sandboxed globals per app +app = { + id = "com.example.myapp", + storage = AppStorage("com.example.myapp"), + permissions = {"camera", "storage"}, +} + +-- Restricted stdlib +-- Remove: os.execute, io.popen, loadfile, dofile +-- Keep: string, table, math, coroutine +``` + +**Sandbox Implementation** (`src/main/kernel/src/lua_sandbox.cpp`): + +```cpp +class LuaSandbox { +public: + lua_State* CreateAppState(const std::string& app_id, + const std::vector& permissions); + + void RestrictGlobals(lua_State* L); + void InjectAppAPIs(lua_State* L, const AppManifest& manifest); + +private: + void RemoveDangerousFunctions(lua_State* L); + void SetupPermissionChecks(lua_State* L); +}; +``` + +--- + +## Package Format (.mpkg) + +### Directory Structure + +``` +myapp.mpkg/ +├── manifest.json # Required: metadata, permissions +├── ui/ +│ ├── main.rml # Entry point +│ ├── screens/ # Additional screens +│ │ └── settings.rml +│ ├── styles/ +│ │ └── app.rcss +│ └── scripts/ +│ └── app.lua +├── assets/ +│ ├── icon.png # 48x48 app icon +│ ├── icon_large.png # 192x192 for store +│ └── images/ +└── locales/ # Optional: i18n + ├── en.json + └── es.json +``` + +### Manifest Schema + +**File**: `manifest.json` + +```json +{ + "$schema": "https://mosis.dev/schemas/manifest-v1.json", + + "id": "com.example.myapp", + "name": "My App", + "version": "1.0.0", + "version_code": 1, + + "description": "A sample Mosis app", + "author": "Developer Name", + "website": "https://example.com", + + "entry": "ui/main.rml", + "icon": "assets/icon.png", + + "permissions": [ + "camera", + "microphone", + "storage", + "network", + "contacts.read", + "contacts.write" + ], + + "min_mosis_version": "1.0.0", + + "intents": { + "share": { + "types": ["image/*", "text/plain"], + "action": "ui/share.rml" + } + } +} +``` + +### Manifest TypeScript Interface (for validation) + +```typescript +interface MosisManifest { + id: string; // Reverse domain notation + name: string; // Display name + version: string; // SemVer + version_code: number; // Incremental integer + + description?: string; + author?: string; + website?: string; + + entry: string; // Path to main RML + icon: string; // Path to icon + + permissions: Permission[]; + min_mosis_version?: string; + + intents?: Record; +} + +type Permission = + | "camera" + | "microphone" + | "speaker" + | "storage" + | "network" + | "contacts.read" + | "contacts.write" + | "messages.read" + | "messages.write" + | "location" + | "phone.call" + | "notifications"; +``` + +--- + +## Permission System + +### Permission Levels + +| Level | Description | Example | +|-------|-------------|---------| +| Normal | Auto-granted | `storage` (app's own data) | +| Dangerous | User prompt required | `camera`, `contacts.read` | +| Signature | System apps only | `phone.call`, `system.settings` | + +### Permission Request Flow + +``` +App declares permission in manifest + ↓ +User installs app + ↓ +On first use of protected API: + ↓ +┌─────────────────────────────────┐ +│ "My App" wants to access your │ +│ camera. Allow? │ +│ │ +│ [Deny] [Allow Once] [Allow] │ +└─────────────────────────────────┘ + ↓ +Decision stored in PermissionManager + ↓ +API call proceeds or fails +``` + +### Permission Manager + +**File**: `src/main/kernel/include/permission_manager.h` + +```cpp +namespace mosis { + +enum class PermissionStatus { + NotRequested, + Granted, + Denied, + AllowedOnce +}; + +class IPermissionManager { +public: + virtual ~IPermissionManager() = default; + + // Check if app has permission + virtual PermissionStatus Check(const std::string& app_id, + const std::string& permission) = 0; + + // Request permission (may show UI) + using PermissionCallback = std::function; + virtual void Request(const std::string& app_id, + const std::string& permission, + PermissionCallback callback) = 0; + + // Revoke permission + virtual void Revoke(const std::string& app_id, + const std::string& permission) = 0; + + // Get all permissions for app + virtual std::map + GetAppPermissions(const std::string& app_id) = 0; +}; + +} // namespace mosis +``` + +--- + +## App Lifecycle + +### States + +``` +INSTALLED → LAUNCHING → RUNNING → PAUSED → STOPPED → UNINSTALLED + ↑ ↓ + └──────────────────────┘ + (resume) +``` + +### Lifecycle Events + +```lua +-- In app.lua +function onAppCreate() + -- Initialize app state +end + +function onAppResume() + -- Returning from background +end + +function onAppPause() + -- Going to background, save state +end + +function onAppDestroy() + -- Cleanup +end +``` + +### App Manager + +**File**: `src/main/kernel/include/app_manager.h` + +```cpp +namespace mosis { + +struct InstalledApp { + std::string id; + std::string name; + std::string version; + std::string icon_path; + std::string install_path; + int64_t installed_time; +}; + +class IAppManager { +public: + virtual ~IAppManager() = default; + + // Installation + virtual bool Install(const std::string& mpkg_path) = 0; + virtual bool Uninstall(const std::string& app_id) = 0; + virtual bool Update(const std::string& mpkg_path) = 0; + + // Query + virtual std::vector GetInstalledApps() = 0; + virtual std::optional GetApp(const std::string& app_id) = 0; + + // Lifecycle + virtual bool Launch(const std::string& app_id) = 0; + virtual bool Stop(const std::string& app_id) = 0; + virtual bool IsRunning(const std::string& app_id) = 0; + + // Inter-app communication + virtual void SendIntent(const std::string& action, + const std::map& data) = 0; +}; + +} // namespace mosis +``` + +--- + +## Storage Isolation + +### Per-App Storage + +```cpp +class AppStorage { +public: + AppStorage(const std::string& app_id); + + // Key-value storage (like SharedPreferences) + void SetString(const std::string& key, const std::string& value); + std::string GetString(const std::string& key, const std::string& default_value = ""); + void SetInt(const std::string& key, int value); + int GetInt(const std::string& key, int default_value = 0); + void SetBool(const std::string& key, bool value); + bool GetBool(const std::string& key, bool default_value = false); + + // File storage (app-private) + std::string GetFilesDir(); + std::string GetCacheDir(); + +private: + std::string m_app_id; + std::string m_base_path; +}; +``` + +### Lua API + +```lua +-- Key-value storage +app.storage:set("username", "john") +local name = app.storage:get("username", "anonymous") + +-- File access (sandboxed) +local data = app.files:read("state.json") +app.files:write("state.json", json.encode(state)) +``` + +--- + +## Implementation Plan + +### Phase 1: Package Format +- [ ] Define manifest schema +- [ ] Create manifest parser/validator +- [ ] Implement .mpkg directory loader + +### Phase 2: App Manager +- [ ] Install/uninstall apps +- [ ] App registry (installed apps database) +- [ ] Launch apps from package + +### Phase 3: Lua Sandbox +- [ ] Restrict dangerous globals +- [ ] Inject app-specific APIs +- [ ] Per-app Lua state management + +### Phase 4: Permission System +- [ ] Permission declaration in manifest +- [ ] Runtime permission checks +- [ ] Permission request UI + +### Phase 5: Storage Isolation +- [ ] Per-app directories +- [ ] Key-value storage +- [ ] Quota management + +--- + +## Security Considerations + +1. **Lua Sandbox Escape**: Audit all exposed functions +2. **Path Traversal**: Validate all file paths +3. **Memory Limits**: Set Lua memory quotas +4. **CPU Limits**: Timeout long-running scripts +5. **Network Isolation**: Apps only access allowed domains + +--- + +## Acceptance Criteria + +- [ ] Apps installable from .mpkg directories +- [ ] Apps launch in isolated Lua environment +- [ ] Permission requests shown to user +- [ ] App data isolated per app +- [ ] Apps can be uninstalled cleanly +- [ ] Store app can browse and install packages diff --git a/MILESTONE-5.md b/MILESTONE-5.md new file mode 100644 index 0000000..1b35fbe --- /dev/null +++ b/MILESTONE-5.md @@ -0,0 +1,407 @@ +# Milestone 5: WebRTC Bridge + +**Status**: Not Started +**Goal**: Cross-device communication via WebRTC for calls, messaging, and file sharing. + +--- + +## Overview + +WebRTC enables: +- Voice/video calls between virtual phones +- Text messaging across different games +- File sharing between devices +- Screen sharing +- Connection to real smartphones (companion app) + +--- + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Virtual Phone A │ +│ (VR Game on User's PC) │ +│ ┌───────────────────────────────────────────────────────────────┐ │ +│ │ WebRTCBridge │ │ +│ │ ├── DataChannel (messages, files) │ │ +│ │ ├── AudioTrack (voice) │ │ +│ │ └── VideoTrack (camera/screen share) │ │ +│ └───────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────┘ + │ + │ WebRTC (UDP/DTLS-SRTP) + │ + ┌─────────┴─────────┐ + │ Signaling Server │ + │ (WebSocket) │ + └─────────┬─────────┘ + │ + ┌───────────────┼───────────────┐ + │ │ │ + ▼ ▼ ▼ +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Virtual Phone B │ │ Virtual Phone C │ │ Real Smartphone │ +│ (Different Game)│ │ (Same Game) │ │ (Companion App) │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ +``` + +--- + +## Dependencies + +### libdatachannel + +C++ WebRTC library for data channels, audio, and video. + +**vcpkg installation**: +```bash +vcpkg install libdatachannel +``` + +**CMakeLists.txt**: +```cmake +find_package(LibDataChannel CONFIG REQUIRED) +target_link_libraries(mosis-kernel PRIVATE LibDataChannel::LibDataChannel) +``` + +--- + +## Core Components + +### WebRTC Bridge + +**File**: `src/main/kernel/include/webrtc_bridge.h` + +```cpp +namespace mosis { + +struct PeerInfo { + std::string peer_id; + std::string display_name; + bool is_online; +}; + +struct CallState { + std::string peer_id; + bool is_active; + bool is_video; + bool is_muted; + int64_t start_time; +}; + +class IWebRTCBridge { +public: + virtual ~IWebRTCBridge() = default; + + // Connection + virtual void Connect(const std::string& signaling_server_url) = 0; + virtual void Disconnect() = 0; + virtual bool IsConnected() const = 0; + + // Identity + virtual void SetIdentity(const std::string& phone_id, + const std::string& display_name) = 0; + virtual std::string GetPhoneId() const = 0; + + // Peer discovery + virtual std::vector GetOnlinePeers() = 0; + + // Messaging + using MessageCallback = std::function; + virtual void SetMessageCallback(MessageCallback callback) = 0; + virtual void SendMessage(const std::string& to, const std::string& message) = 0; + + // Voice calls + using CallCallback = std::function; + virtual void SetCallCallback(CallCallback callback) = 0; + virtual void StartCall(const std::string& peer_id, bool with_video) = 0; + virtual void AnswerCall(const std::string& peer_id) = 0; + virtual void EndCall(const std::string& peer_id) = 0; + virtual void MuteCall(bool muted) = 0; + + // File transfer + using FileTransferCallback = std::function& data)>; + virtual void SetFileTransferCallback(FileTransferCallback callback) = 0; + virtual void SendFile(const std::string& to, + const std::string& filename, + const std::vector& data) = 0; +}; + +} // namespace mosis +``` + +### Implementation + +**File**: `src/main/kernel/src/webrtc_bridge.cpp` + +```cpp +#include +#include "webrtc_bridge.h" + +namespace mosis { + +class WebRTCBridgeImpl : public IWebRTCBridge { +public: + WebRTCBridgeImpl() { + rtc::InitLogger(rtc::LogLevel::Warning); + } + + void Connect(const std::string& signaling_url) override { + m_ws = std::make_shared(); + + m_ws->onOpen([this]() { + SendSignaling({{"type", "register"}, {"id", m_phone_id}}); + }); + + m_ws->onMessage([this](std::variant msg) { + if (auto* str = std::get_if(&msg)) { + HandleSignaling(nlohmann::json::parse(*str)); + } + }); + + m_ws->open(signaling_url); + } + + void SendMessage(const std::string& to, const std::string& message) override { + if (auto it = m_peers.find(to); it != m_peers.end()) { + auto& peer = it->second; + if (peer.data_channel && peer.data_channel->isOpen()) { + nlohmann::json msg = { + {"type", "message"}, + {"text", message} + }; + peer.data_channel->send(msg.dump()); + } + } + } + +private: + struct PeerConnection { + std::shared_ptr pc; + std::shared_ptr data_channel; + std::shared_ptr audio_track; + std::shared_ptr video_track; + }; + + void CreatePeerConnection(const std::string& peer_id) { + rtc::Configuration config; + config.iceServers.emplace_back("stun:stun.l.google.com:19302"); + + auto pc = std::make_shared(config); + + pc->onStateChange([this, peer_id](rtc::PeerConnection::State state) { + // Handle connection state changes + }); + + pc->onLocalDescription([this, peer_id](rtc::Description desc) { + SendSignaling({ + {"type", desc.typeString()}, + {"to", peer_id}, + {"sdp", std::string(desc)} + }); + }); + + pc->onLocalCandidate([this, peer_id](rtc::Candidate cand) { + SendSignaling({ + {"type", "candidate"}, + {"to", peer_id}, + {"candidate", std::string(cand)} + }); + }); + + m_peers[peer_id] = {pc, nullptr, nullptr, nullptr}; + } + + void HandleSignaling(const nlohmann::json& msg); + void SendSignaling(const nlohmann::json& msg); + + std::string m_phone_id; + std::shared_ptr m_ws; + std::map m_peers; + MessageCallback m_message_cb; + CallCallback m_call_cb; +}; + +} // namespace mosis +``` + +--- + +## Signaling Protocol + +### Message Types + +```json +// Register with server +{"type": "register", "id": "phone_123", "name": "John's Phone"} + +// Peer discovery +{"type": "get_peers"} +{"type": "peers", "list": [{"id": "phone_456", "name": "Jane's Phone", "online": true}]} + +// WebRTC signaling +{"type": "offer", "to": "phone_456", "sdp": "v=0\r\n..."} +{"type": "answer", "to": "phone_123", "sdp": "v=0\r\n..."} +{"type": "candidate", "to": "phone_456", "candidate": "candidate:..."} + +// Call signaling +{"type": "call_request", "to": "phone_456", "video": false} +{"type": "call_accept", "to": "phone_123"} +{"type": "call_reject", "to": "phone_123", "reason": "busy"} +{"type": "call_end", "to": "phone_456"} +``` + +### Signaling Server + +**Simple Node.js reference implementation**: + +```javascript +const WebSocket = require('ws'); +const wss = new WebSocket.Server({ port: 8080 }); + +const peers = new Map(); + +wss.on('connection', (ws) => { + ws.on('message', (data) => { + const msg = JSON.parse(data); + + if (msg.type === 'register') { + peers.set(msg.id, { ws, name: msg.name }); + broadcast({ type: 'peer_joined', id: msg.id, name: msg.name }); + } + else if (msg.to) { + // Forward to recipient + const peer = peers.get(msg.to); + if (peer) { + msg.from = [...peers.entries()].find(([id, p]) => p.ws === ws)?.[0]; + peer.ws.send(JSON.stringify(msg)); + } + } + }); + + ws.on('close', () => { + const id = [...peers.entries()].find(([_, p]) => p.ws === ws)?.[0]; + if (id) { + peers.delete(id); + broadcast({ type: 'peer_left', id }); + } + }); +}); + +function broadcast(msg) { + peers.forEach((peer) => peer.ws.send(JSON.stringify(msg))); +} +``` + +--- + +## Lua API + +```lua +-- Connect to signaling server +mosis.webrtc.connect("wss://signal.mosis.dev") + +-- Set identity +mosis.webrtc.setIdentity("user_12345", "John's Phone") + +-- Get online peers +local peers = mosis.webrtc.getPeers() +for _, peer in ipairs(peers) do + print(peer.id, peer.name, peer.online) +end + +-- Send message +mosis.webrtc.sendMessage("peer_id", "Hello!") + +-- Receive messages +mosis.webrtc.onMessage(function(from, message) + print("Message from " .. from .. ": " .. message) +end) + +-- Make a call +mosis.webrtc.call("peer_id", { video = false }) + +-- Handle incoming call +mosis.webrtc.onIncomingCall(function(from, hasVideo) + -- Show call UI + -- Accept: mosis.webrtc.answerCall(from) + -- Reject: mosis.webrtc.rejectCall(from) +end) +``` + +--- + +## Real Smartphone Bridge + +### Companion App + +A mobile app (Android/iOS) that: +1. Connects to same signaling server +2. Bridges to real phone's contacts, messages +3. Allows receiving calls from virtual phone +4. Sends notifications to virtual phone + +**Use Cases**: +- Receive real SMS in virtual phone +- Make real calls from VR +- Sync contacts between real and virtual phone +- Get push notifications in VR + +--- + +## Implementation Plan + +### Phase 1: Core WebRTC +- [ ] Add libdatachannel to vcpkg.json +- [ ] Implement WebRTCBridgeImpl +- [ ] Basic signaling protocol + +### Phase 2: Messaging +- [ ] Data channel messaging +- [ ] Message history storage +- [ ] Messages app integration + +### Phase 3: Voice Calls +- [ ] Audio track setup +- [ ] Microphone/speaker integration +- [ ] Dialer app integration + +### Phase 4: Signaling Server +- [ ] Production signaling server +- [ ] User authentication +- [ ] Peer discovery service + +### Phase 5: Companion App +- [ ] Android companion app +- [ ] iOS companion app (future) +- [ ] Contact/message sync + +--- + +## Network Requirements + +- UDP ports for WebRTC media +- WebSocket for signaling +- TURN server for NAT traversal (optional) + +**TURN/STUN Configuration**: +```cpp +rtc::Configuration config; +config.iceServers.emplace_back("stun:stun.l.google.com:19302"); +config.iceServers.emplace_back("turn:turn.mosis.dev:3478", "user", "pass"); +``` + +--- + +## Acceptance Criteria + +- [ ] Two virtual phones can exchange messages +- [ ] Voice calls work between virtual phones +- [ ] Files can be transferred between devices +- [ ] Connection survives game restarts (rejoin) +- [ ] Messages app shows real-time chat +- [ ] Dialer app can place/receive calls diff --git a/MILESTONE-6.md b/MILESTONE-6.md new file mode 100644 index 0000000..cd88799 --- /dev/null +++ b/MILESTONE-6.md @@ -0,0 +1,500 @@ +# Milestone 6: System Apps + +**Status**: 75% Complete +**Goal**: Core smartphone apps with full functionality. + +--- + +## Overview + +System apps provide the essential smartphone experience: +- Home launcher +- Phone/Dialer +- Messages +- Contacts +- Settings +- Browser +- Store (TODO) +- Camera (TODO) +- Music (TODO) + +--- + +## App Status + +### Completed Apps + +| App | Location | Features | Status | +|-----|----------|----------|--------| +| Home | `apps/home/` | App grid, dock, navigation | Complete | +| Dialer | `apps/dialer/` | Keypad, call UI (mock) | Complete | +| Messages | `apps/messages/` | Conversation list, chat | Complete | +| Contacts | `apps/contacts/` | List, search, detail | Complete | +| Settings | `apps/settings/` | Display, sound, about | Complete | +| Browser | `apps/browser/` | URL bar, placeholder | Complete | + +### Remaining Apps + +| App | Priority | Description | +|-----|----------|-------------| +| Store | High | Browse and install apps | +| Camera | Medium | Viewfinder, capture photos | +| Music | Low | Audio playback | + +--- + +## App: Store + +**Location**: `src/main/assets/apps/store/` + +### Features + +1. **Browse Apps** + - Featured apps carousel + - Categories (Games, Utilities, Social) + - Search functionality + +2. **App Details** + - Name, icon, description + - Screenshots + - Permissions list + - Install/Update button + +3. **My Apps** + - Installed apps list + - Update available indicator + - Uninstall option + +### UI Screens + +``` +store/ +├── store.rml # Main store screen +├── store.rcss # Store styles +├── category.rml # Category listing +├── detail.rml # App detail page +└── scripts/ + └── store.lua # Store logic +``` + +### Main Screen (`store.rml`) + +```html + + + + + + +
+
+ +
+ Store +
+ +
+
+ +
+ +
+

Featured

+
+
+ + +
+
+
+ + +
+

Categories

+
+
+ + Games +
+
+ + Utilities +
+
+
+ + +
+

Top Apps

+
+
+ +
+ + +
+ +
+
+
+
+ + +``` + +### Data Model + +```cpp +// In data_models.cpp +struct StoreApp { + std::string id; + std::string name; + std::string icon; + std::string banner; + std::string category; + std::string description; + std::string version; + std::vector screenshots; + std::vector permissions; + bool installed; +}; + +void setupStoreDataModel(Rml::Context* context) { + auto model = context->CreateDataModel("store"); + + model.Bind("featured_apps", &g_featured_apps); + model.Bind("top_apps", &g_top_apps); + model.Bind("categories", &g_categories); + model.Bind("current_app", &g_current_app); + + model.BindEventCallback("install", [](auto& event, auto& args) { + // Install app + }); +} +``` + +--- + +## App: Camera + +**Location**: `src/main/assets/apps/camera/` + +### Features + +1. **Viewfinder** + - Live camera preview (from ICamera) + - Capture button + - Switch camera (front/back) + - Flash toggle + +2. **Capture** + - Take photo + - Save to gallery + - Share option + +3. **Gallery** + - View captured photos + - Delete photos + - Share photos + +### UI Screens + +``` +camera/ +├── camera.rml # Viewfinder +├── camera.rcss # Camera styles +├── gallery.rml # Photo gallery +└── scripts/ + └── camera.lua # Camera logic +``` + +### Viewfinder (`camera.rml`) + +```html + + + + + + +
+ +
+ + +
+
+ +
+
+ +
+
+ + +
+
+ +
+ +
+
+
+ +
+ +
+
+ + +``` + +### Camera Lua Script + +```lua +-- camera.lua +local camera = mosis.platform.getCamera() +local capture = mosis.testing.VisualCapture(540, 960) + +local is_front_camera = false +local flash_on = false + +function onAppCreate() + if camera:isAvailable() then + camera:startCapture(function(frame) + -- Update preview texture + document:GetElementById("preview-frame"):SetAttribute("src", frame.texture_url) + end) + else + -- Show "no camera" message + end +end + +function capture() + local path = mosis.filesystem:getSharedMediaPath() .. "/photos/" .. os.time() .. ".png" + capture:CaptureScreenshot(path) + + -- Show capture animation + playSound("shutter") + flashScreen() + + -- Update last photo thumbnail + document:GetElementById("last-photo"):SetAttribute("src", path) +end + +function switchCamera() + is_front_camera = not is_front_camera + -- camera:setFacing(is_front_camera and "front" or "back") +end + +function toggleFlash() + flash_on = not flash_on + local icon = flash_on and "flash_on" or "flash_off" + document:GetElementById("flash-icon"):SetAttribute("src", "../../icons/" .. icon .. ".tga") +end + +function openGallery() + navigateTo("camera/gallery") +end +``` + +--- + +## App: Music + +**Location**: `src/main/assets/apps/music/` + +### Features + +1. **Library** + - Songs list + - Albums + - Artists + - Playlists + +2. **Player** + - Play/pause + - Next/previous + - Seek bar + - Volume control + - Shuffle/repeat + +3. **Now Playing** + - Album art + - Song info + - Progress bar + +### UI Screens + +``` +music/ +├── music.rml # Library view +├── music.rcss # Music styles +├── player.rml # Now playing +├── playlist.rml # Playlist view +└── scripts/ + └── music.lua # Player logic +``` + +--- + +## Data Persistence + +### Storage Layer + +Apps need persistent storage for: +- Contacts +- Messages +- Settings +- Photos + +**Implementation Options**: + +1. **JSON Files** (Simple) + ``` + /data/contacts.json + /data/messages.json + /data/settings.json + ``` + +2. **SQLite** (Robust) + ``` + /data/mosis.db + - contacts table + - messages table + - settings table + ``` + +### Contact Storage + +```cpp +// contact_storage.h +struct Contact { + std::string id; + std::string name; + std::string phone; + std::string email; + std::string avatar; +}; + +class ContactStorage { +public: + std::vector GetAll(); + std::optional GetById(const std::string& id); + bool Save(const Contact& contact); + bool Delete(const std::string& id); + std::vector Search(const std::string& query); +}; +``` + +### Message Storage + +```cpp +// message_storage.h +struct Message { + std::string id; + std::string conversation_id; + std::string sender; + std::string text; + int64_t timestamp; + bool read; +}; + +struct Conversation { + std::string id; + std::string contact_id; + std::string last_message; + int64_t last_timestamp; + int unread_count; +}; + +class MessageStorage { +public: + std::vector GetConversations(); + std::vector GetMessages(const std::string& conversation_id); + bool SaveMessage(const Message& message); + bool MarkAsRead(const std::string& conversation_id); +}; +``` + +--- + +## Implementation Plan + +### Phase 1: Store App UI +- [ ] Main store screen layout +- [ ] Category browsing +- [ ] App detail page +- [ ] Mock data for testing + +### Phase 2: Camera App +- [ ] Viewfinder UI +- [ ] Capture to file +- [ ] Gallery view +- [ ] Integration with ICamera + +### Phase 3: Music App +- [ ] Library UI +- [ ] Player UI +- [ ] Audio playback (stub) + +### Phase 4: Data Persistence +- [ ] JSON storage layer +- [ ] Contact CRUD +- [ ] Message storage +- [ ] Settings persistence + +### Phase 5: Real Functionality +- [ ] Store: Install real .mpkg files +- [ ] Camera: Real camera frames +- [ ] Music: Audio playback + +--- + +## Testing + +### Test IDs for Store + +| ID | Element | +|----|---------| +| `store-search` | Search button | +| `store-featured` | Featured carousel | +| `store-categories` | Category grid | +| `app-install-btn` | Install button on detail | + +### Test IDs for Camera + +| ID | Element | +|----|---------| +| `camera-preview` | Preview area | +| `capture-btn` | Capture button | +| `gallery-btn` | Gallery button | +| `switch-camera-btn` | Switch camera | + +--- + +## Acceptance Criteria + +### Store +- [ ] Browse featured and top apps +- [ ] View app details +- [ ] See permission requirements +- [ ] Install apps (mock or real) + +### Camera +- [ ] Display camera preview +- [ ] Capture photos +- [ ] View in gallery +- [ ] Share photos + +### Music +- [ ] Display music library +- [ ] Play/pause audio +- [ ] Show now playing + +### Persistence +- [ ] Contacts persist across sessions +- [ ] Messages persist across sessions +- [ ] Settings persist across sessions diff --git a/MILESTONE-7.md b/MILESTONE-7.md new file mode 100644 index 0000000..d097439 --- /dev/null +++ b/MILESTONE-7.md @@ -0,0 +1,496 @@ +# Milestone 7: Game Integration + +**Status**: Not Started +**Goal**: Production-ready Unity and Unreal plugins for seamless VR phone integration. + +--- + +## Overview + +Game engine plugins enable: +- Rendering the phone in VR scenes +- Touch/raycast interaction +- Virtual hardware provision (camera, mic, speaker) +- Event callbacks to game code + +--- + +## Current State + +| Platform | Location | Status | +|----------|----------|--------| +| Unity | `D:\Dev\Mosis\Mosis Unity` | Basic Binder client | +| Unreal | `D:\Dev\Mosis\Mosis Unreal` | WIP | + +--- + +## Unity Plugin + +### Package Structure + +``` +com.mosis.phone/ +├── package.json +├── Runtime/ +│ ├── MosisPhone.cs # Main component +│ ├── MosisService.cs # Binder client +│ ├── MosisInputHandler.cs # Touch/raycast +│ ├── MosisHardwareProvider.cs # Virtual hardware +│ └── Native/ +│ ├── MosisNative.cs # P/Invoke declarations +│ └── Plugins/ +│ └── Android/ +│ └── libmosis-client.so +├── Prefabs/ +│ ├── MosisPhone.prefab # Ready-to-use phone +│ └── MosisPhoneVR.prefab # VR-optimized version +├── Samples~/ +│ └── BasicIntegration/ +│ └── PhoneDemo.unity +└── Documentation~/ + └── integration-guide.md +``` + +### Main Component + +**File**: `MosisPhone.cs` + +```csharp +using UnityEngine; +using UnityEngine.Events; + +namespace Mosis { + public class MosisPhone : MonoBehaviour { + [Header("Display")] + [SerializeField] private MeshRenderer phoneScreen; + [SerializeField] private Vector2Int resolution = new Vector2Int(540, 960); + + [Header("Input")] + [SerializeField] private bool enableRaycast = true; + [SerializeField] private LayerMask raycastLayers; + + [Header("Virtual Hardware")] + [SerializeField] private Camera virtualCamera; + [SerializeField] private AudioSource virtualMicrophone; + [SerializeField] private AudioSource virtualSpeaker; + + [Header("Events")] + public UnityEvent onPhoneReady; + public UnityEvent onNavigate; + public UnityEvent onMessageReceived; + public UnityEvent onCallStarted; + + private MosisService service; + private RenderTexture screenTexture; + private Material screenMaterial; + + void Awake() { + // Create render texture for phone screen + screenTexture = new RenderTexture(resolution.x, resolution.y, 0); + screenMaterial = new Material(Shader.Find("Unlit/Texture")); + screenMaterial.mainTexture = screenTexture; + phoneScreen.material = screenMaterial; + } + + void Start() { + service = new MosisService(); + service.OnFrameAvailable += OnFrameAvailable; + service.OnServiceInitialized += () => onPhoneReady?.Invoke(); + service.Connect(); + } + + void Update() { + if (enableRaycast) { + HandleRaycastInput(); + } + } + + private void HandleRaycastInput() { + // VR controller raycast + if (OVRInput.GetDown(OVRInput.Button.PrimaryIndexTrigger)) { + var ray = new Ray( + OVRInput.GetLocalControllerPosition(OVRInput.Controller.RTouch), + OVRInput.GetLocalControllerRotation(OVRInput.Controller.RTouch) * Vector3.forward + ); + + if (Physics.Raycast(ray, out var hit, 10f, raycastLayers)) { + if (hit.collider.gameObject == phoneScreen.gameObject) { + var uv = hit.textureCoord; + var x = uv.x * resolution.x; + var y = (1 - uv.y) * resolution.y; + service.SendTouchDown(x, y); + } + } + } + } + + private void OnFrameAvailable(byte[] pixels) { + // Update screen texture + screenTexture.LoadRawTextureData(pixels); + screenTexture.Apply(); + } + + // Public API + public void NavigateTo(string screen) { + service.Navigate(screen); + } + + public void SendMessage(string to, string text) { + service.SendMessage(to, text); + } + + public void MakeCall(string number) { + service.MakeCall(number); + } + + void OnDestroy() { + service?.Disconnect(); + Destroy(screenTexture); + Destroy(screenMaterial); + } + } +} +``` + +### Hardware Provider + +**File**: `MosisHardwareProvider.cs` + +```csharp +namespace Mosis { + public class MosisHardwareProvider : MonoBehaviour { + [SerializeField] private MosisPhone phone; + [SerializeField] private Camera gameCamera; + [SerializeField] private AudioListener audioListener; + + private RenderTexture cameraTexture; + + void Start() { + // Setup virtual camera + cameraTexture = new RenderTexture(640, 480, 0); + gameCamera.targetTexture = cameraTexture; + + // Register with phone service + phone.Service.SetCameraProvider(() => { + var pixels = new byte[cameraTexture.width * cameraTexture.height * 4]; + // Read pixels from RenderTexture + return pixels; + }); + + phone.Service.SetMicrophoneProvider(() => { + // Get audio from AudioListener + return audioSamples; + }); + } + + public void PlayAudio(float[] samples) { + // Play on virtual speaker + var audioSource = GetComponent(); + audioSource.clip = AudioClip.Create("phone", samples.Length, 1, 44100, false); + audioSource.clip.SetData(samples, 0); + audioSource.Play(); + } + } +} +``` + +### VR Interaction + +**File**: `MosisInputHandler.cs` + +```csharp +namespace Mosis { + public class MosisInputHandler : MonoBehaviour { + [SerializeField] private MosisPhone phone; + [SerializeField] private Transform pointerOrigin; + [SerializeField] private LineRenderer laserPointer; + + private bool isTouching; + private Vector2 lastTouchPos; + + void Update() { + // Update laser pointer + var ray = new Ray(pointerOrigin.position, pointerOrigin.forward); + + if (Physics.Raycast(ray, out var hit, 10f)) { + laserPointer.SetPosition(1, hit.point); + + if (hit.collider.GetComponent() != null) { + var uv = hit.textureCoord; + var touchPos = new Vector2( + uv.x * phone.Resolution.x, + (1 - uv.y) * phone.Resolution.y + ); + + HandleTouch(touchPos); + } + } + } + + private void HandleTouch(Vector2 pos) { + bool triggerDown = OVRInput.Get(OVRInput.Button.PrimaryIndexTrigger); + + if (triggerDown && !isTouching) { + phone.Service.SendTouchDown(pos.x, pos.y); + isTouching = true; + } + else if (triggerDown && isTouching) { + if (Vector2.Distance(pos, lastTouchPos) > 2f) { + phone.Service.SendTouchMove(pos.x, pos.y); + } + } + else if (!triggerDown && isTouching) { + phone.Service.SendTouchUp(pos.x, pos.y); + isTouching = false; + } + + lastTouchPos = pos; + } + } +} +``` + +--- + +## Unreal Plugin + +### Module Structure + +``` +MosisPhone/ +├── MosisPhone.uplugin +├── Source/ +│ ├── MosisPhone/ +│ │ ├── MosisPhone.Build.cs +│ │ ├── Public/ +│ │ │ ├── MosisPhoneActor.h +│ │ │ ├── MosisService.h +│ │ │ └── MosisHardwareProvider.h +│ │ └── Private/ +│ │ ├── MosisPhoneActor.cpp +│ │ ├── MosisService.cpp +│ │ └── MosisHardwareProvider.cpp +│ └── ThirdParty/ +│ └── MosisClient/ +│ └── libmosis-client.so +├── Content/ +│ ├── BP_MosisPhone.uasset +│ └── M_PhoneScreen.uasset +└── Documentation/ + └── integration-guide.md +``` + +### Main Actor + +**File**: `MosisPhoneActor.h` + +```cpp +#pragma once + +#include "CoreMinimal.h" +#include "GameFramework/Actor.h" +#include "MosisPhoneActor.generated.h" + +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnPhoneReady); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnNavigate, FString, Screen); + +UCLASS() +class MOSISPHONE_API AMosisPhoneActor : public AActor { + GENERATED_BODY() + +public: + AMosisPhoneActor(); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Display") + FIntPoint Resolution = FIntPoint(540, 960); + + UPROPERTY(BlueprintAssignable, Category = "Events") + FOnPhoneReady OnPhoneReady; + + UPROPERTY(BlueprintAssignable, Category = "Events") + FOnNavigate OnNavigate; + + UFUNCTION(BlueprintCallable, Category = "Mosis") + void NavigateTo(const FString& Screen); + + UFUNCTION(BlueprintCallable, Category = "Mosis") + void SendMessage(const FString& To, const FString& Text); + + UFUNCTION(BlueprintCallable, Category = "Mosis") + void MakeCall(const FString& Number); + +protected: + virtual void BeginPlay() override; + virtual void Tick(float DeltaTime) override; + virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override; + +private: + UPROPERTY() + UTextureRenderTarget2D* ScreenTexture; + + UPROPERTY() + UMaterialInstanceDynamic* ScreenMaterial; + + TSharedPtr Service; + + void HandleRaycastInput(); + void OnFrameAvailable(const TArray& Pixels); +}; +``` + +### Blueprint Integration + +```cpp +// Blueprint callable functions for easy integration + +UFUNCTION(BlueprintCallable, Category = "Mosis", meta = (WorldContext = "WorldContextObject")) +static AMosisPhoneActor* SpawnMosisPhone( + UObject* WorldContextObject, + FTransform SpawnTransform, + FIntPoint Resolution = FIntPoint(540, 960) +); +``` + +--- + +## Hardware Button Simulation + +### Physical Buttons + +| Button | Function | +|--------|----------| +| Power | Lock/wake screen | +| Volume Up | Increase volume | +| Volume Down | Decrease volume | +| Home (soft) | Go to home screen | +| Back (soft) | Navigation back | + +### Unity Implementation + +```csharp +public class MosisPhoneButtons : MonoBehaviour { + [SerializeField] private MosisPhone phone; + [SerializeField] private Collider powerButton; + [SerializeField] private Collider volumeUp; + [SerializeField] private Collider volumeDown; + + void Update() { + // Check for button presses via physics overlap + if (IsButtonPressed(powerButton)) { + phone.Service.SendButton("power"); + } + // ... + } +} +``` + +### Unreal Implementation + +```cpp +// In Blueprint or C++ +UFUNCTION(BlueprintCallable) +void PressButton(EMosisButton Button) { + switch (Button) { + case EMosisButton::Power: + Service->SendButton("power"); + break; + case EMosisButton::VolumeUp: + Service->SendButton("volume_up"); + break; + case EMosisButton::VolumeDown: + Service->SendButton("volume_down"); + break; + } +} +``` + +--- + +## Implementation Plan + +### Phase 1: Unity Core +- [ ] Package structure +- [ ] MosisPhone component +- [ ] Basic touch input +- [ ] Frame rendering + +### Phase 2: Unity VR +- [ ] VR raycast interaction +- [ ] Laser pointer visualization +- [ ] Controller haptics +- [ ] Two-handed support + +### Phase 3: Unity Hardware +- [ ] Camera provider (RenderTexture) +- [ ] Microphone provider +- [ ] Speaker output +- [ ] Vibration + +### Phase 4: Unreal Core +- [ ] Plugin structure +- [ ] MosisPhoneActor +- [ ] Blueprint integration +- [ ] Touch input + +### Phase 5: Unreal VR +- [ ] Motion controller input +- [ ] Widget interaction +- [ ] Hardware providers + +### Phase 6: Documentation +- [ ] Quick start guide +- [ ] API reference +- [ ] Sample scenes +- [ ] Troubleshooting + +--- + +## Testing + +### Unity Test Scene + +1. Phone mounted in VR scene +2. Touch interaction works +3. Virtual camera shows game view +4. Audio plays through virtual speaker + +### Unreal Test Level + +1. Blueprint phone actor +2. Motion controller interaction +3. Event callbacks work +4. Performance acceptable + +--- + +## Performance Targets + +| Metric | Target | +|--------|--------| +| Frame latency | < 16ms | +| Touch latency | < 50ms | +| Memory usage | < 100MB | +| Draw calls | < 5 per phone | + +--- + +## Acceptance Criteria + +### Unity +- [ ] One-click prefab placement +- [ ] VR raycast touch works +- [ ] Events fire correctly +- [ ] Camera feed from game +- [ ] Audio bidirectional + +### Unreal +- [ ] Blueprint spawnable +- [ ] Motion controller support +- [ ] Blueprint events +- [ ] Hardware providers + +### Documentation +- [ ] Integration guide +- [ ] API docs +- [ ] Example projects +- [ ] Video tutorial diff --git a/designer/src/main.cpp b/designer/src/main.cpp index 33f87bb..edcdcdf 100644 --- a/designer/src/main.cpp +++ b/designer/src/main.cpp @@ -20,6 +20,8 @@ #include "hot_reload.h" #include "desktop_platform.h" #include "testing/ui_inspector.h" +#include "testing/action_recorder.h" +#include "testing/action_player.h" // Command-line options struct Options { @@ -31,6 +33,8 @@ struct Options { std::string output_dir = "dump"; std::string log_file; // If set, write logs to this file std::string hierarchy_file; // If set, dump UI hierarchy to this file each frame + std::string record_file; // If set, record actions to JSON file + std::string playback_file; // If set, play back actions from JSON file }; // Global log file stream @@ -89,6 +93,43 @@ static mosis::IKernel* g_kernel = nullptr; static std::filesystem::path g_assets_path; static mosis::testing::UIInspector g_ui_inspector; +// Recording/playback state +static std::unique_ptr g_recorder; +static std::unique_ptr g_player; +static std::string g_record_file_path; + +// Key callback for F5 (recording control) +bool HandleKeyDown(Rml::Context* context, Rml::Input::KeyIdentifier key, int key_modifier, float native_dp_ratio, bool priority) { + // F5: Toggle recording / Save recording + if (key == Rml::Input::KI_F5 && g_recorder) { + if (g_recorder->IsRecording()) { + g_recorder->StopRecording(); + if (g_recorder->SaveToFile(g_record_file_path)) { + LogMessage("Recording saved to: " + g_record_file_path); + } else { + LogMessage("ERROR: Failed to save recording to: " + g_record_file_path); + } + } else { + g_recorder->StartRecording(); + LogMessage("Recording started (press F5 to stop and save)"); + } + return true; // Consumed + } + + // F6: Pause/resume playback + if (key == Rml::Input::KI_F6 && g_player) { + if (g_player->IsPlaying()) { + g_player->Pause(); + LogMessage("Playback paused"); + } else if (g_player->IsPaused()) { + g_player->Resume(); + LogMessage("Playback resumed"); + } + return true; // Consumed + } + + return false; // Not consumed, let RmlUi handle it +} int main(int argc, const char* argv[]) { @@ -206,6 +247,25 @@ int main(int argc, const char* argv[]) // Start kernel kernel->Start(); + // Initialize recording if enabled + if (!opts.record_file.empty()) { + g_recorder = std::make_unique(opts.width, opts.height); + g_record_file_path = opts.record_file; + LogMessage("Recording mode enabled. Press F5 to start recording."); + } + + // Initialize playback if enabled + if (!opts.playback_file.empty()) { + g_player = std::make_unique(context); + if (g_player->LoadFromFile(opts.playback_file)) { + LogMessage("Loaded playback file: " + opts.playback_file); + g_player->Start(); + } else { + LogMessage("ERROR: Failed to load playback file: " + opts.playback_file); + g_player.reset(); + } + } + // Setup hot-reload std::unique_ptr hot_reload; if (!opts.dump_mode) { @@ -228,8 +288,16 @@ int main(int argc, const char* argv[]) hot_reload->CheckForChanges(); } - // Process events and update - running = Backend::ProcessEvents(context); + // Process events and update (with key callback for F5/F6 control) + running = Backend::ProcessEvents(context, HandleKeyDown); + + // Update playback if active + if (g_player && g_player->IsPlaying()) { + g_player->Update(); + if (g_player->IsFinished()) { + LogMessage("Playback complete"); + } + } // Update kernel (processes tasks, updates time, etc.) kernel->Update(); @@ -249,6 +317,16 @@ int main(int argc, const char* argv[]) } // Cleanup + // Stop and save recording if still active + if (g_recorder && g_recorder->IsRecording()) { + g_recorder->StopRecording(); + if (g_recorder->SaveToFile(g_record_file_path)) { + LogMessage("Recording saved on exit to: " + g_record_file_path); + } + } + g_recorder.reset(); + g_player.reset(); + kernel->Stop(); kernel.reset(); g_kernel = nullptr; @@ -282,11 +360,19 @@ void PrintUsage(const char* program) std::cout << " --output DIR Output directory for dump mode (default: dump)" << std::endl; std::cout << " --log FILE Write log output to file (for automated testing)" << std::endl; std::cout << " --hierarchy FILE Continuously dump UI hierarchy to JSON file" << std::endl; + std::cout << " --record FILE Record actions to JSON file (F5 to start/stop)" << std::endl; + std::cout << " --playback FILE Play back recorded actions from JSON file" << std::endl; + std::cout << std::endl; + std::cout << "Recording Controls:" << std::endl; + std::cout << " F5 Start/stop recording (when --record is enabled)" << std::endl; + std::cout << " F6 Pause/resume playback (when --playback is enabled)" << std::endl; std::cout << std::endl; std::cout << "Examples:" << std::endl; std::cout << " " << program << " assets/apps/home/home.rml" << std::endl; std::cout << " " << program << " assets/apps/home/home.rml --resolution 720x1280" << std::endl; std::cout << " " << program << " assets/apps/home/home.rml --dump" << std::endl; + std::cout << " " << program << " assets/apps/home/home.rml --record test.json" << std::endl; + std::cout << " " << program << " assets/apps/home/home.rml --playback test.json" << std::endl; } Options ParseOptions(int argc, const char* argv[]) @@ -313,6 +399,10 @@ Options ParseOptions(int argc, const char* argv[]) opts.log_file = argv[++i]; } else if (arg == "--hierarchy" && i + 1 < argc) { opts.hierarchy_file = argv[++i]; + } else if (arg == "--record" && i + 1 < argc) { + opts.record_file = argv[++i]; + } else if (arg == "--playback" && i + 1 < argc) { + opts.playback_file = argv[++i]; } else if (arg[0] != '-') { opts.document_path = arg; } diff --git a/designer/src/testing/visual_capture.cpp b/designer/src/testing/visual_capture.cpp index fa76a23..d7e7e37 100644 --- a/designer/src/testing/visual_capture.cpp +++ b/designer/src/testing/visual_capture.cpp @@ -84,35 +84,132 @@ bool VisualCapture::CaptureScreenshot(const std::string& path) { return true; } +// Helper struct for loaded PNG data +struct PNGImage { + int width = 0; + int height = 0; + std::vector pixels; // RGBA format + bool valid = false; +}; + +// Load a PNG file into memory +static PNGImage LoadPNG(const std::string& path) { + PNGImage image; + + FILE* fp = fopen(path.c_str(), "rb"); + if (!fp) return image; + + // Check PNG signature + uint8_t header[8]; + if (fread(header, 1, 8, fp) != 8 || png_sig_cmp(header, 0, 8)) { + fclose(fp); + return image; + } + + png_structp png = png_create_read_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr); + if (!png) { + fclose(fp); + return image; + } + + png_infop info = png_create_info_struct(png); + if (!info) { + png_destroy_read_struct(&png, nullptr, nullptr); + fclose(fp); + return image; + } + + if (setjmp(png_jmpbuf(png))) { + png_destroy_read_struct(&png, &info, nullptr); + fclose(fp); + return image; + } + + png_init_io(png, fp); + png_set_sig_bytes(png, 8); + png_read_info(png, info); + + image.width = png_get_image_width(png, info); + image.height = png_get_image_height(png, info); + int color_type = png_get_color_type(png, info); + int bit_depth = png_get_bit_depth(png, info); + + // Convert to RGBA if necessary + if (bit_depth == 16) + png_set_strip_16(png); + if (color_type == PNG_COLOR_TYPE_PALETTE) + png_set_palette_to_rgb(png); + if (color_type == PNG_COLOR_TYPE_GRAY && bit_depth < 8) + png_set_expand_gray_1_2_4_to_8(png); + if (png_get_valid(png, info, PNG_INFO_tRNS)) + png_set_tRNS_to_alpha(png); + if (color_type == PNG_COLOR_TYPE_RGB || + color_type == PNG_COLOR_TYPE_GRAY || + color_type == PNG_COLOR_TYPE_PALETTE) + png_set_filler(png, 0xFF, PNG_FILLER_AFTER); + if (color_type == PNG_COLOR_TYPE_GRAY || + color_type == PNG_COLOR_TYPE_GRAY_ALPHA) + png_set_gray_to_rgb(png); + + png_read_update_info(png, info); + + // Allocate memory and read image + image.pixels.resize(image.width * image.height * 4); + std::vector row_pointers(image.height); + for (int y = 0; y < image.height; y++) { + row_pointers[y] = image.pixels.data() + y * image.width * 4; + } + + png_read_image(png, row_pointers.data()); + png_read_end(png, nullptr); + + png_destroy_read_struct(&png, &info, nullptr); + fclose(fp); + + image.valid = true; + return image; +} + +// Compare two pixels with tolerance +static bool PixelsMatch(const uint8_t* p1, const uint8_t* p2, int tolerance = 2) { + for (int c = 0; c < 4; c++) { + if (std::abs(static_cast(p1[c]) - static_cast(p2[c])) > tolerance) { + return false; + } + } + return true; +} + float VisualCapture::CompareImages(const std::string& path1, const std::string& path2) { - // Load both images and compare pixel by pixel - // For simplicity, just check if files exist for now - // A full implementation would load PNGs and compute difference - - FILE* fp1 = fopen(path1.c_str(), "rb"); - FILE* fp2 = fopen(path2.c_str(), "rb"); - - if (!fp1 || !fp2) { - if (fp1) fclose(fp1); - if (fp2) fclose(fp2); - return 1.0f; // Can't compare, assume different + // Load both images + PNGImage img1 = LoadPNG(path1); + PNGImage img2 = LoadPNG(path2); + + // Check if both loaded successfully + if (!img1.valid || !img2.valid) { + return 1.0f; // Can't compare, assume completely different } - - // Read and compare file sizes as quick check - fseek(fp1, 0, SEEK_END); - fseek(fp2, 0, SEEK_END); - long size1 = ftell(fp1); - long size2 = ftell(fp2); - - fclose(fp1); - fclose(fp2); - - // If sizes differ significantly, images are different - if (std::abs(size1 - size2) > 1000) { - return 0.5f; // Moderately different + + // Check dimensions match + if (img1.width != img2.width || img1.height != img2.height) { + return 1.0f; // Different dimensions = completely different } - - return 0.0f; // Assume similar if sizes match + + // Count differing pixels + int total_pixels = img1.width * img1.height; + int diff_pixels = 0; + + for (int i = 0; i < total_pixels; i++) { + const uint8_t* p1 = img1.pixels.data() + i * 4; + const uint8_t* p2 = img2.pixels.data() + i * 4; + + if (!PixelsMatch(p1, p2)) { + diff_pixels++; + } + } + + // Return difference ratio (0.0 = identical, 1.0 = completely different) + return static_cast(diff_pixels) / static_cast(total_pixels); } } // namespace mosis::testing