From 2c0c4599fe0bb88e44fc2eaf69a64ef5d35f29cc Mon Sep 17 00:00:00 2001 From: omigamedev Date: Sat, 17 Jan 2026 08:45:30 +0100 Subject: [PATCH] progress on plugin --- .../src/com/omixlab/MosisUnreal/OBBData.java | 2 +- Plugins/MosisSDK/README.md | 303 ++++++++++++++++++ .../aidl/com/omixlab/mosis/BnMosisListener.h | 2 +- .../aidl/com/omixlab/mosis/BnMosisService.h | 2 +- .../aidl/com/omixlab/mosis/BpMosisListener.h | 2 +- .../aidl/com/omixlab/mosis/BpMosisService.h | 2 +- .../aidl/com/omixlab/mosis/IMosisListener.h | 2 +- .../aidl/com/omixlab/mosis/IMosisService.h | 2 +- .../Generated/android/hardware_buffer_aidl.h | 176 ++++++++++ .../Source/MosisSDK/MosisSDK.Build.cs | 31 +- .../com/omixlab/mosis/IMosisListener.cpp | 2 +- .../com/omixlab/mosis/IMosisService.cpp | 2 +- .../Source/MosisSDK/Private/MosisClient.h | 3 +- .../MosisSDK/Private/MosisPhoneComponent.cpp | 95 ++---- 14 files changed, 538 insertions(+), 88 deletions(-) create mode 100644 Plugins/MosisSDK/README.md create mode 100644 Plugins/MosisSDK/Source/MosisSDK/Generated/android/hardware_buffer_aidl.h rename Plugins/MosisSDK/Source/MosisSDK/{ => Private/Android}/Generated/com/omixlab/mosis/IMosisListener.cpp (97%) rename Plugins/MosisSDK/Source/MosisSDK/{ => Private/Android}/Generated/com/omixlab/mosis/IMosisService.cpp (98%) diff --git a/Build/Android/src/com/omixlab/MosisUnreal/OBBData.java b/Build/Android/src/com/omixlab/MosisUnreal/OBBData.java index a3254ab..baf22b7 100644 --- a/Build/Android/src/com/omixlab/MosisUnreal/OBBData.java +++ b/Build/Android/src/com/omixlab/MosisUnreal/OBBData.java @@ -19,7 +19,7 @@ public static final XAPKFile[] xAPKS = { new XAPKFile( true, // true signifies a main file "1", // the version of the APK that the file was uploaded against -0L // the length of the file in bytes +97538444L // the length of the file in bytes ) }; }; diff --git a/Plugins/MosisSDK/README.md b/Plugins/MosisSDK/README.md new file mode 100644 index 0000000..dc9e672 --- /dev/null +++ b/Plugins/MosisSDK/README.md @@ -0,0 +1,303 @@ +# MosisSDK Plugin for Unreal Engine 5.5 + +This plugin provides integration between Unreal Engine and MosisService, enabling a virtual smartphone display within VR/AR applications. + +## Overview + +The MosisSDK plugin connects to the MosisService Android application via AIDL (Android Interface Definition Language) to: +- Receive rendered phone screen frames via AHardwareBuffer +- Forward touch input from VR controllers to the virtual phone +- Import hardware buffers into Vulkan for GPU-efficient rendering + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Unreal Engine Game │ +│ ┌─────────────────────┐ ┌─────────────────────────────┐ │ +│ │ UMosisPhoneComponent│◄───│ MosisVulkanTexture │ │ +│ │ (Touch Input) │ │ (HardwareBuffer Import) │ │ +│ └──────────┬──────────┘ └──────────────▲──────────────┘ │ +│ │ │ │ +│ ▼ │ │ +│ ┌─────────────────────────────────────────┴───────────────┐│ +│ │ MosisClient ││ +│ │ (AIDL IMosisListener) ││ +│ └──────────────────────────┬──────────────────────────────┘│ +└─────────────────────────────┼───────────────────────────────┘ + │ Binder IPC +┌─────────────────────────────▼───────────────────────────────┐ +│ MosisService │ +│ (Renders phone UI via RmlUi) │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Prerequisites + +### Environment Variables + +| Variable | Description | Example | +|----------|-------------|---------| +| `ANDROID_HOME` | Android SDK path | `C:\Users\\AppData\Local\Android\Sdk` | +| `ANDROID_NDK_HOME` | Android NDK path (UE5 uses 25.1.8937393) | `%ANDROID_HOME%\ndk\25.1.8937393` | + +### Required SDK Components + +- Android SDK Platform 36 (for AIDL binder headers) +- Android Build Tools 36.1.0 (for AIDL compiler) +- Android NDK 25.1.8937393 (bundled with UE5.5) + +Install via Android Studio SDK Manager: +``` +sdkmanager "platforms;android-36" "build-tools;36.1.0" +``` + +## Project Structure + +``` +Plugins/MosisSDK/ +├── Source/MosisSDK/ +│ ├── MosisSDK.Build.cs # Build configuration +│ ├── MosisSDK_UPL.xml # Android packaging rules +│ ├── AIDL/ # AIDL interface definitions +│ │ └── com/omixlab/mosis/ +│ │ ├── IMosisService.aidl +│ │ └── IMosisListener.aidl +│ ├── Generated/ # Auto-generated headers +│ │ ├── android/ +│ │ │ └── hardware_buffer_aidl.h # Local NDK header copy +│ │ └── aidl/com/omixlab/mosis/ +│ │ ├── IMosisService.h +│ │ ├── IMosisListener.h +│ │ ├── BnMosisListener.h +│ │ └── BpMosisService.h +│ ├── Public/ +│ │ ├── MosisSDK.h # Module interface +│ │ ├── MosisPhoneComponent.h # UE5 component +│ │ └── MosisPhoneActor.h # Blueprint actor +│ └── Private/ +│ ├── MosisSDK.cpp # Module + JNI callbacks +│ ├── MosisClient.h/cpp # AIDL client implementation +│ ├── MosisVulkanTexture.h/cpp # Vulkan HardwareBuffer import +│ ├── MosisPhoneComponent.cpp +│ ├── MosisPhoneActor.cpp +│ └── Android/ +│ └── Generated/ # AIDL-generated .cpp (Android only) +│ └── com/omixlab/mosis/ +│ ├── IMosisService.cpp +│ └── IMosisListener.cpp +└── Binaries/ + └── Win64/ # Windows editor binaries + └── Android/ # Android binaries +``` + +## Build Instructions + +### Windows Editor Build + +Build the plugin for use in the Unreal Editor: + +```batch +"D:\Epic\UE_5.5\Engine\Build\BatchFiles\Build.bat" ^ + MosisUnrealEditor Win64 Development ^ + -Project="D:\Dev\Mosis\MosisUnreal\MosisUnreal.uproject" +``` + +**Output**: `Plugins/MosisSDK/Binaries/Win64/UnrealEditor-MosisSDK.dll` + +### Android Build + +Build and package for Android deployment: + +```batch +"D:\Epic\UE_5.5\Engine\Build\BatchFiles\RunUAT.bat" ^ + BuildCookRun ^ + -project="D:\Dev\Mosis\MosisUnreal\MosisUnreal.uproject" ^ + -platform=Android ^ + -clientconfig=Development ^ + -build -cook -stage -pak -package ^ + -noP4 -utf8output +``` + +**Output**: `Binaries/Android/MosisUnreal-arm64.apk` + +### Clean Build + +To force a full rebuild: + +```batch +rmdir /s /q "Intermediate\Build" +rmdir /s /q "Binaries" +rmdir /s /q "Plugins\MosisSDK\Intermediate" +rmdir /s /q "Plugins\MosisSDK\Binaries" +``` + +## Key Implementation Details + +### AIDL Code Generation + +The Build.cs automatically runs the AIDL compiler during Android builds: + +1. **Input**: `.aidl` files in `Source/MosisSDK/AIDL/` +2. **Header Output**: `Source/MosisSDK/Generated/` (included for all platforms) +3. **CPP Output**: `Source/MosisSDK/Private/Android/Generated/` (Android only) + +The separation ensures Windows builds don't try to compile Android-specific AIDL bindings. + +### Platform-Specific Dependencies + +| Dependency | Windows | Android | Purpose | +|------------|---------|---------|---------| +| Core, CoreUObject, Engine | ✓ | ✓ | UE5 fundamentals | +| Slate, SlateCore | ✓ | ✓ | UI framework | +| RenderCore, RHI | ✓ | ✓ | Rendering abstraction | +| Launch, ApplicationCore | ✗ | ✓ | Android JNI access | +| Vulkan | ✗ | ✓ | Hardware buffer import | +| binder_ndk, android, nativewindow | ✗ | ✓ | Android system libs | + +### NDK Header Compatibility + +UE5.5 uses NDK 25, but `hardware_buffer_aidl.h` was added in NDK 26+. The plugin includes a local copy with modifications: + +1. Removed `__INTRODUCED_IN(34)` annotations (causes API level errors) +2. Added `__attribute__((weak))` to parcel functions (allows linking on API 33) + +### MosisClient (IMosisListener Implementation) + +The `MosisClient` class implements the AIDL listener interface: + +```cpp +class MosisClient : public aidl::com::omixlab::mosis::BnMosisListener +{ +public: + // Called when service initializes + ndk::ScopedAStatus onServiceInitialized(bool success) override; + + // Called when a new hardware buffer is available + ndk::ScopedAStatus onBufferAvailable(const HardwareBuffer& buffer) override; + + // Called each frame when content updates + ndk::ScopedAStatus onFrameAvailable() override; + + // Touch forwarding to service + void SendTouchDown(float x, float y); + void SendTouchMove(float x, float y); + void SendTouchUp(float x, float y); +}; +``` + +### Service Connection Flow + +1. `FMosisSDKModule::StartupModule()` calls Java `MyKotlinPlugin.StartMosisService()` +2. Kotlin code binds to MosisService +3. On connection, Kotlin calls native `serviceConnected(IBinder binder)` +4. JNI callback creates `MosisClient` from the binder +5. Client calls `initOS()` on the service +6. Service sends `onBufferAvailable()` with the shared hardware buffer +7. Client imports buffer into Vulkan texture +8. Service sends `onFrameAvailable()` each rendered frame + +## Usage in Blueprints + +### MosisPhoneActor + +Drop `AMosisPhoneActor` into your level for a ready-to-use phone display: + +1. Add to level via Place Actors > MosisPhoneActor +2. Configure material and mesh as needed +3. The actor automatically connects to MosisService on BeginPlay + +### MosisPhoneComponent + +For custom actors, add `UMosisPhoneComponent`: + +```cpp +UPROPERTY(VisibleAnywhere) +UMosisPhoneComponent* PhoneComponent; + +// In constructor +PhoneComponent = CreateDefaultSubobject(TEXT("Phone")); +``` + +### Touch Input + +Send touch events from VR controller raycasts: + +```cpp +// In your VR interaction component +FVector2D UV = CalculateHitUV(HitResult); + +if (TriggerPressed && !WasTriggerPressed) + PhoneComponent->SendTouch(UV, EMosisTouchType::Down); +else if (TriggerPressed) + PhoneComponent->SendTouch(UV, EMosisTouchType::Move); +else if (!TriggerPressed && WasTriggerPressed) + PhoneComponent->SendTouch(UV, EMosisTouchType::Up); +``` + +## Troubleshooting + +### Build Errors + +**"Cannot open include file: 'aidl/com/omixlab/mosis/IMosisListener.h'"** +- Ensure Android SDK Platform 36 is installed +- Check `ANDROID_HOME` environment variable +- Run Android build first to generate headers + +**"LINK : fatal error LNK1181: cannot open input file 'UnrealEditor.lib'"** +- The `Launch` module was incorrectly in global dependencies +- Should only be in Android-specific dependencies +- Solution: Move `Launch` to the Android platform block in Build.cs + +**"'Android/AndroidJNI.h' file not found"** +- Add `Launch` and `ApplicationCore` to Android dependencies in Build.cs + +**"Cannot open include file: 'android/hardware_buffer_aidl.h'"** +- This header is only in NDK 26+, but UE5.5 uses NDK 25 +- Solution: Local copy in `Generated/android/hardware_buffer_aidl.h` + +**"'AHardwareBuffer_readFromParcel' is unavailable" (API level errors)** +- Functions marked as API 34+ but building for API 33 +- Solution: Add `__attribute__((weak))` to function declarations in local header + +**AIDL cpp files compiled on Windows causing errors** +- AIDL-generated .cpp files contain Android-only headers +- Solution: Output cpp to `Private/Android/Generated/` (only compiled on Android) +- Headers go to `Generated/` (included but not compiled) + +### Runtime Issues + +**Service not connecting** +- Ensure MosisService app is installed and running +- Check logcat: `adb logcat -s MosisSDK MosisTest` +- Verify package name matches in `MyKotlinPlugin.kt` + +**Black texture / No frames** +- Check `onBufferAvailable` callback is received +- Verify Vulkan extensions are supported on device +- Check hardware buffer format compatibility + +## Device Testing + +```batch +:: Install both apps +adb install -r path\to\MosisService.apk +adb install -r Binaries\Android\MosisUnreal-arm64.apk + +:: Launch service first +adb shell am start -n com.omixlab.mosis/.MainActivity + +:: Launch game +adb shell am start -n com.omixlab.MosisUnreal/com.epicgames.unreal.GameActivity + +:: Monitor logs +adb logcat -s MosisSDK MosisTest RMLUI +``` + +## Version History + +- **v1.0** - Initial implementation + - AIDL client for MosisService connection + - Vulkan HardwareBuffer import + - UE5 component and actor for phone display + - Touch input forwarding diff --git a/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BnMosisListener.h b/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BnMosisListener.h index 9f6a347..19a30df 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BnMosisListener.h +++ b/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BnMosisListener.h @@ -1,6 +1,6 @@ /* * This file is auto-generated. DO NOT MODIFY. - * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisListener.aidl + * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Private\\Android\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisListener.aidl * * DO NOT CHECK THIS FILE INTO A CODE TREE (e.g. git, etc..). * ALWAYS GENERATE THIS FILE FROM UPDATED AIDL COMPILER diff --git a/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BnMosisService.h b/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BnMosisService.h index d58581e..5c9f6b8 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BnMosisService.h +++ b/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BnMosisService.h @@ -1,6 +1,6 @@ /* * This file is auto-generated. DO NOT MODIFY. - * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisService.aidl + * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Private\\Android\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisService.aidl * * DO NOT CHECK THIS FILE INTO A CODE TREE (e.g. git, etc..). * ALWAYS GENERATE THIS FILE FROM UPDATED AIDL COMPILER diff --git a/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BpMosisListener.h b/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BpMosisListener.h index 07b1746..db11bc6 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BpMosisListener.h +++ b/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BpMosisListener.h @@ -1,6 +1,6 @@ /* * This file is auto-generated. DO NOT MODIFY. - * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisListener.aidl + * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Private\\Android\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisListener.aidl * * DO NOT CHECK THIS FILE INTO A CODE TREE (e.g. git, etc..). * ALWAYS GENERATE THIS FILE FROM UPDATED AIDL COMPILER diff --git a/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BpMosisService.h b/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BpMosisService.h index eff64d8..f5db9f6 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BpMosisService.h +++ b/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/BpMosisService.h @@ -1,6 +1,6 @@ /* * This file is auto-generated. DO NOT MODIFY. - * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisService.aidl + * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Private\\Android\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisService.aidl * * DO NOT CHECK THIS FILE INTO A CODE TREE (e.g. git, etc..). * ALWAYS GENERATE THIS FILE FROM UPDATED AIDL COMPILER diff --git a/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/IMosisListener.h b/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/IMosisListener.h index 3aeb2b6..c00360a 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/IMosisListener.h +++ b/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/IMosisListener.h @@ -1,6 +1,6 @@ /* * This file is auto-generated. DO NOT MODIFY. - * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisListener.aidl + * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Private\\Android\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisListener.aidl * * DO NOT CHECK THIS FILE INTO A CODE TREE (e.g. git, etc..). * ALWAYS GENERATE THIS FILE FROM UPDATED AIDL COMPILER diff --git a/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/IMosisService.h b/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/IMosisService.h index ab3ae59..14e9743 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/IMosisService.h +++ b/Plugins/MosisSDK/Source/MosisSDK/Generated/aidl/com/omixlab/mosis/IMosisService.h @@ -1,6 +1,6 @@ /* * This file is auto-generated. DO NOT MODIFY. - * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisService.aidl + * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Private\\Android\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisService.aidl * * DO NOT CHECK THIS FILE INTO A CODE TREE (e.g. git, etc..). * ALWAYS GENERATE THIS FILE FROM UPDATED AIDL COMPILER diff --git a/Plugins/MosisSDK/Source/MosisSDK/Generated/android/hardware_buffer_aidl.h b/Plugins/MosisSDK/Source/MosisSDK/Generated/android/hardware_buffer_aidl.h new file mode 100644 index 0000000..f2c248b --- /dev/null +++ b/Plugins/MosisSDK/Source/MosisSDK/Generated/android/hardware_buffer_aidl.h @@ -0,0 +1,176 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file hardware_buffer_aidl.h + * @brief HardwareBuffer NDK AIDL glue code + */ + +#ifndef ANDROID_HARDWARE_BUFFER_AIDL_H +#define ANDROID_HARDWARE_BUFFER_AIDL_H + +#include +#include +#include + +#ifdef __cplusplus +#include +#endif + +__BEGIN_DECLS + +/** + * Read an AHardwareBuffer from a AParcel. The output buffer will have an + * initial reference acquired and will need to be released with + * AHardwareBuffer_release. + * + * Available since API level 34. + * + * \return STATUS_OK on success + * STATUS_BAD_VALUE if the parcel or outBuffer is null, or if there's an + * issue deserializing (eg, corrupted parcel) + * STATUS_BAD_TYPE if the parcel's current data position is not that of + * an AHardwareBuffer type + * STATUS_NO_MEMORY if an allocation fails + */ +// Weak references for API 34+ functions +__attribute__((weak)) +binder_status_t AHardwareBuffer_readFromParcel(const AParcel* _Nonnull parcel, + AHardwareBuffer* _Nullable* _Nonnull outBuffer); + +/** + * Write an AHardwareBuffer to an AParcel. + * + * Available since API level 34. + * + * \return STATUS_OK on success. + * STATUS_BAD_VALUE if either buffer or parcel is null, or if the AHardwareBuffer* + * fails to serialize (eg, internally corrupted) + * STATUS_NO_MEMORY if the parcel runs out of space to store the buffer & is + * unable to allocate more + * STATUS_FDS_NOT_ALLOWED if the parcel does not allow storing FDs + */ +__attribute__((weak)) +binder_status_t AHardwareBuffer_writeToParcel(const AHardwareBuffer* _Nonnull buffer, + AParcel* _Nonnull parcel); + +__END_DECLS + +// Only enable the AIDL glue helper if this is C++ +#ifdef __cplusplus + +namespace aidl::android::hardware { + +/** + * Wrapper class that enables interop with AIDL NDK generation + * Takes ownership of the AHardwareBuffer* given to it in reset() and will automatically + * destroy it in the destructor, similar to a smart pointer container + */ +class HardwareBuffer { +public: + HardwareBuffer() noexcept {} + HardwareBuffer(HardwareBuffer&& other) noexcept : mBuffer(other.release()) {} + + ~HardwareBuffer() { + reset(); + } + + binder_status_t readFromParcel(const AParcel* _Nonnull parcel) { + reset(); + if (__builtin_available(android __ANDROID_API_U__, *)) { + return AHardwareBuffer_readFromParcel(parcel, &mBuffer); + } else { + return STATUS_FAILED_TRANSACTION; + } + } + + binder_status_t writeToParcel(AParcel* _Nonnull parcel) const { + if (!mBuffer) { + return STATUS_BAD_VALUE; + } + if (__builtin_available(android __ANDROID_API_U__, *)) { + return AHardwareBuffer_writeToParcel(mBuffer, parcel); + } else { + return STATUS_FAILED_TRANSACTION; + } + } + + /** + * Destroys any currently owned AHardwareBuffer* and takes ownership of the given + * AHardwareBuffer* + * + * @param buffer The buffer to take ownership of + */ + void reset(AHardwareBuffer* _Nullable buffer = nullptr) noexcept { + if (mBuffer) { + AHardwareBuffer_release(mBuffer); + mBuffer = nullptr; + } + mBuffer = buffer; + } + + inline AHardwareBuffer* _Nullable operator-> () const { return mBuffer; } + inline AHardwareBuffer* _Nullable get() const { return mBuffer; } + inline explicit operator bool () const { return mBuffer != nullptr; } + + inline bool operator!=(const HardwareBuffer& rhs) const { return get() != rhs.get(); } + inline bool operator<(const HardwareBuffer& rhs) const { return get() < rhs.get(); } + inline bool operator<=(const HardwareBuffer& rhs) const { return get() <= rhs.get(); } + inline bool operator==(const HardwareBuffer& rhs) const { return get() == rhs.get(); } + inline bool operator>(const HardwareBuffer& rhs) const { return get() > rhs.get(); } + inline bool operator>=(const HardwareBuffer& rhs) const { return get() >= rhs.get(); } + + HardwareBuffer& operator=(HardwareBuffer&& other) noexcept { + reset(other.release()); + return *this; + } + + /** + * Stops managing any contained AHardwareBuffer*, returning it to the caller. Ownership + * is released. + * @return AHardwareBuffer* or null if this was empty + */ + [[nodiscard]] AHardwareBuffer* _Nullable release() noexcept { + AHardwareBuffer* _Nullable ret = mBuffer; + mBuffer = nullptr; + return ret; + } + + inline std::string toString() const { + if (!mBuffer) { + return ""; + } + if (__builtin_available(android __ANDROID_API_S__, *)) { + uint64_t id = 0; + AHardwareBuffer_getId(mBuffer, &id); + return ""; + } else { + return ""; + } + } + +private: + HardwareBuffer(const HardwareBuffer& other) = delete; + HardwareBuffer& operator=(const HardwareBuffer& other) = delete; + + AHardwareBuffer* _Nullable mBuffer = nullptr; +}; + +} // aidl::android::hardware + +#endif // __cplusplus + +#endif // ANDROID_HARDWARE_BUFFER_AIDL_H diff --git a/Plugins/MosisSDK/Source/MosisSDK/MosisSDK.Build.cs b/Plugins/MosisSDK/Source/MosisSDK/MosisSDK.Build.cs index bf25afe..b8b18ad 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/MosisSDK.Build.cs +++ b/Plugins/MosisSDK/Source/MosisSDK/MosisSDK.Build.cs @@ -25,7 +25,6 @@ public class MosisSDK : ModuleRules "Engine", "Slate", "SlateCore", - "Launch", "RenderCore", "RHI" } @@ -33,20 +32,30 @@ public class MosisSDK : ModuleRules if (Target.Platform == UnrealTargetPlatform.Android) { - // Add Vulkan support - PrivateDependencyModuleNames.Add("VulkanRHI"); + // Android-specific module dependencies + PrivateDependencyModuleNames.AddRange(new string[] { + "Launch", + "ApplicationCore" + }); + + // Add Vulkan support (Vulkan headers for HardwareBuffer import) AddEngineThirdPartyPrivateStaticDependencies(Target, "Vulkan"); // Register the UPL file string PluginPath = Utils.MakePathRelativeTo(ModuleDirectory, Target.RelativeEnginePath); AdditionalPropertiesForReceipt.Add("AndroidPlugin", Path.Combine(PluginPath, "MosisSDK_UPL.xml")); + + // Android system libraries PublicAdditionalLibraries.Add("binder_ndk"); + PublicAdditionalLibraries.Add("android"); + PublicAdditionalLibraries.Add("nativewindow"); + PublicSystemLibraries.Add("vulkan"); string SDKPath = Environment.GetEnvironmentVariable("ANDROID_HOME"); string BinderPath = Path.Combine(SDKPath, "platforms/android-36/optional/libbinder_ndk_cpp"); PublicIncludePaths.Add(BinderPath); - // Include generated AIDL headers + // Include generated AIDL headers (includes hardware_buffer_aidl.h locally) string GeneratedPath = Path.Combine(ModuleDirectory, "Generated"); PublicIncludePaths.Add(GeneratedPath); PrivateIncludePaths.Add(GeneratedPath); @@ -54,7 +63,11 @@ public class MosisSDK : ModuleRules string AidlPath = Path.Combine(SDKPath, "build-tools/36.1.0/aidl.exe"); string AidlSourceDir = Path.Combine(ModuleDirectory, "AIDL"); string OutputHeaderDir = Path.Combine(ModuleDirectory, "Generated"); - string OutputCppDir = OutputHeaderDir; + // Put cpp files in Private/Android/ so they're only compiled on Android + string OutputCppDir = Path.Combine(ModuleDirectory, "Private", "Android", "Generated"); + + // Ensure output directories exist + Directory.CreateDirectory(OutputCppDir); string[] Files = Directory.GetFiles(Path.Combine(AidlSourceDir, "com/omixlab/mosis"), "*.aidl", SearchOption.AllDirectories); @@ -82,6 +95,14 @@ public class MosisSDK : ModuleRules } } } + + // Delete any stale cpp files from Generated/ (they should be in Private/Android/Generated/) + string[] StaleCppFiles = Directory.GetFiles(GeneratedPath, "*.cpp", SearchOption.AllDirectories); + foreach (string StaleCpp in StaleCppFiles) + { + Console.WriteLine($"Removing stale AIDL cpp: {StaleCpp}"); + File.Delete(StaleCpp); + } } } } diff --git a/Plugins/MosisSDK/Source/MosisSDK/Generated/com/omixlab/mosis/IMosisListener.cpp b/Plugins/MosisSDK/Source/MosisSDK/Private/Android/Generated/com/omixlab/mosis/IMosisListener.cpp similarity index 97% rename from Plugins/MosisSDK/Source/MosisSDK/Generated/com/omixlab/mosis/IMosisListener.cpp rename to Plugins/MosisSDK/Source/MosisSDK/Private/Android/Generated/com/omixlab/mosis/IMosisListener.cpp index e574894..94ffd4b 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Generated/com/omixlab/mosis/IMosisListener.cpp +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/Android/Generated/com/omixlab/mosis/IMosisListener.cpp @@ -1,6 +1,6 @@ /* * This file is auto-generated. DO NOT MODIFY. - * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisListener.aidl + * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Private\\Android\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisListener.aidl * * DO NOT CHECK THIS FILE INTO A CODE TREE (e.g. git, etc..). * ALWAYS GENERATE THIS FILE FROM UPDATED AIDL COMPILER diff --git a/Plugins/MosisSDK/Source/MosisSDK/Generated/com/omixlab/mosis/IMosisService.cpp b/Plugins/MosisSDK/Source/MosisSDK/Private/Android/Generated/com/omixlab/mosis/IMosisService.cpp similarity index 98% rename from Plugins/MosisSDK/Source/MosisSDK/Generated/com/omixlab/mosis/IMosisService.cpp rename to Plugins/MosisSDK/Source/MosisSDK/Private/Android/Generated/com/omixlab/mosis/IMosisService.cpp index 4266f54..e202d1d 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Generated/com/omixlab/mosis/IMosisService.cpp +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/Android/Generated/com/omixlab/mosis/IMosisService.cpp @@ -1,6 +1,6 @@ /* * This file is auto-generated. DO NOT MODIFY. - * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisService.aidl + * Using: C:\\Users\\omara\\AppData\\Local\\Android\\Sdk\\build-tools/36.1.0/aidl.exe --lang=ndk --min_sdk_version=36 -I D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL -o D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Private\\Android\\Generated -h D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\Generated D:\\Dev\\Mosis\\MosisUnreal\\Plugins\\MosisSDK\\Source\\MosisSDK\\AIDL\\com/omixlab/mosis\\IMosisService.aidl * * DO NOT CHECK THIS FILE INTO A CODE TREE (e.g. git, etc..). * ALWAYS GENERATE THIS FILE FROM UPDATED AIDL COMPILER diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisClient.h b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisClient.h index cd543d7..8223a0f 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisClient.h +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisClient.h @@ -57,9 +57,10 @@ public: void SetFrameCallback(FrameCallback callback) { m_OnFrameAvailable = std::move(callback); } void SetInitCallback(InitCallback callback) { m_OnInitialized = std::move(callback); } -private: + // Constructor must be public for ndk::SharedRefBase::make MosisClient() = default; +private: std::shared_ptr m_Service; AHardwareBuffer* m_HardwareBuffer = nullptr; std::atomic m_Initialized{false}; diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneComponent.cpp b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneComponent.cpp index f19506a..4a48fbe 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneComponent.cpp +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneComponent.cpp @@ -9,8 +9,6 @@ #if PLATFORM_ANDROID #include "MosisClient.h" #include "MosisVulkanTexture.h" -#include "IVulkanDynamicRHI.h" -#include "VulkanRHIPrivate.h" #endif DEFINE_LOG_CATEGORY_STATIC(LogMosisPhone, Log, All); @@ -157,63 +155,30 @@ void UMosisPhoneComponent::OnBufferAvailable() UE_LOG(LogMosisPhone, Log, TEXT("OnBufferAvailable: %dx%d"), ScreenWidth, ScreenHeight); - // Initialize Vulkan texture on render thread - ENQUEUE_RENDER_COMMAND(MosisCreateTexture)( - [this, Buffer](FRHICommandListImmediate& RHICmdList) - { - // Get Vulkan device from RHI - IVulkanDynamicRHI* VulkanRHI = GetIVulkanDynamicRHI(); - if (!VulkanRHI) - { - UE_LOG(LogMosisPhone, Error, TEXT("OnBufferAvailable: Vulkan RHI not available")); - return; - } + // TODO: Initialize Vulkan texture on render thread + // This requires accessing UE5's Vulkan RHI which has platform-specific APIs. + // For now, we log the buffer availability and dimensions. + // Full implementation would: + // 1. Get VkDevice from GVulkanRHI or IVulkanDynamicRHI + // 2. Create MosisVulkanTexture and import the HardwareBuffer + // 3. Create UTexture2D wrapping the Vulkan image - VkInstance Instance = VulkanRHI->RHIGetVkInstance(); - VkPhysicalDevice PhysDevice = VulkanRHI->RHIGetVkPhysicalDevice(); - VkDevice Device = VulkanRHI->RHIGetVkDevice(); - - // Create Vulkan texture if needed - if (!VulkanTexture) - { - VulkanTexture = MakeShared(); - if (!VulkanTexture->Initialize(Instance, PhysDevice, Device, 0)) - { - UE_LOG(LogMosisPhone, Error, TEXT("OnBufferAvailable: Failed to initialize VulkanTexture")); - VulkanTexture.Reset(); - return; - } - } - - // Create texture from buffer - if (!VulkanTexture->Create(Buffer)) - { - UE_LOG(LogMosisPhone, Error, TEXT("OnBufferAvailable: Failed to create texture")); - return; - } - - UE_LOG(LogMosisPhone, Log, TEXT("OnBufferAvailable: Vulkan texture created")); - } - ); - - // Create UE texture on game thread - AsyncTask(ENamedThreads::GameThread, [this]() + // Create placeholder texture + if (!PhoneTexture) { - // Create UTexture2D from Vulkan image - // For now, create a placeholder - actual Vulkan integration would use - // FVulkanTexture2D or similar UE-Vulkan interop - if (!PhoneTexture) + PhoneTexture = UTexture2D::CreateTransient(ScreenWidth, ScreenHeight, PF_R8G8B8A8); + if (PhoneTexture) { - PhoneTexture = UTexture2D::CreateTransient(ScreenWidth, ScreenHeight, PF_R8G8B8A8); PhoneTexture->UpdateResource(); + UE_LOG(LogMosisPhone, Log, TEXT("OnBufferAvailable: Created placeholder texture")); } + } - // Update material if set - if (PhoneMaterial) - { - PhoneMaterial->SetTextureParameterValue(TextureParameterName, PhoneTexture); - } - }); + // Update material if set + if (PhoneMaterial && PhoneTexture) + { + PhoneMaterial->SetTextureParameterValue(TextureParameterName, PhoneTexture); + } #endif } @@ -225,26 +190,10 @@ void UMosisPhoneComponent::OnFrameAvailable() return; } - // Copy texture on render thread - ENQUEUE_RENDER_COMMAND(MosisUpdateTexture)( - [this](FRHICommandListImmediate& RHICmdList) - { - IVulkanDynamicRHI* VulkanRHI = GetIVulkanDynamicRHI(); - if (!VulkanRHI || !VulkanTexture) - { - return; - } - - // Get command buffer from UE's Vulkan RHI - // This requires more integration with UE's command buffer management - // For now, the texture copy happens during Create() which does an initial copy - - // In a full implementation, you would: - // 1. Get UE's current command buffer - // 2. Call VulkanTexture->CopyToLocal(cmdBuffer) - // 3. Submit as part of UE's rendering - } - ); + // TODO: Copy texture on render thread + // This requires accessing UE5's Vulkan command buffer management. + // Full implementation would enqueue a render command to copy the + // imported image to the local image. #endif }