From e7a514a7130a9434a21f5831ef4c025060ec0438 Mon Sep 17 00:00:00 2001 From: omigamedev Date: Sat, 17 Jan 2026 08:46:11 +0100 Subject: [PATCH] progress --- Assets/PhoneInteraction.cs | 77 ++- .../Plugins/Android/cpp/CMakeLists.txt | 21 +- .../Plugins/Android/cpp/my_native_code.cpp | 287 ++++++----- .../Plugins/Android/cpp/opengl_backend.cpp | 178 +++++++ .../Plugins/Android/cpp/opengl_backend.h | 42 ++ .../Plugins/Android/cpp/texture_backend.h | 51 ++ .../Plugins/Android/cpp/vulkan_backend.cpp | 465 ++++++++++++++++++ .../Plugins/Android/cpp/vulkan_backend.h | 82 +++ Packages/com.omarator.mosissdk/README.md | 222 ++++++++- .../Runtime/KotlinBridge.cs | 72 ++- 10 files changed, 1371 insertions(+), 126 deletions(-) create mode 100644 Packages/com.omarator.mosissdk/Plugins/Android/cpp/opengl_backend.cpp create mode 100644 Packages/com.omarator.mosissdk/Plugins/Android/cpp/opengl_backend.h create mode 100644 Packages/com.omarator.mosissdk/Plugins/Android/cpp/texture_backend.h create mode 100644 Packages/com.omarator.mosissdk/Plugins/Android/cpp/vulkan_backend.cpp create mode 100644 Packages/com.omarator.mosissdk/Plugins/Android/cpp/vulkan_backend.h diff --git a/Assets/PhoneInteraction.cs b/Assets/PhoneInteraction.cs index 319728e..b492c47 100644 --- a/Assets/PhoneInteraction.cs +++ b/Assets/PhoneInteraction.cs @@ -1,20 +1,89 @@ using UnityEngine; +using UnityEngine.XR; public class PhoneInteraction : MonoBehaviour { public GameObject Controller; public GameObject Plane; + [Header("Input")] + [Tooltip("The XR input device to read trigger from. Leave at None for automatic detection.")] + public XRNode inputDevice = XRNode.RightHand; + + [Tooltip("Trigger threshold to register as pressed")] + [Range(0.1f, 0.9f)] + public float triggerThreshold = 0.5f; + + private bool wasTouching = false; + private bool wasTriggerPressed = false; + private Vector2 lastNormalized; + void Update() { var t = Controller.GetComponent(); - if (Physics.Raycast(t.position, t.forward, out RaycastHit hit) && hit.collider == Plane.GetComponent()) + + // Check if ray hits the phone plane + bool isTouching = Physics.Raycast(t.position, t.forward, out RaycastHit hit) && + hit.collider == Plane.GetComponent(); + + if (isTouching) { + // Calculate normalized UV coordinates var local = Plane.transform.InverseTransformPoint(hit.point); var normalized = new Vector2((local.x + 5) / 10, (local.z + 5) / 10); - KotlinBridge.SendTouchMove(normalized.x, 1 - normalized.y); - //Debug.Log("MOVE " + normalized); - //Debug.DrawLine(t.position, hit.point, Color.green); + lastNormalized = new Vector2(normalized.x, 1 - normalized.y); + + // Check trigger input + bool triggerPressed = IsTriggerPressed(); + + if (triggerPressed) + { + if (!wasTriggerPressed) + { + // Trigger just pressed - send touch down + KotlinBridge.SendTouchDown(lastNormalized.x, lastNormalized.y); + } + else + { + // Trigger held - send touch move + KotlinBridge.SendTouchMove(lastNormalized.x, lastNormalized.y); + } + } + else if (wasTriggerPressed) + { + // Trigger just released - send touch up + KotlinBridge.SendTouchUp(lastNormalized.x, lastNormalized.y); + } + + wasTriggerPressed = triggerPressed; } + else if (wasTriggerPressed) + { + // Lost contact while touching - send touch up at last position + KotlinBridge.SendTouchUp(lastNormalized.x, lastNormalized.y); + wasTriggerPressed = false; + } + + wasTouching = isTouching; + } + + private bool IsTriggerPressed() + { + // Try to get trigger value from XR input + InputDevice device = InputDevices.GetDeviceAtXRNode(inputDevice); + if (device.isValid) + { + if (device.TryGetFeatureValue(CommonUsages.trigger, out float triggerValue)) + { + return triggerValue > triggerThreshold; + } + if (device.TryGetFeatureValue(CommonUsages.triggerButton, out bool triggerButton)) + { + return triggerButton; + } + } + + // Fallback to Unity's legacy input (for testing in editor) + return Input.GetMouseButton(0); } } diff --git a/Packages/com.omarator.mosissdk/Plugins/Android/cpp/CMakeLists.txt b/Packages/com.omarator.mosissdk/Plugins/Android/cpp/CMakeLists.txt index 26c5494..48820c9 100644 --- a/Packages/com.omarator.mosissdk/Plugins/Android/cpp/CMakeLists.txt +++ b/Packages/com.omarator.mosissdk/Plugins/Android/cpp/CMakeLists.txt @@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.22.1) project("MyNativePlugin") find_library(log-lib log) +find_library(vulkan-lib vulkan) set(CMAKE_CXX_STANDARD 23) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -35,6 +36,8 @@ add_custom_command( add_library(my_native_lib SHARED my_native_code.cpp + opengl_backend.cpp + vulkan_backend.cpp ${CMAKE_CURRENT_BINARY_DIR}/com/omixlab/mosis/IMosisService.cpp ${CMAKE_CURRENT_BINARY_DIR}/com/omixlab/mosis/IMosisListener.cpp ${SHARED_SRC_DIR}/logger.cpp @@ -43,11 +46,27 @@ add_library(my_native_lib SHARED ${SHARED_SRC_DIR}/glad/src/egl.c ${SHARED_SRC_DIR}/glad/src/gles2.c ) -target_link_libraries(my_native_lib ${log-lib} binder_ndk EGL GLESv2 nativewindow) + +target_link_libraries(my_native_lib + ${log-lib} + ${vulkan-lib} + binder_ndk + EGL + GLESv2 + nativewindow +) + target_include_directories(my_native_lib PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} ${SHARED_SRC_DIR} ${SHARED_SRC_DIR}/glad/include ${CMAKE_CURRENT_BINARY_DIR} ${BINDER_DIR} ${PLUGIN_API} ) + +# Vulkan support definitions +target_compile_definitions(my_native_lib PRIVATE + VK_USE_PLATFORM_ANDROID_KHR + MOSIS_VULKAN_SUPPORT=1 +) diff --git a/Packages/com.omarator.mosissdk/Plugins/Android/cpp/my_native_code.cpp b/Packages/com.omarator.mosissdk/Plugins/Android/cpp/my_native_code.cpp index a26c258..9d94e28 100644 --- a/Packages/com.omarator.mosissdk/Plugins/Android/cpp/my_native_code.cpp +++ b/Packages/com.omarator.mosissdk/Plugins/Android/cpp/my_native_code.cpp @@ -5,90 +5,29 @@ #include #include #include -#include -#include #include -#include -#include + +#include "texture_backend.h" +#include "opengl_backend.h" +#include "vulkan_backend.h" using namespace aidl::com::omixlab::mosis; using namespace aidl::android::hardware; +// Global state std::shared_ptr g_service; std::shared_ptr g_context; +std::unique_ptr g_backend; +IUnityInterfaces* g_unityInterfaces = nullptr; +UnityGfxRenderer g_rendererType = kUnityGfxRendererNull; -class TextureBlitter -{ - GLuint source_texture = 0; - GLuint source_fb = 0; - GLuint dest_texture = 0; - GLuint dest_fb = 0; - GLint width; - GLint height; -public: - bool create(AHardwareBuffer* hardwareBuffer) - { - AHardwareBuffer_Desc desc{}; - AHardwareBuffer_describe(hardwareBuffer, &desc); - width = desc.width; - height = desc.height; - EGLClientBuffer clientBuffer = eglGetNativeClientBufferANDROID(hardwareBuffer); - EGLImageKHR eglImage = eglCreateImageKHR(eglGetCurrentDisplay(), EGL_NO_CONTEXT, - EGL_NATIVE_BUFFER_ANDROID, clientBuffer, nullptr); - if (eglImage == EGL_NO_IMAGE_KHR) - { - Logger::Log("Failed to create EGL image"); - return false; - } - glGenTextures(1, &source_texture); - glBindTexture(GL_TEXTURE_EXTERNAL_OES, source_texture); - glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, eglImage); - eglDestroyImageKHR(eglGetCurrentDisplay(), eglImage); - - glGenFramebuffers(1, &source_fb); - glBindFramebuffer(GL_FRAMEBUFFER, source_fb); - glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, - GL_TEXTURE_EXTERNAL_OES, source_texture, 0); - if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) - return false; - glGenTextures(1, &dest_texture); - glBindTexture(GL_TEXTURE_2D, dest_texture); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, desc.width, desc.height, - 0, GL_RGB, GL_UNSIGNED_BYTE, nullptr); - glGenFramebuffers(1, &dest_fb); - glBindFramebuffer(GL_FRAMEBUFFER, dest_fb); - glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, - GL_TEXTURE_2D, dest_texture, 0); - if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) - return false; - blit(); - return true; - } - [[nodiscard]] GLuint dest_texture_id() const - { - return dest_texture; - } - void blit() - { - glBindFramebuffer(GL_READ_FRAMEBUFFER, source_fb); - glBindFramebuffer(GL_DRAW_FRAMEBUFFER, dest_fb); - glBlitFramebuffer(0, 0, width, height, 0, 0, width, height, GL_COLOR_BUFFER_BIT, GL_NEAREST); - } -}; - +// Callback function pointers typedef void (*OnMessageCallback)(const char*); typedef void (*OnServiceInitializedCallback)(bool success); typedef void (*OnFrameAvailableCallback)(); typedef void (*OnBufferReadyCallback)(); -typedef void (*OnTextureReadyCallback)(GLuint gl_texture); +typedef void (*OnTextureReadyCallback)(void* nativeTexturePtr, int width, int height, bool isVulkan); + struct NativeCallbacks { OnMessageCallback OnMessage; @@ -102,49 +41,156 @@ NativeCallbacks g_callbacks{}; class ServiceContext : public BnMosisListener { AHardwareBuffer* m_hwbuffer = nullptr; - std::unique_ptr m_texture; + public: ndk::ScopedAStatus onServiceInitialized(bool in_success) override { Logger::Log("onServiceInitialized"); - g_callbacks.OnMessage("onServiceInitialized"); - g_callbacks.OnServiceInitialized(in_success); + if (g_callbacks.OnMessage) + g_callbacks.OnMessage("onServiceInitialized"); + if (g_callbacks.OnServiceInitialized) + g_callbacks.OnServiceInitialized(in_success); return ndk::ScopedAStatus::ok(); } + ndk::ScopedAStatus onFrameAvailable() override { Logger::Log("onFrameAvailable"); - g_callbacks.OnMessage("onFrameAvailable"); - g_callbacks.OnFrameAvailable(); + if (g_callbacks.OnMessage) + g_callbacks.OnMessage("onFrameAvailable"); + if (g_callbacks.OnFrameAvailable) + g_callbacks.OnFrameAvailable(); return ndk::ScopedAStatus::ok(); } + ndk::ScopedAStatus onBufferAvailable(const HardwareBuffer &in_buffer) override { Logger::Log("onBufferAvailable"); - g_callbacks.OnMessage("onBufferAvailable"); + if (g_callbacks.OnMessage) + g_callbacks.OnMessage("onBufferAvailable"); m_hwbuffer = in_buffer.get(); AHardwareBuffer_acquire(m_hwbuffer); - g_callbacks.OnBufferReady(); + if (g_callbacks.OnBufferReady) + g_callbacks.OnBufferReady(); return ndk::ScopedAStatus::ok(); } - [[nodiscard]] GLuint create_texture() + + bool CreateTexture() { - m_texture = std::make_unique(); - m_texture->create(m_hwbuffer); - return m_texture->dest_texture_id(); + if (!m_hwbuffer) + { + Logger::Log("CreateTexture: No hardware buffer available"); + return false; + } + + if (!g_backend) + { + Logger::Log("CreateTexture: No backend available"); + return false; + } + + if (!g_backend->Create(m_hwbuffer)) + { + Logger::Log("CreateTexture: Backend Create failed"); + return false; + } + + // Notify C# with texture info + if (g_callbacks.OnTextureReady) + { + g_callbacks.OnTextureReady( + g_backend->GetNativeTexturePtr(), + g_backend->GetWidth(), + g_backend->GetHeight(), + g_backend->IsVulkan() + ); + } + + return true; } - void update_texture() + + void UpdateTexture() { - if (m_texture) - m_texture->blit(); + if (g_backend) + { + g_backend->Update(); + } } }; +// Unity Plugin Interface +extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API +UnityPluginLoad(IUnityInterfaces* unityInterfaces) +{ + g_unityInterfaces = unityInterfaces; + + IUnityGraphics* graphics = unityInterfaces->Get(); + if (!graphics) + { + Logger::Log("UnityPluginLoad: IUnityGraphics not available"); + return; + } + + g_rendererType = graphics->GetRenderer(); + Logger::Log(std::format("UnityPluginLoad: Renderer type = {}", static_cast(g_rendererType))); + + // Try to create Vulkan backend first + if (g_rendererType == kUnityGfxRendererVulkan) + { + auto vulkanBackend = std::make_unique(); + if (vulkanBackend->Initialize(unityInterfaces)) + { + g_backend = std::move(vulkanBackend); + Logger::Log("UnityPluginLoad: Using Vulkan backend"); + return; + } + Logger::Log("UnityPluginLoad: Vulkan initialization failed, falling back to OpenGL"); + } + + // Fall back to OpenGL for GLES renderers + if (g_rendererType == kUnityGfxRendererOpenGLES20 || + g_rendererType == kUnityGfxRendererOpenGLES30) + { + g_backend = std::make_unique(); + Logger::Log("UnityPluginLoad: Using OpenGL backend"); + } + else + { + Logger::Log(std::format("UnityPluginLoad: Unsupported renderer type {}", static_cast(g_rendererType))); + } +} + +extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API +UnityPluginUnload() +{ + Logger::Log("UnityPluginUnload"); + + // Clean up backend + if (g_backend) + { + g_backend->Destroy(); + g_backend.reset(); + } + + g_unityInterfaces = nullptr; + g_rendererType = kUnityGfxRendererNull; +} + +// Native callback setters extern "C" void SetNativeCallbacks(const NativeCallbacks& ptr) { g_callbacks = ptr; } +// Touch input forwarding +extern "C" void SendTouchDown(float x, float y) +{ + if (g_service) + { + g_service->onTouchDown(x, y); + } +} + extern "C" void SendTouchMove(float x, float y) { if (g_service) @@ -153,40 +199,60 @@ extern "C" void SendTouchMove(float x, float y) } } -extern "C" UnityRenderingEvent InitGLAD() +extern "C" void SendTouchUp(float x, float y) { - return [](int eventId){ - gladLoaderLoadEGL(EGL_NO_DISPLAY); - int egl_version = gladLoadEGL(eglGetCurrentDisplay(), eglGetProcAddress); - if (egl_version == 0) + if (g_service) + { + g_service->onTouchUp(x, y); + } +} + +// Unity render thread callbacks +static void UNITY_INTERFACE_API OnInitBackendRenderThread(int eventId) +{ + Logger::Log("OnInitBackendRenderThread"); + + // For OpenGL, we need to initialize GLAD on the render thread + if (!g_backend->IsVulkan()) + { + if (!OpenGLTextureBackend::InitializeGLAD()) { - Logger::Log("Failed to load EGL"); + Logger::Log("OnInitBackendRenderThread: Failed to initialize GLAD"); return; } - int gl_version = gladLoaderLoadGLES2(); - if (gl_version == 0) - { - Logger::Log("Failed to load GL"); - return; - } - if (g_context) - { - GLuint texture = g_context->create_texture(); - g_callbacks.OnTextureReady(texture); - } - }; + } + + if (g_context) + { + g_context->CreateTexture(); + } +} + +static void UNITY_INTERFACE_API OnUpdateTextureRenderThread(int eventId) +{ + if (g_context) + { + g_context->UpdateTexture(); + } +} + +extern "C" UnityRenderingEvent InitBackend() +{ + return OnInitBackendRenderThread; } extern "C" UnityRenderingEvent UpdateTexture() { - return [](int eventId){ - if (g_context) - { - g_context->update_texture(); - } - }; + return OnUpdateTextureRenderThread; } +// Legacy compatibility - redirect to new names +extern "C" UnityRenderingEvent InitGLAD() +{ + return InitBackend(); +} + +// JNI entry point from Kotlin extern "C" JNIEXPORT void JNICALL Java_com_omixlab_mosis_unity_MyKotlinPlugin_serviceConnected(JNIEnv *env, jobject thiz, @@ -196,6 +262,7 @@ Java_com_omixlab_mosis_unity_MyKotlinPlugin_serviceConnected(JNIEnv *env, jobjec const ndk::SpAIBinder spBinder(pBinder); g_service = IMosisService::fromBinder(spBinder); Logger::Log("Service Connected"); + g_context = ndk::SharedRefBase::make(); bool result{}; g_service->initOS(g_context, &result); diff --git a/Packages/com.omarator.mosissdk/Plugins/Android/cpp/opengl_backend.cpp b/Packages/com.omarator.mosissdk/Plugins/Android/cpp/opengl_backend.cpp new file mode 100644 index 0000000..c3d91bb --- /dev/null +++ b/Packages/com.omarator.mosissdk/Plugins/Android/cpp/opengl_backend.cpp @@ -0,0 +1,178 @@ +#include "opengl_backend.h" +#include + +bool OpenGLTextureBackend::InitializeGLAD() +{ + gladLoaderLoadEGL(EGL_NO_DISPLAY); + int egl_version = gladLoadEGL(eglGetCurrentDisplay(), eglGetProcAddress); + if (egl_version == 0) + { + Logger::Log("OpenGLTextureBackend: Failed to load EGL"); + return false; + } + + int gl_version = gladLoaderLoadGLES2(); + if (gl_version == 0) + { + Logger::Log("OpenGLTextureBackend: Failed to load GLES2"); + return false; + } + + Logger::Log("OpenGLTextureBackend: GLAD initialized successfully"); + return true; +} + +bool OpenGLTextureBackend::Create(AHardwareBuffer* buffer) +{ + if (m_Created) + { + Destroy(); + } + + // Get buffer dimensions + AHardwareBuffer_Desc desc{}; + AHardwareBuffer_describe(buffer, &desc); + m_Width = static_cast(desc.width); + m_Height = static_cast(desc.height); + + // Create EGL image from hardware buffer + EGLClientBuffer clientBuffer = eglGetNativeClientBufferANDROID(buffer); + EGLImageKHR eglImage = eglCreateImageKHR( + eglGetCurrentDisplay(), + EGL_NO_CONTEXT, + EGL_NATIVE_BUFFER_ANDROID, + clientBuffer, + nullptr + ); + + if (eglImage == EGL_NO_IMAGE_KHR) + { + Logger::Log("OpenGLTextureBackend: Failed to create EGL image"); + return false; + } + + // Create source texture (external OES format from hardware buffer) + glGenTextures(1, &m_SourceTexture); + glBindTexture(GL_TEXTURE_EXTERNAL_OES, m_SourceTexture); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, eglImage); + eglDestroyImageKHR(eglGetCurrentDisplay(), eglImage); + + // Create source framebuffer + glGenFramebuffers(1, &m_SourceFramebuffer); + glBindFramebuffer(GL_FRAMEBUFFER, m_SourceFramebuffer); + glFramebufferTexture2D( + GL_FRAMEBUFFER, + GL_COLOR_ATTACHMENT0, + GL_TEXTURE_EXTERNAL_OES, + m_SourceTexture, + 0 + ); + + if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) + { + Logger::Log("OpenGLTextureBackend: Source framebuffer incomplete"); + Destroy(); + return false; + } + + // Create destination texture (standard 2D texture for Unity) + glGenTextures(1, &m_DestTexture); + glBindTexture(GL_TEXTURE_2D, m_DestTexture); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glTexImage2D( + GL_TEXTURE_2D, + 0, + GL_RGB, + m_Width, + m_Height, + 0, + GL_RGB, + GL_UNSIGNED_BYTE, + nullptr + ); + + // Create destination framebuffer + glGenFramebuffers(1, &m_DestFramebuffer); + glBindFramebuffer(GL_FRAMEBUFFER, m_DestFramebuffer); + glFramebufferTexture2D( + GL_FRAMEBUFFER, + GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, + m_DestTexture, + 0 + ); + + if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) + { + Logger::Log("OpenGLTextureBackend: Dest framebuffer incomplete"); + Destroy(); + return false; + } + + m_Created = true; + + // Initial blit + Blit(); + + Logger::Log("OpenGLTextureBackend: Created successfully"); + return true; +} + +void OpenGLTextureBackend::Update() +{ + if (m_Created) + { + Blit(); + } +} + +void OpenGLTextureBackend::Blit() +{ + glBindFramebuffer(GL_READ_FRAMEBUFFER, m_SourceFramebuffer); + glBindFramebuffer(GL_DRAW_FRAMEBUFFER, m_DestFramebuffer); + glBlitFramebuffer( + 0, 0, m_Width, m_Height, + 0, 0, m_Width, m_Height, + GL_COLOR_BUFFER_BIT, + GL_NEAREST + ); +} + +void OpenGLTextureBackend::Destroy() +{ + if (m_DestFramebuffer) + { + glDeleteFramebuffers(1, &m_DestFramebuffer); + m_DestFramebuffer = 0; + } + if (m_SourceFramebuffer) + { + glDeleteFramebuffers(1, &m_SourceFramebuffer); + m_SourceFramebuffer = 0; + } + if (m_DestTexture) + { + glDeleteTextures(1, &m_DestTexture); + m_DestTexture = 0; + } + if (m_SourceTexture) + { + glDeleteTextures(1, &m_SourceTexture); + m_SourceTexture = 0; + } + m_Width = 0; + m_Height = 0; + m_Created = false; +} + +void* OpenGLTextureBackend::GetNativeTexturePtr() +{ + return reinterpret_cast(static_cast(m_DestTexture)); +} diff --git a/Packages/com.omarator.mosissdk/Plugins/Android/cpp/opengl_backend.h b/Packages/com.omarator.mosissdk/Plugins/Android/cpp/opengl_backend.h new file mode 100644 index 0000000..49458cd --- /dev/null +++ b/Packages/com.omarator.mosissdk/Plugins/Android/cpp/opengl_backend.h @@ -0,0 +1,42 @@ +#pragma once + +#include "texture_backend.h" +#include +#include + +/** + * OpenGL ES implementation of ITextureBackend. + * Imports AHardwareBuffer via EGL image and blits to a standard GL_TEXTURE_2D. + */ +class OpenGLTextureBackend : public ITextureBackend +{ +public: + OpenGLTextureBackend() = default; + ~OpenGLTextureBackend() override { Destroy(); } + + bool Create(AHardwareBuffer* buffer) override; + void Update() override; + void Destroy() override; + void* GetNativeTexturePtr() override; + int GetWidth() const override { return m_Width; } + int GetHeight() const override { return m_Height; } + bool IsVulkan() const override { return false; } + + /** + * Initialize GLAD for OpenGL ES. + * Must be called on the render thread before Create(). + * @return true if initialization succeeded + */ + static bool InitializeGLAD(); + +private: + void Blit(); + + GLuint m_SourceTexture = 0; // GL_TEXTURE_EXTERNAL_OES from HardwareBuffer + GLuint m_SourceFramebuffer = 0; + GLuint m_DestTexture = 0; // GL_TEXTURE_2D for Unity + GLuint m_DestFramebuffer = 0; + GLint m_Width = 0; + GLint m_Height = 0; + bool m_Created = false; +}; diff --git a/Packages/com.omarator.mosissdk/Plugins/Android/cpp/texture_backend.h b/Packages/com.omarator.mosissdk/Plugins/Android/cpp/texture_backend.h new file mode 100644 index 0000000..52452e8 --- /dev/null +++ b/Packages/com.omarator.mosissdk/Plugins/Android/cpp/texture_backend.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include + +/** + * Abstract interface for texture backends. + * Allows swapping between OpenGL and Vulkan implementations at runtime. + */ +class ITextureBackend +{ +public: + virtual ~ITextureBackend() = default; + + /** + * Create texture resources from a hardware buffer. + * @param buffer The AHardwareBuffer from the Mosis service + * @return true if creation succeeded + */ + virtual bool Create(AHardwareBuffer* buffer) = 0; + + /** + * Update the texture (blit from source to destination). + * Called each frame when a new frame is available. + */ + virtual void Update() = 0; + + /** + * Destroy all texture resources. + */ + virtual void Destroy() = 0; + + /** + * Get the native texture pointer for Unity. + * For OpenGL: GLuint cast to void* + * For Vulkan: VkImage cast to void* + */ + virtual void* GetNativeTexturePtr() = 0; + + /** + * Get texture dimensions. + */ + virtual int GetWidth() const = 0; + virtual int GetHeight() const = 0; + + /** + * Check if the backend is Vulkan-based. + * Unity needs this to know how to interpret the native pointer. + */ + virtual bool IsVulkan() const = 0; +}; diff --git a/Packages/com.omarator.mosissdk/Plugins/Android/cpp/vulkan_backend.cpp b/Packages/com.omarator.mosissdk/Plugins/Android/cpp/vulkan_backend.cpp new file mode 100644 index 0000000..87a8c23 --- /dev/null +++ b/Packages/com.omarator.mosissdk/Plugins/Android/cpp/vulkan_backend.cpp @@ -0,0 +1,465 @@ +#include "vulkan_backend.h" +#include +#include + +bool VulkanTextureBackend::Initialize(IUnityInterfaces* unityInterfaces) +{ + if (m_Initialized) + { + return true; + } + + m_UnityVulkan = unityInterfaces->Get(); + if (!m_UnityVulkan) + { + Logger::Log("VulkanTextureBackend: IUnityGraphicsVulkan not available"); + return false; + } + + // Get Unity's Vulkan instance + UnityVulkanInstance vkInstance = m_UnityVulkan->Instance(); + m_Instance = vkInstance.instance; + m_PhysicalDevice = vkInstance.physicalDevice; + m_Device = vkInstance.device; + m_QueueFamilyIndex = vkInstance.queueFamilyIndex; + + // Get a queue from Unity + vkGetDeviceQueue(m_Device, m_QueueFamilyIndex, 0, &m_Queue); + + // Load extension function + vkGetAndroidHardwareBufferPropertiesANDROID = + reinterpret_cast( + vkGetInstanceProcAddr(m_Instance, "vkGetAndroidHardwareBufferPropertiesANDROID") + ); + + if (!vkGetAndroidHardwareBufferPropertiesANDROID) + { + Logger::Log("VulkanTextureBackend: vkGetAndroidHardwareBufferPropertiesANDROID not available"); + return false; + } + + // Create command pool + VkCommandPoolCreateInfo poolInfo{}; + poolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + poolInfo.queueFamilyIndex = m_QueueFamilyIndex; + poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + + if (vkCreateCommandPool(m_Device, &poolInfo, nullptr, &m_CommandPool) != VK_SUCCESS) + { + Logger::Log("VulkanTextureBackend: Failed to create command pool"); + return false; + } + + // Allocate command buffer + VkCommandBufferAllocateInfo allocInfo{}; + allocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + allocInfo.commandPool = m_CommandPool; + allocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + allocInfo.commandBufferCount = 1; + + if (vkAllocateCommandBuffers(m_Device, &allocInfo, &m_CommandBuffer) != VK_SUCCESS) + { + Logger::Log("VulkanTextureBackend: Failed to allocate command buffer"); + vkDestroyCommandPool(m_Device, m_CommandPool, nullptr); + m_CommandPool = VK_NULL_HANDLE; + return false; + } + + m_Initialized = true; + Logger::Log("VulkanTextureBackend: Initialized successfully"); + return true; +} + +bool VulkanTextureBackend::Create(AHardwareBuffer* buffer) +{ + if (!m_Initialized) + { + Logger::Log("VulkanTextureBackend: Not initialized"); + return false; + } + + if (m_Created) + { + Destroy(); + } + + // Get buffer dimensions + AHardwareBuffer_Desc desc{}; + AHardwareBuffer_describe(buffer, &desc); + m_Width = static_cast(desc.width); + m_Height = static_cast(desc.height); + + Logger::Log(std::format("VulkanTextureBackend: Creating {}x{} texture", m_Width, m_Height)); + + if (!ImportHardwareBuffer(buffer)) + { + Logger::Log("VulkanTextureBackend: Failed to import hardware buffer"); + return false; + } + + if (!CreateLocalImage()) + { + Logger::Log("VulkanTextureBackend: Failed to create local image"); + DestroyImportedResources(); + return false; + } + + m_Created = true; + + // Initial copy + CopyImage(); + + Logger::Log("VulkanTextureBackend: Created successfully"); + return true; +} + +bool VulkanTextureBackend::ImportHardwareBuffer(AHardwareBuffer* buffer) +{ + // Query hardware buffer properties + VkAndroidHardwareBufferFormatPropertiesANDROID formatProps{}; + formatProps.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_FORMAT_PROPERTIES_ANDROID; + + VkAndroidHardwareBufferPropertiesANDROID props{}; + props.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID; + props.pNext = &formatProps; + + if (vkGetAndroidHardwareBufferPropertiesANDROID(m_Device, buffer, &props) != VK_SUCCESS) + { + Logger::Log("VulkanTextureBackend: Failed to get hardware buffer properties"); + return false; + } + + m_Format = formatProps.format; + Logger::Log(std::format("VulkanTextureBackend: Buffer format: {}", static_cast(m_Format))); + + // Create VkImage with external memory + VkExternalMemoryImageCreateInfo extMemImageInfo{}; + extMemImageInfo.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO; + extMemImageInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID; + + VkImageCreateInfo imageInfo{}; + imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + imageInfo.pNext = &extMemImageInfo; + imageInfo.imageType = VK_IMAGE_TYPE_2D; + imageInfo.format = m_Format; + imageInfo.extent = {static_cast(m_Width), static_cast(m_Height), 1}; + imageInfo.mipLevels = 1; + imageInfo.arrayLayers = 1; + imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imageInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + + if (vkCreateImage(m_Device, &imageInfo, nullptr, &m_ImportedImage) != VK_SUCCESS) + { + Logger::Log("VulkanTextureBackend: Failed to create imported image"); + return false; + } + + // Import memory from hardware buffer + VkImportAndroidHardwareBufferInfoANDROID importInfo{}; + importInfo.sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID; + importInfo.buffer = buffer; + + VkMemoryDedicatedAllocateInfo dedicatedInfo{}; + dedicatedInfo.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO; + dedicatedInfo.pNext = &importInfo; + dedicatedInfo.image = m_ImportedImage; + + VkMemoryAllocateInfo allocInfo{}; + allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocInfo.pNext = &dedicatedInfo; + allocInfo.allocationSize = props.allocationSize; + allocInfo.memoryTypeIndex = FindMemoryType(props.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + + if (vkAllocateMemory(m_Device, &allocInfo, nullptr, &m_ImportedMemory) != VK_SUCCESS) + { + Logger::Log("VulkanTextureBackend: Failed to allocate imported memory"); + vkDestroyImage(m_Device, m_ImportedImage, nullptr); + m_ImportedImage = VK_NULL_HANDLE; + return false; + } + + if (vkBindImageMemory(m_Device, m_ImportedImage, m_ImportedMemory, 0) != VK_SUCCESS) + { + Logger::Log("VulkanTextureBackend: Failed to bind imported memory"); + DestroyImportedResources(); + return false; + } + + Logger::Log("VulkanTextureBackend: Hardware buffer imported"); + return true; +} + +bool VulkanTextureBackend::CreateLocalImage() +{ + // Create local VkImage for safe rendering + VkImageCreateInfo imageInfo{}; + imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + imageInfo.imageType = VK_IMAGE_TYPE_2D; + imageInfo.format = m_Format; + imageInfo.extent = {static_cast(m_Width), static_cast(m_Height), 1}; + imageInfo.mipLevels = 1; + imageInfo.arrayLayers = 1; + imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imageInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; + imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + + if (vkCreateImage(m_Device, &imageInfo, nullptr, &m_LocalImage) != VK_SUCCESS) + { + Logger::Log("VulkanTextureBackend: Failed to create local image"); + return false; + } + + // Allocate memory for local image + VkMemoryRequirements memReqs; + vkGetImageMemoryRequirements(m_Device, m_LocalImage, &memReqs); + + VkMemoryAllocateInfo allocInfo{}; + allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocInfo.allocationSize = memReqs.size; + allocInfo.memoryTypeIndex = FindMemoryType(memReqs.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + + if (vkAllocateMemory(m_Device, &allocInfo, nullptr, &m_LocalMemory) != VK_SUCCESS) + { + Logger::Log("VulkanTextureBackend: Failed to allocate local memory"); + vkDestroyImage(m_Device, m_LocalImage, nullptr); + m_LocalImage = VK_NULL_HANDLE; + return false; + } + + if (vkBindImageMemory(m_Device, m_LocalImage, m_LocalMemory, 0) != VK_SUCCESS) + { + Logger::Log("VulkanTextureBackend: Failed to bind local memory"); + DestroyLocalResources(); + return false; + } + + // Create image view + VkImageViewCreateInfo viewInfo{}; + viewInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + viewInfo.image = m_LocalImage; + viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + viewInfo.format = m_Format; + viewInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + viewInfo.subresourceRange.baseMipLevel = 0; + viewInfo.subresourceRange.levelCount = 1; + viewInfo.subresourceRange.baseArrayLayer = 0; + viewInfo.subresourceRange.layerCount = 1; + + if (vkCreateImageView(m_Device, &viewInfo, nullptr, &m_LocalImageView) != VK_SUCCESS) + { + Logger::Log("VulkanTextureBackend: Failed to create image view"); + DestroyLocalResources(); + return false; + } + + Logger::Log("VulkanTextureBackend: Local image created"); + return true; +} + +void VulkanTextureBackend::Update() +{ + if (m_Created) + { + CopyImage(); + } +} + +void VulkanTextureBackend::CopyImage() +{ + // Begin command buffer + VkCommandBufferBeginInfo beginInfo{}; + beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + + vkResetCommandBuffer(m_CommandBuffer, 0); + vkBeginCommandBuffer(m_CommandBuffer, &beginInfo); + + // Transition imported image to transfer src + TransitionImageLayout(m_ImportedImage, + VK_IMAGE_LAYOUT_UNDEFINED, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + + // Transition local image to transfer dst + TransitionImageLayout(m_LocalImage, + VK_IMAGE_LAYOUT_UNDEFINED, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + + // Copy image + VkImageCopy copyRegion{}; + copyRegion.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + copyRegion.srcSubresource.layerCount = 1; + copyRegion.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + copyRegion.dstSubresource.layerCount = 1; + copyRegion.extent = {static_cast(m_Width), static_cast(m_Height), 1}; + + vkCmdCopyImage( + m_CommandBuffer, + m_ImportedImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + m_LocalImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + 1, ©Region + ); + + // Transition local image to shader read + TransitionImageLayout(m_LocalImage, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL); + + vkEndCommandBuffer(m_CommandBuffer); + + // Submit and wait (CPU sync for simplicity) + VkSubmitInfo submitInfo{}; + submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &m_CommandBuffer; + + vkQueueSubmit(m_Queue, 1, &submitInfo, VK_NULL_HANDLE); + vkQueueWaitIdle(m_Queue); +} + +void VulkanTextureBackend::TransitionImageLayout(VkImage image, VkImageLayout oldLayout, VkImageLayout newLayout) +{ + VkImageMemoryBarrier barrier{}; + barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + barrier.oldLayout = oldLayout; + barrier.newLayout = newLayout; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image; + barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + barrier.subresourceRange.baseMipLevel = 0; + barrier.subresourceRange.levelCount = 1; + barrier.subresourceRange.baseArrayLayer = 0; + barrier.subresourceRange.layerCount = 1; + + VkPipelineStageFlags srcStage; + VkPipelineStageFlags dstStage; + + if (oldLayout == VK_IMAGE_LAYOUT_UNDEFINED) + { + barrier.srcAccessMask = 0; + srcStage = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; + } + else if (oldLayout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) + { + barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + srcStage = VK_PIPELINE_STAGE_TRANSFER_BIT; + } + else + { + barrier.srcAccessMask = 0; + srcStage = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; + } + + if (newLayout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) + { + barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + dstStage = VK_PIPELINE_STAGE_TRANSFER_BIT; + } + else if (newLayout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) + { + barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + dstStage = VK_PIPELINE_STAGE_TRANSFER_BIT; + } + else if (newLayout == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL) + { + barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + dstStage = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; + } + else + { + barrier.dstAccessMask = 0; + dstStage = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT; + } + + vkCmdPipelineBarrier( + m_CommandBuffer, + srcStage, dstStage, + 0, + 0, nullptr, + 0, nullptr, + 1, &barrier + ); +} + +uint32_t VulkanTextureBackend::FindMemoryType(uint32_t typeFilter, VkMemoryPropertyFlags properties) +{ + VkPhysicalDeviceMemoryProperties memProps; + vkGetPhysicalDeviceMemoryProperties(m_PhysicalDevice, &memProps); + + for (uint32_t i = 0; i < memProps.memoryTypeCount; i++) + { + if ((typeFilter & (1 << i)) && + (memProps.memoryTypes[i].propertyFlags & properties) == properties) + { + return i; + } + } + + Logger::Log("VulkanTextureBackend: Failed to find suitable memory type"); + return 0; +} + +void VulkanTextureBackend::Destroy() +{ + if (m_Device && m_Created) + { + vkDeviceWaitIdle(m_Device); + } + + DestroyLocalResources(); + DestroyImportedResources(); + + m_Width = 0; + m_Height = 0; + m_Created = false; +} + +void VulkanTextureBackend::DestroyImportedResources() +{ + if (m_Device) + { + if (m_ImportedMemory != VK_NULL_HANDLE) + { + vkFreeMemory(m_Device, m_ImportedMemory, nullptr); + m_ImportedMemory = VK_NULL_HANDLE; + } + if (m_ImportedImage != VK_NULL_HANDLE) + { + vkDestroyImage(m_Device, m_ImportedImage, nullptr); + m_ImportedImage = VK_NULL_HANDLE; + } + } +} + +void VulkanTextureBackend::DestroyLocalResources() +{ + if (m_Device) + { + if (m_LocalImageView != VK_NULL_HANDLE) + { + vkDestroyImageView(m_Device, m_LocalImageView, nullptr); + m_LocalImageView = VK_NULL_HANDLE; + } + if (m_LocalMemory != VK_NULL_HANDLE) + { + vkFreeMemory(m_Device, m_LocalMemory, nullptr); + m_LocalMemory = VK_NULL_HANDLE; + } + if (m_LocalImage != VK_NULL_HANDLE) + { + vkDestroyImage(m_Device, m_LocalImage, nullptr); + m_LocalImage = VK_NULL_HANDLE; + } + } +} + +void* VulkanTextureBackend::GetNativeTexturePtr() +{ + // For Vulkan, Unity expects the VkImage + return reinterpret_cast(m_LocalImage); +} diff --git a/Packages/com.omarator.mosissdk/Plugins/Android/cpp/vulkan_backend.h b/Packages/com.omarator.mosissdk/Plugins/Android/cpp/vulkan_backend.h new file mode 100644 index 0000000..be95b5b --- /dev/null +++ b/Packages/com.omarator.mosissdk/Plugins/Android/cpp/vulkan_backend.h @@ -0,0 +1,82 @@ +#pragma once + +#include "texture_backend.h" +#include +#include +#include + +/** + * Vulkan implementation of ITextureBackend. + * Imports AHardwareBuffer via VK_ANDROID_external_memory_android_hardware_buffer + * and copies to a local VkImage for safe rendering. + */ +class VulkanTextureBackend : public ITextureBackend +{ +public: + VulkanTextureBackend() = default; + ~VulkanTextureBackend() override { Destroy(); } + + /** + * Initialize the Vulkan backend with Unity's Vulkan interface. + * Must be called during UnityPluginLoad. + * @param unityInterfaces Unity's interface registry + * @return true if Vulkan is available and initialization succeeded + */ + bool Initialize(IUnityInterfaces* unityInterfaces); + + bool Create(AHardwareBuffer* buffer) override; + void Update() override; + void Destroy() override; + void* GetNativeTexturePtr() override; + int GetWidth() const override { return m_Width; } + int GetHeight() const override { return m_Height; } + bool IsVulkan() const override { return true; } + + /** + * Get the VkImageView for Unity texture binding. + */ + VkImageView GetLocalImageView() const { return m_LocalImageView; } + +private: + bool ImportHardwareBuffer(AHardwareBuffer* buffer); + bool CreateLocalImage(); + void DestroyImportedResources(); + void DestroyLocalResources(); + uint32_t FindMemoryType(uint32_t typeFilter, VkMemoryPropertyFlags properties); + void TransitionImageLayout(VkImage image, VkImageLayout oldLayout, VkImageLayout newLayout); + void CopyImage(); + + // Unity Vulkan interface + IUnityGraphicsVulkan* m_UnityVulkan = nullptr; + + // Vulkan device objects (from Unity) + VkInstance m_Instance = VK_NULL_HANDLE; + VkPhysicalDevice m_PhysicalDevice = VK_NULL_HANDLE; + VkDevice m_Device = VK_NULL_HANDLE; + VkQueue m_Queue = VK_NULL_HANDLE; + uint32_t m_QueueFamilyIndex = 0; + + // Command pool for copy operations + VkCommandPool m_CommandPool = VK_NULL_HANDLE; + VkCommandBuffer m_CommandBuffer = VK_NULL_HANDLE; + + // Imported hardware buffer resources + VkImage m_ImportedImage = VK_NULL_HANDLE; + VkDeviceMemory m_ImportedMemory = VK_NULL_HANDLE; + + // Local copy resources (for safe rendering) + VkImage m_LocalImage = VK_NULL_HANDLE; + VkDeviceMemory m_LocalMemory = VK_NULL_HANDLE; + VkImageView m_LocalImageView = VK_NULL_HANDLE; + + // Texture format info + VkFormat m_Format = VK_FORMAT_R8G8B8A8_UNORM; + int m_Width = 0; + int m_Height = 0; + + bool m_Initialized = false; + bool m_Created = false; + + // Extension function pointers + PFN_vkGetAndroidHardwareBufferPropertiesANDROID vkGetAndroidHardwareBufferPropertiesANDROID = nullptr; +}; diff --git a/Packages/com.omarator.mosissdk/README.md b/Packages/com.omarator.mosissdk/README.md index 19d1082..ecc482c 100644 --- a/Packages/com.omarator.mosissdk/README.md +++ b/Packages/com.omarator.mosissdk/README.md @@ -1 +1,221 @@ -Use this file to describe your package's features. \ No newline at end of file +# MosisSDK for Unity + +Unity package providing integration with MosisService for virtual smartphone display in VR applications. + +## Overview + +This package connects to the MosisService Android app via AIDL Binder IPC to: +- Receive rendered phone screen frames via AHardwareBuffer +- Forward VR controller touch input to the virtual phone +- Support both OpenGL ES and Vulkan graphics backends + +## Requirements + +- Unity 6000.3.2f1 or later +- Android target platform +- MosisService app installed on device + +## Installation + +1. Open Window > Package Manager +2. Click + > Add package from disk +3. Navigate to `Packages/com.omarator.mosissdk/package.json` + +## Project Structure + +``` +Packages/com.omarator.mosissdk/ +├── package.json +├── README.md +├── Runtime/ +│ ├── KotlinBridge.cs # C# to native bridge +│ └── com.omarator.mosissdk.asmdef +├── Plugins/ +│ └── Android/ +│ ├── MosisSDK.aar # Kotlin service connection +│ ├── libs/ +│ │ └── arm64-v8a/ +│ │ └── libmy_native_lib.so +│ └── cpp/ # Native source +│ ├── CMakeLists.txt +│ ├── my_native_code.cpp +│ ├── texture_backend.h +│ ├── opengl_backend.h/cpp +│ └── vulkan_backend.h/cpp +└── Assets/ + └── PhoneInteraction.cs # VR controller input +``` + +## Build Commands + +### Unity Build + +1. File > Build Settings +2. Switch Platform to Android +3. Player Settings > Other Settings: + - Graphics APIs: Vulkan, OpenGLES3 (in order of preference) + - Minimum API Level: 29 +4. Build or Build and Run + +### Native Plugin Build + +To rebuild the native library manually: + +```batch +cd Packages/com.omarator.mosissdk/Plugins/Android/cpp + +:: Configure with Android NDK +cmake -B build ^ + -DCMAKE_TOOLCHAIN_FILE=%ANDROID_NDK_HOME%/build/cmake/android.toolchain.cmake ^ + -DANDROID_ABI=arm64-v8a ^ + -DANDROID_PLATFORM=android-29 + +:: Build +cmake --build build + +:: Copy output +copy build\libmy_native_lib.so ..\libs\arm64-v8a\ +``` + +## Usage + +### Basic Setup + +1. Add `KotlinBridge` component to a GameObject in your scene +2. Add `PhoneInteraction` component for VR input handling +3. Assign VR controller and phone plane references + +### KotlinBridge + +The `KotlinBridge` component manages the connection to MosisService: + +```csharp +public class KotlinBridge : MonoBehaviour +{ + public Material phoneMaterial; // Material to apply phone texture + + // Called automatically when texture is ready + // Texture is applied to phoneMaterial's _MainTex +} +``` + +### PhoneInteraction + +Handles VR controller raycasting and touch input: + +```csharp +public class PhoneInteraction : MonoBehaviour +{ + public GameObject Controller; // VR controller + public GameObject Plane; // Phone display plane + public XRNode inputDevice; // Controller hand + public float triggerThreshold; // Trigger press threshold +} +``` + +### Touch API + +Send touch events from custom scripts: + +```csharp +// Touch down at normalized UV (0-1) +KotlinBridge.SendTouchDown(0.5f, 0.5f); + +// Touch move +KotlinBridge.SendTouchMove(0.6f, 0.5f); + +// Touch up +KotlinBridge.SendTouchUp(0.6f, 0.5f); +``` + +## Graphics Backend + +The native plugin automatically selects the appropriate backend: + +| Graphics API | Backend | Status | +|--------------|---------|--------| +| Vulkan | VulkanTextureBackend | Preferred | +| OpenGL ES 3.0 | OpenGLTextureBackend | Fallback | +| OpenGL ES 2.0 | OpenGLTextureBackend | Fallback | + +Backend selection happens in `UnityPluginLoad()` based on the active renderer. + +### Vulkan Import + +When using Vulkan, the plugin imports AHardwareBuffer directly as a VkImage: + +1. Query buffer properties via `vkGetAndroidHardwareBufferPropertiesANDROID` +2. Create VkImage with external memory handle type +3. Import memory with `VkImportAndroidHardwareBufferInfoANDROID` +4. Copy to local image each frame for safe rendering + +### OpenGL Import + +When using OpenGL ES, the plugin uses EGL image extension: + +1. Create EGLImage from AHardwareBuffer via `eglCreateImageKHR` +2. Bind to OpenGL texture via `glEGLImageTargetTexture2DOES` +3. Blit to local texture each frame + +## Device Testing + +```bash +# Install MosisService first +adb install -r MosisService-debug.apk + +# Install Unity app +adb install -r MosisVR.apk + +# Launch service +adb shell am start -n com.omixlab.mosis/.MainActivity + +# Launch Unity app +adb shell am start -n com.omixlab.mosisvr/com.unity3d.player.UnityPlayerActivity + +# Monitor logs +adb logcat -s Unity MosisSDK Vulkan +``` + +## Troubleshooting + +### Black Screen / No Texture + +- Verify MosisService is running (`adb shell ps | grep mosis`) +- Check logcat for connection errors +- Ensure both apps have same user ID or are debuggable + +### Vulkan Errors + +- Device must support `VK_ANDROID_external_memory_android_hardware_buffer` +- Check `adb logcat -s Vulkan` for extension availability +- Falls back to OpenGL if Vulkan init fails + +### Touch Not Working + +- Verify `PhoneInteraction` references are set +- Check XR input device selection matches controller +- Test with mouse click fallback in editor + +## Implementation Notes + +### Files Modified in Vulkan Implementation + +| File | Changes | +|------|---------| +| `my_native_code.cpp` | Backend abstraction, `UnityPluginLoad` detection | +| `CMakeLists.txt` | Added Vulkan library, new source files | +| `KotlinBridge.cs` | Added `SendTouchDown`/`SendTouchUp`, dynamic resolution | +| `PhoneInteraction.cs` | Added VR trigger-based touch handling | + +### New Files Added + +| File | Purpose | +|------|---------| +| `texture_backend.h` | ITextureBackend abstract interface | +| `opengl_backend.h/cpp` | OpenGL ES implementation | +| `vulkan_backend.h/cpp` | Vulkan implementation | + +## Version History + +- **1.0.0** - Initial release with OpenGL support +- **1.1.0** - Added Vulkan backend, touch down/up events, dynamic resolution diff --git a/Packages/com.omarator.mosissdk/Runtime/KotlinBridge.cs b/Packages/com.omarator.mosissdk/Runtime/KotlinBridge.cs index 6ab8f2b..8eb3d03 100644 --- a/Packages/com.omarator.mosissdk/Runtime/KotlinBridge.cs +++ b/Packages/com.omarator.mosissdk/Runtime/KotlinBridge.cs @@ -6,6 +6,7 @@ using System.Runtime.InteropServices; public class KotlinBridge : MonoBehaviour { static KotlinBridge Instance; + [StructLayout(LayoutKind.Sequential)] struct NativeCallbacks { @@ -15,24 +16,40 @@ public class KotlinBridge : MonoBehaviour public IntPtr OnBufferReady; public IntPtr OnTextureReady; } + delegate void OnMessageDelegate(string message); delegate void OnServiceInitializedDelegate(bool success); delegate void OnFrameAvailableDelegate(); delegate void OnBufferReadyDelegate(); - delegate void OnTextureReadyDelegate(uint gl_texture); + delegate void OnTextureReadyDelegate(IntPtr nativeTexturePtr, int width, int height, bool isVulkan); + [DllImport("my_native_lib")] static extern void SetNativeCallbacks(ref NativeCallbacks callbacks); + + [DllImport("my_native_lib")] + public static extern void SendTouchDown(float u, float v); + [DllImport("my_native_lib")] public static extern void SendTouchMove(float u, float v); + [DllImport("my_native_lib")] - static extern IntPtr InitGLAD(); + public static extern void SendTouchUp(float u, float v); + + [DllImport("my_native_lib")] + static extern IntPtr InitBackend(); + [DllImport("my_native_lib")] static extern IntPtr UpdateTexture(); + // Legacy compatibility + [DllImport("my_native_lib")] + static extern IntPtr InitGLAD(); + [AOT.MonoPInvokeCallback(typeof(OnMessageDelegate))] static void OnMessage(string message) { - //Debug.Log("High-speed callback: " + message); + // High-speed callback, disabled for performance + // Debug.Log("Native callback: " + message); } [AOT.MonoPInvokeCallback(typeof(OnServiceInitializedDelegate))] @@ -41,7 +58,7 @@ public class KotlinBridge : MonoBehaviour Debug.Log("Service Initialized: " + success); } - [AOT.MonoPInvokeCallback(typeof(OnServiceInitializedDelegate))] + [AOT.MonoPInvokeCallback(typeof(OnFrameAvailableDelegate))] static void OnFrameAvailable() { UnityMainThreadDispatcher.Enqueue(() => { @@ -54,24 +71,59 @@ public class KotlinBridge : MonoBehaviour { Debug.Log("Buffer Ready"); UnityMainThreadDispatcher.Enqueue(() => { - GL.IssuePluginEvent(InitGLAD(), 1); + GL.IssuePluginEvent(InitBackend(), 1); }); } - + [AOT.MonoPInvokeCallback(typeof(OnTextureReadyDelegate))] - static void OnTextureReady(uint gl_texture) + static void OnTextureReady(IntPtr nativeTexturePtr, int width, int height, bool isVulkan) { - Debug.Log("Texture Ready: " + gl_texture); + Debug.Log($"Texture Ready: ptr={nativeTexturePtr}, size={width}x{height}, vulkan={isVulkan}"); + UnityMainThreadDispatcher.Enqueue(() => { var renderer = Instance.GetComponent(); - renderer.materials[2].mainTexture = Texture2D.CreateExternalTexture(1024, 1024, - TextureFormat.RGBA32, false, false, (IntPtr)gl_texture); + if (renderer == null) + { + Debug.LogError("KotlinBridge: No Renderer component found"); + return; + } + + // Create external texture with actual dimensions from the service + // Unity handles both OpenGL and Vulkan native textures via CreateExternalTexture + Texture2D texture = Texture2D.CreateExternalTexture( + width, + height, + TextureFormat.RGBA32, + false, // mipChain + false, // linear + nativeTexturePtr + ); + + if (texture == null) + { + Debug.LogError("KotlinBridge: Failed to create external texture"); + return; + } + + // Set texture on material (index 2 is the phone screen material) + if (renderer.materials.Length > 2) + { + renderer.materials[2].mainTexture = texture; + Debug.Log($"KotlinBridge: Texture set on material[2], {width}x{height}"); + } + else + { + Debug.LogWarning($"KotlinBridge: Not enough materials (have {renderer.materials.Length}, need 3)"); + // Fall back to first material if not enough + renderer.material.mainTexture = texture; + } }); } void Start() { Instance = this; + NativeCallbacks callbacks = new NativeCallbacks(); callbacks.OnMessage = Marshal.GetFunctionPointerForDelegate(new OnMessageDelegate(OnMessage)); callbacks.OnServiceInitialized = Marshal.GetFunctionPointerForDelegate(new OnServiceInitializedDelegate(OnServiceInitialized));