This commit is contained in:
2026-01-17 08:46:11 +01:00
parent 0aea07026d
commit e7a514a713
10 changed files with 1371 additions and 126 deletions

View File

@@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.22.1)
project("MyNativePlugin")
find_library(log-lib log)
find_library(vulkan-lib vulkan)
set(CMAKE_CXX_STANDARD 23)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -35,6 +36,8 @@ add_custom_command(
add_library(my_native_lib SHARED
my_native_code.cpp
opengl_backend.cpp
vulkan_backend.cpp
${CMAKE_CURRENT_BINARY_DIR}/com/omixlab/mosis/IMosisService.cpp
${CMAKE_CURRENT_BINARY_DIR}/com/omixlab/mosis/IMosisListener.cpp
${SHARED_SRC_DIR}/logger.cpp
@@ -43,11 +46,27 @@ add_library(my_native_lib SHARED
${SHARED_SRC_DIR}/glad/src/egl.c
${SHARED_SRC_DIR}/glad/src/gles2.c
)
target_link_libraries(my_native_lib ${log-lib} binder_ndk EGL GLESv2 nativewindow)
target_link_libraries(my_native_lib
${log-lib}
${vulkan-lib}
binder_ndk
EGL
GLESv2
nativewindow
)
target_include_directories(my_native_lib PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}
${SHARED_SRC_DIR}
${SHARED_SRC_DIR}/glad/include
${CMAKE_CURRENT_BINARY_DIR}
${BINDER_DIR}
${PLUGIN_API}
)
# Vulkan support definitions
target_compile_definitions(my_native_lib PRIVATE
VK_USE_PLATFORM_ANDROID_KHR
MOSIS_VULKAN_SUPPORT=1
)

View File

@@ -5,90 +5,29 @@
#include <aidl/com/omixlab/mosis/IMosisService.h>
#include <android/binder_ibinder_jni.h>
#include <format>
#include <glad/gles2.h>
#include <glad/egl.h>
#include <IUnityGraphics.h>
#include <external_texture.h>
#include <render_target.h>
#include "texture_backend.h"
#include "opengl_backend.h"
#include "vulkan_backend.h"
using namespace aidl::com::omixlab::mosis;
using namespace aidl::android::hardware;
// Global state
std::shared_ptr<IMosisService> g_service;
std::shared_ptr<class ServiceContext> g_context;
std::unique_ptr<ITextureBackend> g_backend;
IUnityInterfaces* g_unityInterfaces = nullptr;
UnityGfxRenderer g_rendererType = kUnityGfxRendererNull;
class TextureBlitter
{
GLuint source_texture = 0;
GLuint source_fb = 0;
GLuint dest_texture = 0;
GLuint dest_fb = 0;
GLint width;
GLint height;
public:
bool create(AHardwareBuffer* hardwareBuffer)
{
AHardwareBuffer_Desc desc{};
AHardwareBuffer_describe(hardwareBuffer, &desc);
width = desc.width;
height = desc.height;
EGLClientBuffer clientBuffer = eglGetNativeClientBufferANDROID(hardwareBuffer);
EGLImageKHR eglImage = eglCreateImageKHR(eglGetCurrentDisplay(), EGL_NO_CONTEXT,
EGL_NATIVE_BUFFER_ANDROID, clientBuffer, nullptr);
if (eglImage == EGL_NO_IMAGE_KHR)
{
Logger::Log("Failed to create EGL image");
return false;
}
glGenTextures(1, &source_texture);
glBindTexture(GL_TEXTURE_EXTERNAL_OES, source_texture);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, eglImage);
eglDestroyImageKHR(eglGetCurrentDisplay(), eglImage);
glGenFramebuffers(1, &source_fb);
glBindFramebuffer(GL_FRAMEBUFFER, source_fb);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_TEXTURE_EXTERNAL_OES, source_texture, 0);
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)
return false;
glGenTextures(1, &dest_texture);
glBindTexture(GL_TEXTURE_2D, dest_texture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, desc.width, desc.height,
0, GL_RGB, GL_UNSIGNED_BYTE, nullptr);
glGenFramebuffers(1, &dest_fb);
glBindFramebuffer(GL_FRAMEBUFFER, dest_fb);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_TEXTURE_2D, dest_texture, 0);
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)
return false;
blit();
return true;
}
[[nodiscard]] GLuint dest_texture_id() const
{
return dest_texture;
}
void blit()
{
glBindFramebuffer(GL_READ_FRAMEBUFFER, source_fb);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, dest_fb);
glBlitFramebuffer(0, 0, width, height, 0, 0, width, height, GL_COLOR_BUFFER_BIT, GL_NEAREST);
}
};
// Callback function pointers
typedef void (*OnMessageCallback)(const char*);
typedef void (*OnServiceInitializedCallback)(bool success);
typedef void (*OnFrameAvailableCallback)();
typedef void (*OnBufferReadyCallback)();
typedef void (*OnTextureReadyCallback)(GLuint gl_texture);
typedef void (*OnTextureReadyCallback)(void* nativeTexturePtr, int width, int height, bool isVulkan);
struct NativeCallbacks
{
OnMessageCallback OnMessage;
@@ -102,49 +41,156 @@ NativeCallbacks g_callbacks{};
class ServiceContext : public BnMosisListener
{
AHardwareBuffer* m_hwbuffer = nullptr;
std::unique_ptr<TextureBlitter> m_texture;
public:
ndk::ScopedAStatus onServiceInitialized(bool in_success) override
{
Logger::Log("onServiceInitialized");
g_callbacks.OnMessage("onServiceInitialized");
g_callbacks.OnServiceInitialized(in_success);
if (g_callbacks.OnMessage)
g_callbacks.OnMessage("onServiceInitialized");
if (g_callbacks.OnServiceInitialized)
g_callbacks.OnServiceInitialized(in_success);
return ndk::ScopedAStatus::ok();
}
ndk::ScopedAStatus onFrameAvailable() override
{
Logger::Log("onFrameAvailable");
g_callbacks.OnMessage("onFrameAvailable");
g_callbacks.OnFrameAvailable();
if (g_callbacks.OnMessage)
g_callbacks.OnMessage("onFrameAvailable");
if (g_callbacks.OnFrameAvailable)
g_callbacks.OnFrameAvailable();
return ndk::ScopedAStatus::ok();
}
ndk::ScopedAStatus onBufferAvailable(const HardwareBuffer &in_buffer) override
{
Logger::Log("onBufferAvailable");
g_callbacks.OnMessage("onBufferAvailable");
if (g_callbacks.OnMessage)
g_callbacks.OnMessage("onBufferAvailable");
m_hwbuffer = in_buffer.get();
AHardwareBuffer_acquire(m_hwbuffer);
g_callbacks.OnBufferReady();
if (g_callbacks.OnBufferReady)
g_callbacks.OnBufferReady();
return ndk::ScopedAStatus::ok();
}
[[nodiscard]] GLuint create_texture()
bool CreateTexture()
{
m_texture = std::make_unique<TextureBlitter>();
m_texture->create(m_hwbuffer);
return m_texture->dest_texture_id();
if (!m_hwbuffer)
{
Logger::Log("CreateTexture: No hardware buffer available");
return false;
}
if (!g_backend)
{
Logger::Log("CreateTexture: No backend available");
return false;
}
if (!g_backend->Create(m_hwbuffer))
{
Logger::Log("CreateTexture: Backend Create failed");
return false;
}
// Notify C# with texture info
if (g_callbacks.OnTextureReady)
{
g_callbacks.OnTextureReady(
g_backend->GetNativeTexturePtr(),
g_backend->GetWidth(),
g_backend->GetHeight(),
g_backend->IsVulkan()
);
}
return true;
}
void update_texture()
void UpdateTexture()
{
if (m_texture)
m_texture->blit();
if (g_backend)
{
g_backend->Update();
}
}
};
// Unity Plugin Interface
extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API
UnityPluginLoad(IUnityInterfaces* unityInterfaces)
{
g_unityInterfaces = unityInterfaces;
IUnityGraphics* graphics = unityInterfaces->Get<IUnityGraphics>();
if (!graphics)
{
Logger::Log("UnityPluginLoad: IUnityGraphics not available");
return;
}
g_rendererType = graphics->GetRenderer();
Logger::Log(std::format("UnityPluginLoad: Renderer type = {}", static_cast<int>(g_rendererType)));
// Try to create Vulkan backend first
if (g_rendererType == kUnityGfxRendererVulkan)
{
auto vulkanBackend = std::make_unique<VulkanTextureBackend>();
if (vulkanBackend->Initialize(unityInterfaces))
{
g_backend = std::move(vulkanBackend);
Logger::Log("UnityPluginLoad: Using Vulkan backend");
return;
}
Logger::Log("UnityPluginLoad: Vulkan initialization failed, falling back to OpenGL");
}
// Fall back to OpenGL for GLES renderers
if (g_rendererType == kUnityGfxRendererOpenGLES20 ||
g_rendererType == kUnityGfxRendererOpenGLES30)
{
g_backend = std::make_unique<OpenGLTextureBackend>();
Logger::Log("UnityPluginLoad: Using OpenGL backend");
}
else
{
Logger::Log(std::format("UnityPluginLoad: Unsupported renderer type {}", static_cast<int>(g_rendererType)));
}
}
extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API
UnityPluginUnload()
{
Logger::Log("UnityPluginUnload");
// Clean up backend
if (g_backend)
{
g_backend->Destroy();
g_backend.reset();
}
g_unityInterfaces = nullptr;
g_rendererType = kUnityGfxRendererNull;
}
// Native callback setters
extern "C" void SetNativeCallbacks(const NativeCallbacks& ptr)
{
g_callbacks = ptr;
}
// Touch input forwarding
extern "C" void SendTouchDown(float x, float y)
{
if (g_service)
{
g_service->onTouchDown(x, y);
}
}
extern "C" void SendTouchMove(float x, float y)
{
if (g_service)
@@ -153,40 +199,60 @@ extern "C" void SendTouchMove(float x, float y)
}
}
extern "C" UnityRenderingEvent InitGLAD()
extern "C" void SendTouchUp(float x, float y)
{
return [](int eventId){
gladLoaderLoadEGL(EGL_NO_DISPLAY);
int egl_version = gladLoadEGL(eglGetCurrentDisplay(), eglGetProcAddress);
if (egl_version == 0)
if (g_service)
{
g_service->onTouchUp(x, y);
}
}
// Unity render thread callbacks
static void UNITY_INTERFACE_API OnInitBackendRenderThread(int eventId)
{
Logger::Log("OnInitBackendRenderThread");
// For OpenGL, we need to initialize GLAD on the render thread
if (!g_backend->IsVulkan())
{
if (!OpenGLTextureBackend::InitializeGLAD())
{
Logger::Log("Failed to load EGL");
Logger::Log("OnInitBackendRenderThread: Failed to initialize GLAD");
return;
}
int gl_version = gladLoaderLoadGLES2();
if (gl_version == 0)
{
Logger::Log("Failed to load GL");
return;
}
if (g_context)
{
GLuint texture = g_context->create_texture();
g_callbacks.OnTextureReady(texture);
}
};
}
if (g_context)
{
g_context->CreateTexture();
}
}
static void UNITY_INTERFACE_API OnUpdateTextureRenderThread(int eventId)
{
if (g_context)
{
g_context->UpdateTexture();
}
}
extern "C" UnityRenderingEvent InitBackend()
{
return OnInitBackendRenderThread;
}
extern "C" UnityRenderingEvent UpdateTexture()
{
return [](int eventId){
if (g_context)
{
g_context->update_texture();
}
};
return OnUpdateTextureRenderThread;
}
// Legacy compatibility - redirect to new names
extern "C" UnityRenderingEvent InitGLAD()
{
return InitBackend();
}
// JNI entry point from Kotlin
extern "C"
JNIEXPORT void JNICALL
Java_com_omixlab_mosis_unity_MyKotlinPlugin_serviceConnected(JNIEnv *env, jobject thiz,
@@ -196,6 +262,7 @@ Java_com_omixlab_mosis_unity_MyKotlinPlugin_serviceConnected(JNIEnv *env, jobjec
const ndk::SpAIBinder spBinder(pBinder);
g_service = IMosisService::fromBinder(spBinder);
Logger::Log("Service Connected");
g_context = ndk::SharedRefBase::make<ServiceContext>();
bool result{};
g_service->initOS(g_context, &result);

View File

@@ -0,0 +1,178 @@
#include "opengl_backend.h"
#include <logger.h>
bool OpenGLTextureBackend::InitializeGLAD()
{
gladLoaderLoadEGL(EGL_NO_DISPLAY);
int egl_version = gladLoadEGL(eglGetCurrentDisplay(), eglGetProcAddress);
if (egl_version == 0)
{
Logger::Log("OpenGLTextureBackend: Failed to load EGL");
return false;
}
int gl_version = gladLoaderLoadGLES2();
if (gl_version == 0)
{
Logger::Log("OpenGLTextureBackend: Failed to load GLES2");
return false;
}
Logger::Log("OpenGLTextureBackend: GLAD initialized successfully");
return true;
}
bool OpenGLTextureBackend::Create(AHardwareBuffer* buffer)
{
if (m_Created)
{
Destroy();
}
// Get buffer dimensions
AHardwareBuffer_Desc desc{};
AHardwareBuffer_describe(buffer, &desc);
m_Width = static_cast<GLint>(desc.width);
m_Height = static_cast<GLint>(desc.height);
// Create EGL image from hardware buffer
EGLClientBuffer clientBuffer = eglGetNativeClientBufferANDROID(buffer);
EGLImageKHR eglImage = eglCreateImageKHR(
eglGetCurrentDisplay(),
EGL_NO_CONTEXT,
EGL_NATIVE_BUFFER_ANDROID,
clientBuffer,
nullptr
);
if (eglImage == EGL_NO_IMAGE_KHR)
{
Logger::Log("OpenGLTextureBackend: Failed to create EGL image");
return false;
}
// Create source texture (external OES format from hardware buffer)
glGenTextures(1, &m_SourceTexture);
glBindTexture(GL_TEXTURE_EXTERNAL_OES, m_SourceTexture);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, eglImage);
eglDestroyImageKHR(eglGetCurrentDisplay(), eglImage);
// Create source framebuffer
glGenFramebuffers(1, &m_SourceFramebuffer);
glBindFramebuffer(GL_FRAMEBUFFER, m_SourceFramebuffer);
glFramebufferTexture2D(
GL_FRAMEBUFFER,
GL_COLOR_ATTACHMENT0,
GL_TEXTURE_EXTERNAL_OES,
m_SourceTexture,
0
);
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)
{
Logger::Log("OpenGLTextureBackend: Source framebuffer incomplete");
Destroy();
return false;
}
// Create destination texture (standard 2D texture for Unity)
glGenTextures(1, &m_DestTexture);
glBindTexture(GL_TEXTURE_2D, m_DestTexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(
GL_TEXTURE_2D,
0,
GL_RGB,
m_Width,
m_Height,
0,
GL_RGB,
GL_UNSIGNED_BYTE,
nullptr
);
// Create destination framebuffer
glGenFramebuffers(1, &m_DestFramebuffer);
glBindFramebuffer(GL_FRAMEBUFFER, m_DestFramebuffer);
glFramebufferTexture2D(
GL_FRAMEBUFFER,
GL_COLOR_ATTACHMENT0,
GL_TEXTURE_2D,
m_DestTexture,
0
);
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)
{
Logger::Log("OpenGLTextureBackend: Dest framebuffer incomplete");
Destroy();
return false;
}
m_Created = true;
// Initial blit
Blit();
Logger::Log("OpenGLTextureBackend: Created successfully");
return true;
}
void OpenGLTextureBackend::Update()
{
if (m_Created)
{
Blit();
}
}
void OpenGLTextureBackend::Blit()
{
glBindFramebuffer(GL_READ_FRAMEBUFFER, m_SourceFramebuffer);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, m_DestFramebuffer);
glBlitFramebuffer(
0, 0, m_Width, m_Height,
0, 0, m_Width, m_Height,
GL_COLOR_BUFFER_BIT,
GL_NEAREST
);
}
void OpenGLTextureBackend::Destroy()
{
if (m_DestFramebuffer)
{
glDeleteFramebuffers(1, &m_DestFramebuffer);
m_DestFramebuffer = 0;
}
if (m_SourceFramebuffer)
{
glDeleteFramebuffers(1, &m_SourceFramebuffer);
m_SourceFramebuffer = 0;
}
if (m_DestTexture)
{
glDeleteTextures(1, &m_DestTexture);
m_DestTexture = 0;
}
if (m_SourceTexture)
{
glDeleteTextures(1, &m_SourceTexture);
m_SourceTexture = 0;
}
m_Width = 0;
m_Height = 0;
m_Created = false;
}
void* OpenGLTextureBackend::GetNativeTexturePtr()
{
return reinterpret_cast<void*>(static_cast<uintptr_t>(m_DestTexture));
}

View File

@@ -0,0 +1,42 @@
#pragma once
#include "texture_backend.h"
#include <glad/gles2.h>
#include <glad/egl.h>
/**
* OpenGL ES implementation of ITextureBackend.
* Imports AHardwareBuffer via EGL image and blits to a standard GL_TEXTURE_2D.
*/
class OpenGLTextureBackend : public ITextureBackend
{
public:
OpenGLTextureBackend() = default;
~OpenGLTextureBackend() override { Destroy(); }
bool Create(AHardwareBuffer* buffer) override;
void Update() override;
void Destroy() override;
void* GetNativeTexturePtr() override;
int GetWidth() const override { return m_Width; }
int GetHeight() const override { return m_Height; }
bool IsVulkan() const override { return false; }
/**
* Initialize GLAD for OpenGL ES.
* Must be called on the render thread before Create().
* @return true if initialization succeeded
*/
static bool InitializeGLAD();
private:
void Blit();
GLuint m_SourceTexture = 0; // GL_TEXTURE_EXTERNAL_OES from HardwareBuffer
GLuint m_SourceFramebuffer = 0;
GLuint m_DestTexture = 0; // GL_TEXTURE_2D for Unity
GLuint m_DestFramebuffer = 0;
GLint m_Width = 0;
GLint m_Height = 0;
bool m_Created = false;
};

View File

@@ -0,0 +1,51 @@
#pragma once
#include <android/hardware_buffer.h>
#include <cstdint>
/**
* Abstract interface for texture backends.
* Allows swapping between OpenGL and Vulkan implementations at runtime.
*/
class ITextureBackend
{
public:
virtual ~ITextureBackend() = default;
/**
* Create texture resources from a hardware buffer.
* @param buffer The AHardwareBuffer from the Mosis service
* @return true if creation succeeded
*/
virtual bool Create(AHardwareBuffer* buffer) = 0;
/**
* Update the texture (blit from source to destination).
* Called each frame when a new frame is available.
*/
virtual void Update() = 0;
/**
* Destroy all texture resources.
*/
virtual void Destroy() = 0;
/**
* Get the native texture pointer for Unity.
* For OpenGL: GLuint cast to void*
* For Vulkan: VkImage cast to void*
*/
virtual void* GetNativeTexturePtr() = 0;
/**
* Get texture dimensions.
*/
virtual int GetWidth() const = 0;
virtual int GetHeight() const = 0;
/**
* Check if the backend is Vulkan-based.
* Unity needs this to know how to interpret the native pointer.
*/
virtual bool IsVulkan() const = 0;
};

View File

@@ -0,0 +1,465 @@
#include "vulkan_backend.h"
#include <logger.h>
#include <format>
bool VulkanTextureBackend::Initialize(IUnityInterfaces* unityInterfaces)
{
if (m_Initialized)
{
return true;
}
m_UnityVulkan = unityInterfaces->Get<IUnityGraphicsVulkan>();
if (!m_UnityVulkan)
{
Logger::Log("VulkanTextureBackend: IUnityGraphicsVulkan not available");
return false;
}
// Get Unity's Vulkan instance
UnityVulkanInstance vkInstance = m_UnityVulkan->Instance();
m_Instance = vkInstance.instance;
m_PhysicalDevice = vkInstance.physicalDevice;
m_Device = vkInstance.device;
m_QueueFamilyIndex = vkInstance.queueFamilyIndex;
// Get a queue from Unity
vkGetDeviceQueue(m_Device, m_QueueFamilyIndex, 0, &m_Queue);
// Load extension function
vkGetAndroidHardwareBufferPropertiesANDROID =
reinterpret_cast<PFN_vkGetAndroidHardwareBufferPropertiesANDROID>(
vkGetInstanceProcAddr(m_Instance, "vkGetAndroidHardwareBufferPropertiesANDROID")
);
if (!vkGetAndroidHardwareBufferPropertiesANDROID)
{
Logger::Log("VulkanTextureBackend: vkGetAndroidHardwareBufferPropertiesANDROID not available");
return false;
}
// Create command pool
VkCommandPoolCreateInfo poolInfo{};
poolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
poolInfo.queueFamilyIndex = m_QueueFamilyIndex;
poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
if (vkCreateCommandPool(m_Device, &poolInfo, nullptr, &m_CommandPool) != VK_SUCCESS)
{
Logger::Log("VulkanTextureBackend: Failed to create command pool");
return false;
}
// Allocate command buffer
VkCommandBufferAllocateInfo allocInfo{};
allocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
allocInfo.commandPool = m_CommandPool;
allocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
allocInfo.commandBufferCount = 1;
if (vkAllocateCommandBuffers(m_Device, &allocInfo, &m_CommandBuffer) != VK_SUCCESS)
{
Logger::Log("VulkanTextureBackend: Failed to allocate command buffer");
vkDestroyCommandPool(m_Device, m_CommandPool, nullptr);
m_CommandPool = VK_NULL_HANDLE;
return false;
}
m_Initialized = true;
Logger::Log("VulkanTextureBackend: Initialized successfully");
return true;
}
bool VulkanTextureBackend::Create(AHardwareBuffer* buffer)
{
if (!m_Initialized)
{
Logger::Log("VulkanTextureBackend: Not initialized");
return false;
}
if (m_Created)
{
Destroy();
}
// Get buffer dimensions
AHardwareBuffer_Desc desc{};
AHardwareBuffer_describe(buffer, &desc);
m_Width = static_cast<int>(desc.width);
m_Height = static_cast<int>(desc.height);
Logger::Log(std::format("VulkanTextureBackend: Creating {}x{} texture", m_Width, m_Height));
if (!ImportHardwareBuffer(buffer))
{
Logger::Log("VulkanTextureBackend: Failed to import hardware buffer");
return false;
}
if (!CreateLocalImage())
{
Logger::Log("VulkanTextureBackend: Failed to create local image");
DestroyImportedResources();
return false;
}
m_Created = true;
// Initial copy
CopyImage();
Logger::Log("VulkanTextureBackend: Created successfully");
return true;
}
bool VulkanTextureBackend::ImportHardwareBuffer(AHardwareBuffer* buffer)
{
// Query hardware buffer properties
VkAndroidHardwareBufferFormatPropertiesANDROID formatProps{};
formatProps.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_FORMAT_PROPERTIES_ANDROID;
VkAndroidHardwareBufferPropertiesANDROID props{};
props.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID;
props.pNext = &formatProps;
if (vkGetAndroidHardwareBufferPropertiesANDROID(m_Device, buffer, &props) != VK_SUCCESS)
{
Logger::Log("VulkanTextureBackend: Failed to get hardware buffer properties");
return false;
}
m_Format = formatProps.format;
Logger::Log(std::format("VulkanTextureBackend: Buffer format: {}", static_cast<int>(m_Format)));
// Create VkImage with external memory
VkExternalMemoryImageCreateInfo extMemImageInfo{};
extMemImageInfo.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO;
extMemImageInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID;
VkImageCreateInfo imageInfo{};
imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
imageInfo.pNext = &extMemImageInfo;
imageInfo.imageType = VK_IMAGE_TYPE_2D;
imageInfo.format = m_Format;
imageInfo.extent = {static_cast<uint32_t>(m_Width), static_cast<uint32_t>(m_Height), 1};
imageInfo.mipLevels = 1;
imageInfo.arrayLayers = 1;
imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
imageInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
if (vkCreateImage(m_Device, &imageInfo, nullptr, &m_ImportedImage) != VK_SUCCESS)
{
Logger::Log("VulkanTextureBackend: Failed to create imported image");
return false;
}
// Import memory from hardware buffer
VkImportAndroidHardwareBufferInfoANDROID importInfo{};
importInfo.sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID;
importInfo.buffer = buffer;
VkMemoryDedicatedAllocateInfo dedicatedInfo{};
dedicatedInfo.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO;
dedicatedInfo.pNext = &importInfo;
dedicatedInfo.image = m_ImportedImage;
VkMemoryAllocateInfo allocInfo{};
allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
allocInfo.pNext = &dedicatedInfo;
allocInfo.allocationSize = props.allocationSize;
allocInfo.memoryTypeIndex = FindMemoryType(props.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
if (vkAllocateMemory(m_Device, &allocInfo, nullptr, &m_ImportedMemory) != VK_SUCCESS)
{
Logger::Log("VulkanTextureBackend: Failed to allocate imported memory");
vkDestroyImage(m_Device, m_ImportedImage, nullptr);
m_ImportedImage = VK_NULL_HANDLE;
return false;
}
if (vkBindImageMemory(m_Device, m_ImportedImage, m_ImportedMemory, 0) != VK_SUCCESS)
{
Logger::Log("VulkanTextureBackend: Failed to bind imported memory");
DestroyImportedResources();
return false;
}
Logger::Log("VulkanTextureBackend: Hardware buffer imported");
return true;
}
bool VulkanTextureBackend::CreateLocalImage()
{
// Create local VkImage for safe rendering
VkImageCreateInfo imageInfo{};
imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
imageInfo.imageType = VK_IMAGE_TYPE_2D;
imageInfo.format = m_Format;
imageInfo.extent = {static_cast<uint32_t>(m_Width), static_cast<uint32_t>(m_Height), 1};
imageInfo.mipLevels = 1;
imageInfo.arrayLayers = 1;
imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
imageInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
if (vkCreateImage(m_Device, &imageInfo, nullptr, &m_LocalImage) != VK_SUCCESS)
{
Logger::Log("VulkanTextureBackend: Failed to create local image");
return false;
}
// Allocate memory for local image
VkMemoryRequirements memReqs;
vkGetImageMemoryRequirements(m_Device, m_LocalImage, &memReqs);
VkMemoryAllocateInfo allocInfo{};
allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
allocInfo.allocationSize = memReqs.size;
allocInfo.memoryTypeIndex = FindMemoryType(memReqs.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
if (vkAllocateMemory(m_Device, &allocInfo, nullptr, &m_LocalMemory) != VK_SUCCESS)
{
Logger::Log("VulkanTextureBackend: Failed to allocate local memory");
vkDestroyImage(m_Device, m_LocalImage, nullptr);
m_LocalImage = VK_NULL_HANDLE;
return false;
}
if (vkBindImageMemory(m_Device, m_LocalImage, m_LocalMemory, 0) != VK_SUCCESS)
{
Logger::Log("VulkanTextureBackend: Failed to bind local memory");
DestroyLocalResources();
return false;
}
// Create image view
VkImageViewCreateInfo viewInfo{};
viewInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
viewInfo.image = m_LocalImage;
viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D;
viewInfo.format = m_Format;
viewInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
viewInfo.subresourceRange.baseMipLevel = 0;
viewInfo.subresourceRange.levelCount = 1;
viewInfo.subresourceRange.baseArrayLayer = 0;
viewInfo.subresourceRange.layerCount = 1;
if (vkCreateImageView(m_Device, &viewInfo, nullptr, &m_LocalImageView) != VK_SUCCESS)
{
Logger::Log("VulkanTextureBackend: Failed to create image view");
DestroyLocalResources();
return false;
}
Logger::Log("VulkanTextureBackend: Local image created");
return true;
}
void VulkanTextureBackend::Update()
{
if (m_Created)
{
CopyImage();
}
}
void VulkanTextureBackend::CopyImage()
{
// Begin command buffer
VkCommandBufferBeginInfo beginInfo{};
beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
vkResetCommandBuffer(m_CommandBuffer, 0);
vkBeginCommandBuffer(m_CommandBuffer, &beginInfo);
// Transition imported image to transfer src
TransitionImageLayout(m_ImportedImage,
VK_IMAGE_LAYOUT_UNDEFINED,
VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
// Transition local image to transfer dst
TransitionImageLayout(m_LocalImage,
VK_IMAGE_LAYOUT_UNDEFINED,
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
// Copy image
VkImageCopy copyRegion{};
copyRegion.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
copyRegion.srcSubresource.layerCount = 1;
copyRegion.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
copyRegion.dstSubresource.layerCount = 1;
copyRegion.extent = {static_cast<uint32_t>(m_Width), static_cast<uint32_t>(m_Height), 1};
vkCmdCopyImage(
m_CommandBuffer,
m_ImportedImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
m_LocalImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
1, &copyRegion
);
// Transition local image to shader read
TransitionImageLayout(m_LocalImage,
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
vkEndCommandBuffer(m_CommandBuffer);
// Submit and wait (CPU sync for simplicity)
VkSubmitInfo submitInfo{};
submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
submitInfo.commandBufferCount = 1;
submitInfo.pCommandBuffers = &m_CommandBuffer;
vkQueueSubmit(m_Queue, 1, &submitInfo, VK_NULL_HANDLE);
vkQueueWaitIdle(m_Queue);
}
void VulkanTextureBackend::TransitionImageLayout(VkImage image, VkImageLayout oldLayout, VkImageLayout newLayout)
{
VkImageMemoryBarrier barrier{};
barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
barrier.oldLayout = oldLayout;
barrier.newLayout = newLayout;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.image = image;
barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
barrier.subresourceRange.baseMipLevel = 0;
barrier.subresourceRange.levelCount = 1;
barrier.subresourceRange.baseArrayLayer = 0;
barrier.subresourceRange.layerCount = 1;
VkPipelineStageFlags srcStage;
VkPipelineStageFlags dstStage;
if (oldLayout == VK_IMAGE_LAYOUT_UNDEFINED)
{
barrier.srcAccessMask = 0;
srcStage = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT;
}
else if (oldLayout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL)
{
barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
srcStage = VK_PIPELINE_STAGE_TRANSFER_BIT;
}
else
{
barrier.srcAccessMask = 0;
srcStage = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT;
}
if (newLayout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL)
{
barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
dstStage = VK_PIPELINE_STAGE_TRANSFER_BIT;
}
else if (newLayout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL)
{
barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
dstStage = VK_PIPELINE_STAGE_TRANSFER_BIT;
}
else if (newLayout == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
{
barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
dstStage = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
}
else
{
barrier.dstAccessMask = 0;
dstStage = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT;
}
vkCmdPipelineBarrier(
m_CommandBuffer,
srcStage, dstStage,
0,
0, nullptr,
0, nullptr,
1, &barrier
);
}
uint32_t VulkanTextureBackend::FindMemoryType(uint32_t typeFilter, VkMemoryPropertyFlags properties)
{
VkPhysicalDeviceMemoryProperties memProps;
vkGetPhysicalDeviceMemoryProperties(m_PhysicalDevice, &memProps);
for (uint32_t i = 0; i < memProps.memoryTypeCount; i++)
{
if ((typeFilter & (1 << i)) &&
(memProps.memoryTypes[i].propertyFlags & properties) == properties)
{
return i;
}
}
Logger::Log("VulkanTextureBackend: Failed to find suitable memory type");
return 0;
}
void VulkanTextureBackend::Destroy()
{
if (m_Device && m_Created)
{
vkDeviceWaitIdle(m_Device);
}
DestroyLocalResources();
DestroyImportedResources();
m_Width = 0;
m_Height = 0;
m_Created = false;
}
void VulkanTextureBackend::DestroyImportedResources()
{
if (m_Device)
{
if (m_ImportedMemory != VK_NULL_HANDLE)
{
vkFreeMemory(m_Device, m_ImportedMemory, nullptr);
m_ImportedMemory = VK_NULL_HANDLE;
}
if (m_ImportedImage != VK_NULL_HANDLE)
{
vkDestroyImage(m_Device, m_ImportedImage, nullptr);
m_ImportedImage = VK_NULL_HANDLE;
}
}
}
void VulkanTextureBackend::DestroyLocalResources()
{
if (m_Device)
{
if (m_LocalImageView != VK_NULL_HANDLE)
{
vkDestroyImageView(m_Device, m_LocalImageView, nullptr);
m_LocalImageView = VK_NULL_HANDLE;
}
if (m_LocalMemory != VK_NULL_HANDLE)
{
vkFreeMemory(m_Device, m_LocalMemory, nullptr);
m_LocalMemory = VK_NULL_HANDLE;
}
if (m_LocalImage != VK_NULL_HANDLE)
{
vkDestroyImage(m_Device, m_LocalImage, nullptr);
m_LocalImage = VK_NULL_HANDLE;
}
}
}
void* VulkanTextureBackend::GetNativeTexturePtr()
{
// For Vulkan, Unity expects the VkImage
return reinterpret_cast<void*>(m_LocalImage);
}

View File

@@ -0,0 +1,82 @@
#pragma once
#include "texture_backend.h"
#include <vulkan/vulkan.h>
#include <vulkan/vulkan_android.h>
#include <IUnityGraphicsVulkan.h>
/**
* Vulkan implementation of ITextureBackend.
* Imports AHardwareBuffer via VK_ANDROID_external_memory_android_hardware_buffer
* and copies to a local VkImage for safe rendering.
*/
class VulkanTextureBackend : public ITextureBackend
{
public:
VulkanTextureBackend() = default;
~VulkanTextureBackend() override { Destroy(); }
/**
* Initialize the Vulkan backend with Unity's Vulkan interface.
* Must be called during UnityPluginLoad.
* @param unityInterfaces Unity's interface registry
* @return true if Vulkan is available and initialization succeeded
*/
bool Initialize(IUnityInterfaces* unityInterfaces);
bool Create(AHardwareBuffer* buffer) override;
void Update() override;
void Destroy() override;
void* GetNativeTexturePtr() override;
int GetWidth() const override { return m_Width; }
int GetHeight() const override { return m_Height; }
bool IsVulkan() const override { return true; }
/**
* Get the VkImageView for Unity texture binding.
*/
VkImageView GetLocalImageView() const { return m_LocalImageView; }
private:
bool ImportHardwareBuffer(AHardwareBuffer* buffer);
bool CreateLocalImage();
void DestroyImportedResources();
void DestroyLocalResources();
uint32_t FindMemoryType(uint32_t typeFilter, VkMemoryPropertyFlags properties);
void TransitionImageLayout(VkImage image, VkImageLayout oldLayout, VkImageLayout newLayout);
void CopyImage();
// Unity Vulkan interface
IUnityGraphicsVulkan* m_UnityVulkan = nullptr;
// Vulkan device objects (from Unity)
VkInstance m_Instance = VK_NULL_HANDLE;
VkPhysicalDevice m_PhysicalDevice = VK_NULL_HANDLE;
VkDevice m_Device = VK_NULL_HANDLE;
VkQueue m_Queue = VK_NULL_HANDLE;
uint32_t m_QueueFamilyIndex = 0;
// Command pool for copy operations
VkCommandPool m_CommandPool = VK_NULL_HANDLE;
VkCommandBuffer m_CommandBuffer = VK_NULL_HANDLE;
// Imported hardware buffer resources
VkImage m_ImportedImage = VK_NULL_HANDLE;
VkDeviceMemory m_ImportedMemory = VK_NULL_HANDLE;
// Local copy resources (for safe rendering)
VkImage m_LocalImage = VK_NULL_HANDLE;
VkDeviceMemory m_LocalMemory = VK_NULL_HANDLE;
VkImageView m_LocalImageView = VK_NULL_HANDLE;
// Texture format info
VkFormat m_Format = VK_FORMAT_R8G8B8A8_UNORM;
int m_Width = 0;
int m_Height = 0;
bool m_Initialized = false;
bool m_Created = false;
// Extension function pointers
PFN_vkGetAndroidHardwareBufferPropertiesANDROID vkGetAndroidHardwareBufferPropertiesANDROID = nullptr;
};

View File

@@ -1 +1,221 @@
Use this file to describe your package's features.
# MosisSDK for Unity
Unity package providing integration with MosisService for virtual smartphone display in VR applications.
## Overview
This package connects to the MosisService Android app via AIDL Binder IPC to:
- Receive rendered phone screen frames via AHardwareBuffer
- Forward VR controller touch input to the virtual phone
- Support both OpenGL ES and Vulkan graphics backends
## Requirements
- Unity 6000.3.2f1 or later
- Android target platform
- MosisService app installed on device
## Installation
1. Open Window > Package Manager
2. Click + > Add package from disk
3. Navigate to `Packages/com.omarator.mosissdk/package.json`
## Project Structure
```
Packages/com.omarator.mosissdk/
├── package.json
├── README.md
├── Runtime/
│ ├── KotlinBridge.cs # C# to native bridge
│ └── com.omarator.mosissdk.asmdef
├── Plugins/
│ └── Android/
│ ├── MosisSDK.aar # Kotlin service connection
│ ├── libs/
│ │ └── arm64-v8a/
│ │ └── libmy_native_lib.so
│ └── cpp/ # Native source
│ ├── CMakeLists.txt
│ ├── my_native_code.cpp
│ ├── texture_backend.h
│ ├── opengl_backend.h/cpp
│ └── vulkan_backend.h/cpp
└── Assets/
└── PhoneInteraction.cs # VR controller input
```
## Build Commands
### Unity Build
1. File > Build Settings
2. Switch Platform to Android
3. Player Settings > Other Settings:
- Graphics APIs: Vulkan, OpenGLES3 (in order of preference)
- Minimum API Level: 29
4. Build or Build and Run
### Native Plugin Build
To rebuild the native library manually:
```batch
cd Packages/com.omarator.mosissdk/Plugins/Android/cpp
:: Configure with Android NDK
cmake -B build ^
-DCMAKE_TOOLCHAIN_FILE=%ANDROID_NDK_HOME%/build/cmake/android.toolchain.cmake ^
-DANDROID_ABI=arm64-v8a ^
-DANDROID_PLATFORM=android-29
:: Build
cmake --build build
:: Copy output
copy build\libmy_native_lib.so ..\libs\arm64-v8a\
```
## Usage
### Basic Setup
1. Add `KotlinBridge` component to a GameObject in your scene
2. Add `PhoneInteraction` component for VR input handling
3. Assign VR controller and phone plane references
### KotlinBridge
The `KotlinBridge` component manages the connection to MosisService:
```csharp
public class KotlinBridge : MonoBehaviour
{
public Material phoneMaterial; // Material to apply phone texture
// Called automatically when texture is ready
// Texture is applied to phoneMaterial's _MainTex
}
```
### PhoneInteraction
Handles VR controller raycasting and touch input:
```csharp
public class PhoneInteraction : MonoBehaviour
{
public GameObject Controller; // VR controller
public GameObject Plane; // Phone display plane
public XRNode inputDevice; // Controller hand
public float triggerThreshold; // Trigger press threshold
}
```
### Touch API
Send touch events from custom scripts:
```csharp
// Touch down at normalized UV (0-1)
KotlinBridge.SendTouchDown(0.5f, 0.5f);
// Touch move
KotlinBridge.SendTouchMove(0.6f, 0.5f);
// Touch up
KotlinBridge.SendTouchUp(0.6f, 0.5f);
```
## Graphics Backend
The native plugin automatically selects the appropriate backend:
| Graphics API | Backend | Status |
|--------------|---------|--------|
| Vulkan | VulkanTextureBackend | Preferred |
| OpenGL ES 3.0 | OpenGLTextureBackend | Fallback |
| OpenGL ES 2.0 | OpenGLTextureBackend | Fallback |
Backend selection happens in `UnityPluginLoad()` based on the active renderer.
### Vulkan Import
When using Vulkan, the plugin imports AHardwareBuffer directly as a VkImage:
1. Query buffer properties via `vkGetAndroidHardwareBufferPropertiesANDROID`
2. Create VkImage with external memory handle type
3. Import memory with `VkImportAndroidHardwareBufferInfoANDROID`
4. Copy to local image each frame for safe rendering
### OpenGL Import
When using OpenGL ES, the plugin uses EGL image extension:
1. Create EGLImage from AHardwareBuffer via `eglCreateImageKHR`
2. Bind to OpenGL texture via `glEGLImageTargetTexture2DOES`
3. Blit to local texture each frame
## Device Testing
```bash
# Install MosisService first
adb install -r MosisService-debug.apk
# Install Unity app
adb install -r MosisVR.apk
# Launch service
adb shell am start -n com.omixlab.mosis/.MainActivity
# Launch Unity app
adb shell am start -n com.omixlab.mosisvr/com.unity3d.player.UnityPlayerActivity
# Monitor logs
adb logcat -s Unity MosisSDK Vulkan
```
## Troubleshooting
### Black Screen / No Texture
- Verify MosisService is running (`adb shell ps | grep mosis`)
- Check logcat for connection errors
- Ensure both apps have same user ID or are debuggable
### Vulkan Errors
- Device must support `VK_ANDROID_external_memory_android_hardware_buffer`
- Check `adb logcat -s Vulkan` for extension availability
- Falls back to OpenGL if Vulkan init fails
### Touch Not Working
- Verify `PhoneInteraction` references are set
- Check XR input device selection matches controller
- Test with mouse click fallback in editor
## Implementation Notes
### Files Modified in Vulkan Implementation
| File | Changes |
|------|---------|
| `my_native_code.cpp` | Backend abstraction, `UnityPluginLoad` detection |
| `CMakeLists.txt` | Added Vulkan library, new source files |
| `KotlinBridge.cs` | Added `SendTouchDown`/`SendTouchUp`, dynamic resolution |
| `PhoneInteraction.cs` | Added VR trigger-based touch handling |
### New Files Added
| File | Purpose |
|------|---------|
| `texture_backend.h` | ITextureBackend abstract interface |
| `opengl_backend.h/cpp` | OpenGL ES implementation |
| `vulkan_backend.h/cpp` | Vulkan implementation |
## Version History
- **1.0.0** - Initial release with OpenGL support
- **1.1.0** - Added Vulkan backend, touch down/up events, dynamic resolution

View File

@@ -6,6 +6,7 @@ using System.Runtime.InteropServices;
public class KotlinBridge : MonoBehaviour
{
static KotlinBridge Instance;
[StructLayout(LayoutKind.Sequential)]
struct NativeCallbacks
{
@@ -15,24 +16,40 @@ public class KotlinBridge : MonoBehaviour
public IntPtr OnBufferReady;
public IntPtr OnTextureReady;
}
delegate void OnMessageDelegate(string message);
delegate void OnServiceInitializedDelegate(bool success);
delegate void OnFrameAvailableDelegate();
delegate void OnBufferReadyDelegate();
delegate void OnTextureReadyDelegate(uint gl_texture);
delegate void OnTextureReadyDelegate(IntPtr nativeTexturePtr, int width, int height, bool isVulkan);
[DllImport("my_native_lib")]
static extern void SetNativeCallbacks(ref NativeCallbacks callbacks);
[DllImport("my_native_lib")]
public static extern void SendTouchDown(float u, float v);
[DllImport("my_native_lib")]
public static extern void SendTouchMove(float u, float v);
[DllImport("my_native_lib")]
static extern IntPtr InitGLAD();
public static extern void SendTouchUp(float u, float v);
[DllImport("my_native_lib")]
static extern IntPtr InitBackend();
[DllImport("my_native_lib")]
static extern IntPtr UpdateTexture();
// Legacy compatibility
[DllImport("my_native_lib")]
static extern IntPtr InitGLAD();
[AOT.MonoPInvokeCallback(typeof(OnMessageDelegate))]
static void OnMessage(string message)
{
//Debug.Log("High-speed callback: " + message);
// High-speed callback, disabled for performance
// Debug.Log("Native callback: " + message);
}
[AOT.MonoPInvokeCallback(typeof(OnServiceInitializedDelegate))]
@@ -41,7 +58,7 @@ public class KotlinBridge : MonoBehaviour
Debug.Log("Service Initialized: " + success);
}
[AOT.MonoPInvokeCallback(typeof(OnServiceInitializedDelegate))]
[AOT.MonoPInvokeCallback(typeof(OnFrameAvailableDelegate))]
static void OnFrameAvailable()
{
UnityMainThreadDispatcher.Enqueue(() => {
@@ -54,24 +71,59 @@ public class KotlinBridge : MonoBehaviour
{
Debug.Log("Buffer Ready");
UnityMainThreadDispatcher.Enqueue(() => {
GL.IssuePluginEvent(InitGLAD(), 1);
GL.IssuePluginEvent(InitBackend(), 1);
});
}
[AOT.MonoPInvokeCallback(typeof(OnTextureReadyDelegate))]
static void OnTextureReady(uint gl_texture)
static void OnTextureReady(IntPtr nativeTexturePtr, int width, int height, bool isVulkan)
{
Debug.Log("Texture Ready: " + gl_texture);
Debug.Log($"Texture Ready: ptr={nativeTexturePtr}, size={width}x{height}, vulkan={isVulkan}");
UnityMainThreadDispatcher.Enqueue(() => {
var renderer = Instance.GetComponent<Renderer>();
renderer.materials[2].mainTexture = Texture2D.CreateExternalTexture(1024, 1024,
TextureFormat.RGBA32, false, false, (IntPtr)gl_texture);
if (renderer == null)
{
Debug.LogError("KotlinBridge: No Renderer component found");
return;
}
// Create external texture with actual dimensions from the service
// Unity handles both OpenGL and Vulkan native textures via CreateExternalTexture
Texture2D texture = Texture2D.CreateExternalTexture(
width,
height,
TextureFormat.RGBA32,
false, // mipChain
false, // linear
nativeTexturePtr
);
if (texture == null)
{
Debug.LogError("KotlinBridge: Failed to create external texture");
return;
}
// Set texture on material (index 2 is the phone screen material)
if (renderer.materials.Length > 2)
{
renderer.materials[2].mainTexture = texture;
Debug.Log($"KotlinBridge: Texture set on material[2], {width}x{height}");
}
else
{
Debug.LogWarning($"KotlinBridge: Not enough materials (have {renderer.materials.Length}, need 3)");
// Fall back to first material if not enough
renderer.material.mainTexture = texture;
}
});
}
void Start()
{
Instance = this;
NativeCallbacks callbacks = new NativeCallbacks();
callbacks.OnMessage = Marshal.GetFunctionPointerForDelegate(new OnMessageDelegate(OnMessage));
callbacks.OnServiceInitialized = Marshal.GetFunctionPointerForDelegate(new OnServiceInitializedDelegate(OnServiceInitialized));