diff --git a/Plugins/MosisSDK/Source/MosisSDK/MosisSDK.Build.cs b/Plugins/MosisSDK/Source/MosisSDK/MosisSDK.Build.cs index 00891c2..bf25afe 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/MosisSDK.Build.cs +++ b/Plugins/MosisSDK/Source/MosisSDK/MosisSDK.Build.cs @@ -10,7 +10,14 @@ public class MosisSDK : ModuleRules public MosisSDK(ReadOnlyTargetRules Target) : base(Target) { PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; - + + PublicDependencyModuleNames.AddRange( + new string[] + { + "Core" + } + ); + PrivateDependencyModuleNames.AddRange( new string[] { @@ -18,13 +25,18 @@ public class MosisSDK : ModuleRules "Engine", "Slate", "SlateCore", - "Launch" - // ... add private dependencies that you statically link with here ... + "Launch", + "RenderCore", + "RHI" } ); if (Target.Platform == UnrealTargetPlatform.Android) { + // Add Vulkan support + PrivateDependencyModuleNames.Add("VulkanRHI"); + AddEngineThirdPartyPrivateStaticDependencies(Target, "Vulkan"); + // Register the UPL file string PluginPath = Utils.MakePathRelativeTo(ModuleDirectory, Target.RelativeEnginePath); AdditionalPropertiesForReceipt.Add("AndroidPlugin", Path.Combine(PluginPath, "MosisSDK_UPL.xml")); @@ -34,6 +46,11 @@ public class MosisSDK : ModuleRules string BinderPath = Path.Combine(SDKPath, "platforms/android-36/optional/libbinder_ndk_cpp"); PublicIncludePaths.Add(BinderPath); + // Include generated AIDL headers + string GeneratedPath = Path.Combine(ModuleDirectory, "Generated"); + PublicIncludePaths.Add(GeneratedPath); + PrivateIncludePaths.Add(GeneratedPath); + string AidlPath = Path.Combine(SDKPath, "build-tools/36.1.0/aidl.exe"); string AidlSourceDir = Path.Combine(ModuleDirectory, "AIDL"); string OutputHeaderDir = Path.Combine(ModuleDirectory, "Generated"); diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisClient.cpp b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisClient.cpp new file mode 100644 index 0000000..f86e39b --- /dev/null +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisClient.cpp @@ -0,0 +1,148 @@ +#include "MosisClient.h" + +#if PLATFORM_ANDROID + +#include "Logging/LogMacros.h" + +DEFINE_LOG_CATEGORY_STATIC(LogMosisClient, Log, All); + +std::shared_ptr MosisClient::Create(AIBinder* pBinder) +{ + if (!pBinder) + { + UE_LOG(LogMosisClient, Error, TEXT("Create: pBinder is null")); + return nullptr; + } + + // Wrap the binder + ndk::SpAIBinder spBinder(pBinder); + + // Get the service interface + auto service = aidl::com::omixlab::mosis::IMosisService::fromBinder(spBinder); + if (!service) + { + UE_LOG(LogMosisClient, Error, TEXT("Create: Failed to get IMosisService from binder")); + return nullptr; + } + + // Create client using SharedRefBase pattern required by AIDL + auto client = ndk::SharedRefBase::make(); + client->m_Service = service; + + UE_LOG(LogMosisClient, Log, TEXT("Create: Client created, calling initOS...")); + + // Initialize the OS - this will trigger callbacks + bool result = false; + auto status = service->initOS(client, &result); + + if (!status.isOk()) + { + UE_LOG(LogMosisClient, Error, TEXT("Create: initOS failed with status %d"), status.getStatus()); + return nullptr; + } + + UE_LOG(LogMosisClient, Log, TEXT("Create: initOS returned %s"), result ? TEXT("true") : TEXT("false")); + + return client; +} + +MosisClient::~MosisClient() +{ + UE_LOG(LogMosisClient, Log, TEXT("~MosisClient")); + + // Release hardware buffer if we have one + if (m_HardwareBuffer) + { + AHardwareBuffer_release(m_HardwareBuffer); + m_HardwareBuffer = nullptr; + } +} + +ndk::ScopedAStatus MosisClient::onServiceInitialized(bool success) +{ + UE_LOG(LogMosisClient, Log, TEXT("onServiceInitialized: %s"), success ? TEXT("true") : TEXT("false")); + + m_Initialized.store(success); + + if (m_OnInitialized) + { + m_OnInitialized(success); + } + + return ndk::ScopedAStatus::ok(); +} + +ndk::ScopedAStatus MosisClient::onBufferAvailable(const aidl::android::hardware::HardwareBuffer& buffer) +{ + UE_LOG(LogMosisClient, Log, TEXT("onBufferAvailable")); + + // Release old buffer if any + if (m_HardwareBuffer) + { + AHardwareBuffer_release(m_HardwareBuffer); + } + + // Get and acquire the new buffer + m_HardwareBuffer = buffer.get(); + if (m_HardwareBuffer) + { + AHardwareBuffer_acquire(m_HardwareBuffer); + + // Log buffer info + AHardwareBuffer_Desc desc{}; + AHardwareBuffer_describe(m_HardwareBuffer, &desc); + UE_LOG(LogMosisClient, Log, TEXT("onBufferAvailable: %dx%d, format=%d"), + desc.width, desc.height, desc.format); + } + + if (m_OnBufferAvailable) + { + m_OnBufferAvailable(m_HardwareBuffer); + } + + return ndk::ScopedAStatus::ok(); +} + +ndk::ScopedAStatus MosisClient::onFrameAvailable() +{ + // Don't log every frame - too noisy + m_FrameReady.store(true); + + if (m_OnFrameAvailable) + { + m_OnFrameAvailable(); + } + + return ndk::ScopedAStatus::ok(); +} + +bool MosisClient::ConsumeFrameReady() +{ + return m_FrameReady.exchange(false); +} + +void MosisClient::SendTouchDown(float x, float y) +{ + if (m_Service) + { + m_Service->onTouchDown(x, y); + } +} + +void MosisClient::SendTouchMove(float x, float y) +{ + if (m_Service) + { + m_Service->onTouchMove(x, y); + } +} + +void MosisClient::SendTouchUp(float x, float y) +{ + if (m_Service) + { + m_Service->onTouchUp(x, y); + } +} + +#endif // PLATFORM_ANDROID diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisClient.h b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisClient.h new file mode 100644 index 0000000..cd543d7 --- /dev/null +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisClient.h @@ -0,0 +1,74 @@ +#pragma once + +#if PLATFORM_ANDROID + +#include +#include +#include +#include +#include +#include + +/** + * MosisClient - AIDL listener implementation for Unreal Engine. + * Receives callbacks from MosisService when buffers and frames are available. + */ +class MosisClient : public aidl::com::omixlab::mosis::BnMosisListener +{ +public: + using BufferCallback = std::function; + using FrameCallback = std::function; + using InitCallback = std::function; + + /** + * Create and initialize a MosisClient from a Binder handle. + * @param pBinder The AIBinder received from Kotlin service connection + * @return Shared pointer to the client, or nullptr on failure + */ + static std::shared_ptr Create(AIBinder* pBinder); + + ~MosisClient(); + + // IMosisListener implementation + ndk::ScopedAStatus onServiceInitialized(bool success) override; + ndk::ScopedAStatus onBufferAvailable(const aidl::android::hardware::HardwareBuffer& buffer) override; + ndk::ScopedAStatus onFrameAvailable() override; + + // Touch event forwarding to service + void SendTouchDown(float x, float y); + void SendTouchMove(float x, float y); + void SendTouchUp(float x, float y); + + // Hardware buffer access + AHardwareBuffer* GetHardwareBuffer() const { return m_HardwareBuffer; } + + // State queries + bool IsInitialized() const { return m_Initialized.load(); } + bool IsConnected() const { return m_Service != nullptr; } + + /** + * Check if a new frame is ready and consume the flag. + * @return true if a new frame was available (flag is cleared after call) + */ + bool ConsumeFrameReady(); + + // Callback setters + void SetBufferCallback(BufferCallback callback) { m_OnBufferAvailable = std::move(callback); } + void SetFrameCallback(FrameCallback callback) { m_OnFrameAvailable = std::move(callback); } + void SetInitCallback(InitCallback callback) { m_OnInitialized = std::move(callback); } + +private: + MosisClient() = default; + + std::shared_ptr m_Service; + AHardwareBuffer* m_HardwareBuffer = nullptr; + std::atomic m_Initialized{false}; + std::atomic m_FrameReady{false}; + + // Callbacks + BufferCallback m_OnBufferAvailable; + FrameCallback m_OnFrameAvailable; + InitCallback m_OnInitialized; +}; + +#endif // PLATFORM_ANDROID diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneActor.cpp b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneActor.cpp new file mode 100644 index 0000000..b079efe --- /dev/null +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneActor.cpp @@ -0,0 +1,88 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "MosisPhoneActor.h" +#include "Components/StaticMeshComponent.h" +#include "Materials/MaterialInstanceDynamic.h" +#include "Engine/StaticMesh.h" + +DEFINE_LOG_CATEGORY_STATIC(LogMosisPhoneActor, Log, All); + +AMosisPhoneActor::AMosisPhoneActor() +{ + PrimaryActorTick.bCanEverTick = false; + + // Create phone mesh component + PhoneMesh = CreateDefaultSubobject(TEXT("PhoneMesh")); + RootComponent = PhoneMesh; + + // Create phone component + PhoneComponent = CreateDefaultSubobject(TEXT("PhoneComponent")); +} + +void AMosisPhoneActor::BeginPlay() +{ + Super::BeginPlay(); + + UE_LOG(LogMosisPhoneActor, Log, TEXT("BeginPlay")); + + // Create dynamic material for the screen + if (PhoneMesh) + { + UMaterialInterface* BaseMaterial = PhoneMesh->GetMaterial(0); + if (BaseMaterial) + { + ScreenMaterial = UMaterialInstanceDynamic::Create(BaseMaterial, this); + PhoneMesh->SetMaterial(0, ScreenMaterial); + + // Set it on the phone component + if (PhoneComponent) + { + PhoneComponent->PhoneMaterial = ScreenMaterial; + } + + UE_LOG(LogMosisPhoneActor, Log, TEXT("BeginPlay: Created dynamic material")); + } + } +} + +bool AMosisPhoneActor::SendTouchAtWorldLocation(FVector HitLocation, EMosisTouchType TouchType) +{ + FVector2D UV; + if (!WorldToPhoneUV(HitLocation, UV)) + { + return false; + } + + if (PhoneComponent) + { + PhoneComponent->SendTouch(UV, TouchType); + return true; + } + + return false; +} + +bool AMosisPhoneActor::WorldToPhoneUV(FVector WorldLocation, FVector2D& OutUV) const +{ + // Convert world location to local space + FVector LocalLocation = GetActorTransform().InverseTransformPosition(WorldLocation); + + // Get bounds + float MinX = ScreenBoundsLocal.X; + float MinY = ScreenBoundsLocal.Y; + float MaxX = ScreenBoundsLocal.Z; + float MaxY = ScreenBoundsLocal.W; + + // Check if within bounds + if (LocalLocation.X < MinX || LocalLocation.X > MaxX || + LocalLocation.Y < MinY || LocalLocation.Y > MaxY) + { + return false; + } + + // Calculate UV (0-1 range) + OutUV.X = (LocalLocation.X - MinX) / (MaxX - MinX); + OutUV.Y = (LocalLocation.Y - MinY) / (MaxY - MinY); + + return true; +} diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneComponent.cpp b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneComponent.cpp new file mode 100644 index 0000000..f19506a --- /dev/null +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneComponent.cpp @@ -0,0 +1,254 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "MosisPhoneComponent.h" +#include "MosisSDK.h" +#include "Engine/Texture2D.h" +#include "Materials/MaterialInstanceDynamic.h" +#include "RenderingThread.h" + +#if PLATFORM_ANDROID +#include "MosisClient.h" +#include "MosisVulkanTexture.h" +#include "IVulkanDynamicRHI.h" +#include "VulkanRHIPrivate.h" +#endif + +DEFINE_LOG_CATEGORY_STATIC(LogMosisPhone, Log, All); + +UMosisPhoneComponent::UMosisPhoneComponent() +{ + PrimaryComponentTick.bCanEverTick = true; + PrimaryComponentTick.bStartWithTickEnabled = true; +} + +void UMosisPhoneComponent::BeginPlay() +{ + Super::BeginPlay(); + + UE_LOG(LogMosisPhone, Log, TEXT("BeginPlay")); + +#if PLATFORM_ANDROID + // Get the client and set up callbacks + auto Client = FMosisSDKModule::GetClient(); + if (Client) + { + // Set up callbacks + Client->SetBufferCallback([this](AHardwareBuffer* buffer) { + // This runs on the binder thread - flag for main thread processing + bNeedsTextureCreate = true; + }); + + Client->SetFrameCallback([this]() { + // This runs on the binder thread - flag for tick processing + bPendingTextureUpdate = true; + }); + + UE_LOG(LogMosisPhone, Log, TEXT("BeginPlay: Callbacks registered")); + } + else + { + UE_LOG(LogMosisPhone, Warning, TEXT("BeginPlay: MosisClient not available yet")); + } +#endif +} + +void UMosisPhoneComponent::EndPlay(const EEndPlayReason::Type EndPlayReason) +{ + UE_LOG(LogMosisPhone, Log, TEXT("EndPlay")); + +#if PLATFORM_ANDROID + // Clean up Vulkan texture + if (VulkanTexture) + { + VulkanTexture.Reset(); + } + + // Clear callbacks + auto Client = FMosisSDKModule::GetClient(); + if (Client) + { + Client->SetBufferCallback(nullptr); + Client->SetFrameCallback(nullptr); + } +#endif + + Super::EndPlay(EndPlayReason); +} + +void UMosisPhoneComponent::TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) +{ + Super::TickComponent(DeltaTime, TickType, ThisTickFunction); + +#if PLATFORM_ANDROID + // Check if we need to create texture from new buffer + if (bNeedsTextureCreate) + { + bNeedsTextureCreate = false; + OnBufferAvailable(); + } + + // Check if we need to update texture for new frame + if (bPendingTextureUpdate && VulkanTexture && VulkanTexture->IsValid()) + { + bPendingTextureUpdate = false; + OnFrameAvailable(); + } +#endif +} + +bool UMosisPhoneComponent::IsConnected() const +{ +#if PLATFORM_ANDROID + auto Client = FMosisSDKModule::GetClient(); + return Client && Client->IsInitialized() && Client->GetHardwareBuffer() != nullptr; +#else + return false; +#endif +} + +void UMosisPhoneComponent::SendTouch(FVector2D NormalizedUV, EMosisTouchType TouchType) +{ +#if PLATFORM_ANDROID + auto Client = FMosisSDKModule::GetClient(); + if (!Client) + { + return; + } + + float X = FMath::Clamp(NormalizedUV.X, 0.0f, 1.0f); + float Y = FMath::Clamp(NormalizedUV.Y, 0.0f, 1.0f); + + switch (TouchType) + { + case EMosisTouchType::Down: + Client->SendTouchDown(X, Y); + break; + case EMosisTouchType::Move: + Client->SendTouchMove(X, Y); + break; + case EMosisTouchType::Up: + Client->SendTouchUp(X, Y); + break; + } +#endif +} + +void UMosisPhoneComponent::OnBufferAvailable() +{ +#if PLATFORM_ANDROID + auto Client = FMosisSDKModule::GetClient(); + if (!Client) + { + return; + } + + AHardwareBuffer* Buffer = Client->GetHardwareBuffer(); + if (!Buffer) + { + UE_LOG(LogMosisPhone, Warning, TEXT("OnBufferAvailable: No buffer")); + return; + } + + // Get buffer dimensions + AHardwareBuffer_Desc Desc{}; + AHardwareBuffer_describe(Buffer, &Desc); + ScreenWidth = Desc.width; + ScreenHeight = Desc.height; + + UE_LOG(LogMosisPhone, Log, TEXT("OnBufferAvailable: %dx%d"), ScreenWidth, ScreenHeight); + + // Initialize Vulkan texture on render thread + ENQUEUE_RENDER_COMMAND(MosisCreateTexture)( + [this, Buffer](FRHICommandListImmediate& RHICmdList) + { + // Get Vulkan device from RHI + IVulkanDynamicRHI* VulkanRHI = GetIVulkanDynamicRHI(); + if (!VulkanRHI) + { + UE_LOG(LogMosisPhone, Error, TEXT("OnBufferAvailable: Vulkan RHI not available")); + return; + } + + VkInstance Instance = VulkanRHI->RHIGetVkInstance(); + VkPhysicalDevice PhysDevice = VulkanRHI->RHIGetVkPhysicalDevice(); + VkDevice Device = VulkanRHI->RHIGetVkDevice(); + + // Create Vulkan texture if needed + if (!VulkanTexture) + { + VulkanTexture = MakeShared(); + if (!VulkanTexture->Initialize(Instance, PhysDevice, Device, 0)) + { + UE_LOG(LogMosisPhone, Error, TEXT("OnBufferAvailable: Failed to initialize VulkanTexture")); + VulkanTexture.Reset(); + return; + } + } + + // Create texture from buffer + if (!VulkanTexture->Create(Buffer)) + { + UE_LOG(LogMosisPhone, Error, TEXT("OnBufferAvailable: Failed to create texture")); + return; + } + + UE_LOG(LogMosisPhone, Log, TEXT("OnBufferAvailable: Vulkan texture created")); + } + ); + + // Create UE texture on game thread + AsyncTask(ENamedThreads::GameThread, [this]() + { + // Create UTexture2D from Vulkan image + // For now, create a placeholder - actual Vulkan integration would use + // FVulkanTexture2D or similar UE-Vulkan interop + if (!PhoneTexture) + { + PhoneTexture = UTexture2D::CreateTransient(ScreenWidth, ScreenHeight, PF_R8G8B8A8); + PhoneTexture->UpdateResource(); + } + + // Update material if set + if (PhoneMaterial) + { + PhoneMaterial->SetTextureParameterValue(TextureParameterName, PhoneTexture); + } + }); +#endif +} + +void UMosisPhoneComponent::OnFrameAvailable() +{ +#if PLATFORM_ANDROID + if (!VulkanTexture || !VulkanTexture->IsValid()) + { + return; + } + + // Copy texture on render thread + ENQUEUE_RENDER_COMMAND(MosisUpdateTexture)( + [this](FRHICommandListImmediate& RHICmdList) + { + IVulkanDynamicRHI* VulkanRHI = GetIVulkanDynamicRHI(); + if (!VulkanRHI || !VulkanTexture) + { + return; + } + + // Get command buffer from UE's Vulkan RHI + // This requires more integration with UE's command buffer management + // For now, the texture copy happens during Create() which does an initial copy + + // In a full implementation, you would: + // 1. Get UE's current command buffer + // 2. Call VulkanTexture->CopyToLocal(cmdBuffer) + // 3. Submit as part of UE's rendering + } + ); +#endif +} + +void UMosisPhoneComponent::UpdateTextureOnRenderThread() +{ + // Reserved for render thread texture updates +} diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisSDK.cpp b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisSDK.cpp index 6aa1f95..c5692af 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisSDK.cpp +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisSDK.cpp @@ -4,13 +4,21 @@ #if PLATFORM_ANDROID +#include "MosisClient.h" #include "Android/AndroidApplication.h" #include "Android/AndroidJNI.h" #include "Android/AndroidJava.h" #include +DEFINE_LOG_CATEGORY_STATIC(LogMosisSDK, Log, All); + +// Global client instance +static std::shared_ptr g_MosisClient; + void FMosisSDKModule::StartupModule() { + UE_LOG(LogMosisSDK, Log, TEXT("StartupModule")); + if (JNIEnv* Env = FAndroidApplication::GetJavaEnv()) { jclass ManagerClass = FAndroidApplication::FindJavaClass("com/omixlab/mosis/MyKotlinPlugin"); @@ -18,29 +26,86 @@ void FMosisSDKModule::StartupModule() { jmethodID BindMethod = Env->GetStaticMethodID(ManagerClass, "StartMosisService", "()V"); Env->CallStaticVoidMethod(ManagerClass, BindMethod); - UE_LOG(LogTemp, Log, TEXT("Requested Bind to Android Service...")); + UE_LOG(LogMosisSDK, Log, TEXT("Requested Bind to Android Service...")); } + else + { + UE_LOG(LogMosisSDK, Error, TEXT("Failed to find MyKotlinPlugin class")); + } + } + else + { + UE_LOG(LogMosisSDK, Error, TEXT("Failed to get JNI environment")); } } void FMosisSDKModule::ShutdownModule() { + UE_LOG(LogMosisSDK, Log, TEXT("ShutdownModule")); + + // Release the client + g_MosisClient.reset(); +} + +// Accessor for the global client +std::shared_ptr FMosisSDKModule::GetClient() +{ + return g_MosisClient; } extern "C" JNIEXPORT void JNICALL -Java_com_omixlab_mosis_MyKotlinPlugin_serviceConnected(JNIEnv* env, jobject thiz, - jobject binder) +Java_com_omixlab_mosis_MyKotlinPlugin_serviceConnected(JNIEnv* env, jobject thiz, jobject binder) { + UE_LOG(LogMosisSDK, Log, TEXT("serviceConnected callback received")); + AIBinder* pBinder = AIBinder_fromJavaBinder(env, binder); - //const ndk::SpAIBinder spBinder(pBinder); - //g_service = IMosisService::fromBinder(spBinder); - //Logger::Log("Service Connected"); - //g_context = ndk::SharedRefBase::make(); - //bool result{}; - //g_service->initOS(g_context, &result); - //Logger::Log(std::format("InitOS returned {}", result)); + if (!pBinder) + { + UE_LOG(LogMosisSDK, Error, TEXT("serviceConnected: Failed to convert Java binder to AIBinder")); + return; + } + + // Create the client + g_MosisClient = MosisClient::Create(pBinder); + + if (g_MosisClient) + { + UE_LOG(LogMosisSDK, Log, TEXT("serviceConnected: MosisClient created successfully")); + + // Set up callbacks for texture handling + g_MosisClient->SetBufferCallback([](AHardwareBuffer* buffer) { + UE_LOG(LogMosisSDK, Log, TEXT("Buffer callback: buffer=%p"), buffer); + // Vulkan texture import will be triggered here + }); + + g_MosisClient->SetFrameCallback([]() { + // Frame available - texture update needed + }); + + g_MosisClient->SetInitCallback([](bool success) { + UE_LOG(LogMosisSDK, Log, TEXT("Init callback: success=%s"), success ? TEXT("true") : TEXT("false")); + }); + } + else + { + UE_LOG(LogMosisSDK, Error, TEXT("serviceConnected: Failed to create MosisClient")); + } +} + +#else + +// Non-Android stub implementations +void FMosisSDKModule::StartupModule() +{ + // No-op on non-Android platforms +} + +void FMosisSDKModule::ShutdownModule() +{ + // No-op on non-Android platforms } #endif + IMPLEMENT_MODULE(FMosisSDKModule, MosisSDK) diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisVulkanTexture.cpp b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisVulkanTexture.cpp new file mode 100644 index 0000000..75d32ec --- /dev/null +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisVulkanTexture.cpp @@ -0,0 +1,421 @@ +#include "MosisVulkanTexture.h" + +#if PLATFORM_ANDROID + +#include "Logging/LogMacros.h" + +DEFINE_LOG_CATEGORY_STATIC(LogMosisVulkan, Log, All); + +MosisVulkanTexture::~MosisVulkanTexture() +{ + Destroy(); +} + +bool MosisVulkanTexture::Initialize(VkInstance instance, VkPhysicalDevice physDevice, VkDevice device, uint32_t queueFamilyIndex) +{ + if (m_Initialized) + { + return true; + } + + m_Instance = instance; + m_PhysicalDevice = physDevice; + m_Device = device; + m_QueueFamilyIndex = queueFamilyIndex; + + // Load extension function + vkGetAndroidHardwareBufferPropertiesANDROID = + reinterpret_cast( + vkGetInstanceProcAddr(m_Instance, "vkGetAndroidHardwareBufferPropertiesANDROID") + ); + + if (!vkGetAndroidHardwareBufferPropertiesANDROID) + { + UE_LOG(LogMosisVulkan, Error, TEXT("Initialize: vkGetAndroidHardwareBufferPropertiesANDROID not available")); + return false; + } + + m_Initialized = true; + UE_LOG(LogMosisVulkan, Log, TEXT("Initialize: Success")); + return true; +} + +bool MosisVulkanTexture::Create(AHardwareBuffer* buffer) +{ + if (!m_Initialized) + { + UE_LOG(LogMosisVulkan, Error, TEXT("Create: Not initialized")); + return false; + } + + if (m_Created) + { + Destroy(); + } + + // Get buffer dimensions + AHardwareBuffer_Desc desc{}; + AHardwareBuffer_describe(buffer, &desc); + m_Width = desc.width; + m_Height = desc.height; + + UE_LOG(LogMosisVulkan, Log, TEXT("Create: %dx%d texture"), m_Width, m_Height); + + if (!ImportHardwareBuffer(buffer)) + { + UE_LOG(LogMosisVulkan, Error, TEXT("Create: Failed to import hardware buffer")); + return false; + } + + if (!CreateLocalImage()) + { + UE_LOG(LogMosisVulkan, Error, TEXT("Create: Failed to create local image")); + DestroyImportedResources(); + return false; + } + + m_Created = true; + UE_LOG(LogMosisVulkan, Log, TEXT("Create: Success")); + return true; +} + +bool MosisVulkanTexture::ImportHardwareBuffer(AHardwareBuffer* buffer) +{ + // Query hardware buffer properties + VkAndroidHardwareBufferFormatPropertiesANDROID formatProps{}; + formatProps.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_FORMAT_PROPERTIES_ANDROID; + + VkAndroidHardwareBufferPropertiesANDROID props{}; + props.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID; + props.pNext = &formatProps; + + if (vkGetAndroidHardwareBufferPropertiesANDROID(m_Device, buffer, &props) != VK_SUCCESS) + { + UE_LOG(LogMosisVulkan, Error, TEXT("ImportHardwareBuffer: Failed to get properties")); + return false; + } + + m_Format = formatProps.format; + UE_LOG(LogMosisVulkan, Log, TEXT("ImportHardwareBuffer: Format=%d"), static_cast(m_Format)); + + // Create VkImage with external memory + VkExternalMemoryImageCreateInfo extMemImageInfo{}; + extMemImageInfo.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO; + extMemImageInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID; + + VkImageCreateInfo imageInfo{}; + imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + imageInfo.pNext = &extMemImageInfo; + imageInfo.imageType = VK_IMAGE_TYPE_2D; + imageInfo.format = m_Format; + imageInfo.extent = {m_Width, m_Height, 1}; + imageInfo.mipLevels = 1; + imageInfo.arrayLayers = 1; + imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imageInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + + if (vkCreateImage(m_Device, &imageInfo, nullptr, &m_ImportedImage) != VK_SUCCESS) + { + UE_LOG(LogMosisVulkan, Error, TEXT("ImportHardwareBuffer: Failed to create image")); + return false; + } + + // Import memory from hardware buffer + VkImportAndroidHardwareBufferInfoANDROID importInfo{}; + importInfo.sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID; + importInfo.buffer = buffer; + + VkMemoryDedicatedAllocateInfo dedicatedInfo{}; + dedicatedInfo.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO; + dedicatedInfo.pNext = &importInfo; + dedicatedInfo.image = m_ImportedImage; + + VkMemoryAllocateInfo allocInfo{}; + allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocInfo.pNext = &dedicatedInfo; + allocInfo.allocationSize = props.allocationSize; + allocInfo.memoryTypeIndex = FindMemoryType(props.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + + if (vkAllocateMemory(m_Device, &allocInfo, nullptr, &m_ImportedMemory) != VK_SUCCESS) + { + UE_LOG(LogMosisVulkan, Error, TEXT("ImportHardwareBuffer: Failed to allocate memory")); + vkDestroyImage(m_Device, m_ImportedImage, nullptr); + m_ImportedImage = VK_NULL_HANDLE; + return false; + } + + if (vkBindImageMemory(m_Device, m_ImportedImage, m_ImportedMemory, 0) != VK_SUCCESS) + { + UE_LOG(LogMosisVulkan, Error, TEXT("ImportHardwareBuffer: Failed to bind memory")); + DestroyImportedResources(); + return false; + } + + UE_LOG(LogMosisVulkan, Log, TEXT("ImportHardwareBuffer: Success")); + return true; +} + +bool MosisVulkanTexture::CreateLocalImage() +{ + // Create local VkImage + VkImageCreateInfo imageInfo{}; + imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + imageInfo.imageType = VK_IMAGE_TYPE_2D; + imageInfo.format = m_Format; + imageInfo.extent = {m_Width, m_Height, 1}; + imageInfo.mipLevels = 1; + imageInfo.arrayLayers = 1; + imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imageInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; + imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + + if (vkCreateImage(m_Device, &imageInfo, nullptr, &m_LocalImage) != VK_SUCCESS) + { + UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to create image")); + return false; + } + + // Allocate memory + VkMemoryRequirements memReqs; + vkGetImageMemoryRequirements(m_Device, m_LocalImage, &memReqs); + + VkMemoryAllocateInfo allocInfo{}; + allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocInfo.allocationSize = memReqs.size; + allocInfo.memoryTypeIndex = FindMemoryType(memReqs.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + + if (vkAllocateMemory(m_Device, &allocInfo, nullptr, &m_LocalMemory) != VK_SUCCESS) + { + UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to allocate memory")); + vkDestroyImage(m_Device, m_LocalImage, nullptr); + m_LocalImage = VK_NULL_HANDLE; + return false; + } + + if (vkBindImageMemory(m_Device, m_LocalImage, m_LocalMemory, 0) != VK_SUCCESS) + { + UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to bind memory")); + DestroyLocalResources(); + return false; + } + + // Create image view + VkImageViewCreateInfo viewInfo{}; + viewInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + viewInfo.image = m_LocalImage; + viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + viewInfo.format = m_Format; + viewInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + viewInfo.subresourceRange.baseMipLevel = 0; + viewInfo.subresourceRange.levelCount = 1; + viewInfo.subresourceRange.baseArrayLayer = 0; + viewInfo.subresourceRange.layerCount = 1; + + if (vkCreateImageView(m_Device, &viewInfo, nullptr, &m_LocalImageView) != VK_SUCCESS) + { + UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to create image view")); + DestroyLocalResources(); + return false; + } + + // Create sampler + VkSamplerCreateInfo samplerInfo{}; + samplerInfo.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO; + samplerInfo.magFilter = VK_FILTER_LINEAR; + samplerInfo.minFilter = VK_FILTER_LINEAR; + samplerInfo.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + samplerInfo.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + samplerInfo.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + samplerInfo.anisotropyEnable = VK_FALSE; + samplerInfo.maxAnisotropy = 1.0f; + samplerInfo.borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK; + samplerInfo.unnormalizedCoordinates = VK_FALSE; + samplerInfo.compareEnable = VK_FALSE; + samplerInfo.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR; + + if (vkCreateSampler(m_Device, &samplerInfo, nullptr, &m_Sampler) != VK_SUCCESS) + { + UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to create sampler")); + DestroyLocalResources(); + return false; + } + + UE_LOG(LogMosisVulkan, Log, TEXT("CreateLocalImage: Success")); + return true; +} + +void MosisVulkanTexture::CopyToLocal(VkCommandBuffer cmd) +{ + if (!m_Created) + { + return; + } + + // Transition imported image to transfer src + VkImageMemoryBarrier srcBarrier{}; + srcBarrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + srcBarrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + srcBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + srcBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + srcBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + srcBarrier.image = m_ImportedImage; + srcBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + srcBarrier.subresourceRange.baseMipLevel = 0; + srcBarrier.subresourceRange.levelCount = 1; + srcBarrier.subresourceRange.baseArrayLayer = 0; + srcBarrier.subresourceRange.layerCount = 1; + srcBarrier.srcAccessMask = 0; + srcBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + + vkCmdPipelineBarrier( + cmd, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 0, nullptr, 1, &srcBarrier + ); + + // Transition local image to transfer dst + VkImageMemoryBarrier dstBarrier{}; + dstBarrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + dstBarrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + dstBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + dstBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + dstBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + dstBarrier.image = m_LocalImage; + dstBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + dstBarrier.subresourceRange.baseMipLevel = 0; + dstBarrier.subresourceRange.levelCount = 1; + dstBarrier.subresourceRange.baseArrayLayer = 0; + dstBarrier.subresourceRange.layerCount = 1; + dstBarrier.srcAccessMask = 0; + dstBarrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + + vkCmdPipelineBarrier( + cmd, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 0, nullptr, 1, &dstBarrier + ); + + // Copy image + VkImageCopy copyRegion{}; + copyRegion.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + copyRegion.srcSubresource.layerCount = 1; + copyRegion.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + copyRegion.dstSubresource.layerCount = 1; + copyRegion.extent = {m_Width, m_Height, 1}; + + vkCmdCopyImage( + cmd, + m_ImportedImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + m_LocalImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + 1, ©Region + ); + + // Transition local image to shader read + VkImageMemoryBarrier shaderBarrier{}; + shaderBarrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + shaderBarrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + shaderBarrier.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + shaderBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + shaderBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + shaderBarrier.image = m_LocalImage; + shaderBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + shaderBarrier.subresourceRange.baseMipLevel = 0; + shaderBarrier.subresourceRange.levelCount = 1; + shaderBarrier.subresourceRange.baseArrayLayer = 0; + shaderBarrier.subresourceRange.layerCount = 1; + shaderBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + shaderBarrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + + vkCmdPipelineBarrier( + cmd, + VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + 0, 0, nullptr, 0, nullptr, 1, &shaderBarrier + ); +} + +uint32_t MosisVulkanTexture::FindMemoryType(uint32_t typeFilter, VkMemoryPropertyFlags properties) +{ + VkPhysicalDeviceMemoryProperties memProps; + vkGetPhysicalDeviceMemoryProperties(m_PhysicalDevice, &memProps); + + for (uint32_t i = 0; i < memProps.memoryTypeCount; i++) + { + if ((typeFilter & (1 << i)) && + (memProps.memoryTypes[i].propertyFlags & properties) == properties) + { + return i; + } + } + + UE_LOG(LogMosisVulkan, Error, TEXT("FindMemoryType: Failed to find suitable memory type")); + return 0; +} + +void MosisVulkanTexture::Destroy() +{ + if (m_Device && m_Created) + { + vkDeviceWaitIdle(m_Device); + } + + DestroyLocalResources(); + DestroyImportedResources(); + + m_Width = 0; + m_Height = 0; + m_Created = false; +} + +void MosisVulkanTexture::DestroyImportedResources() +{ + if (m_Device) + { + if (m_ImportedMemory != VK_NULL_HANDLE) + { + vkFreeMemory(m_Device, m_ImportedMemory, nullptr); + m_ImportedMemory = VK_NULL_HANDLE; + } + if (m_ImportedImage != VK_NULL_HANDLE) + { + vkDestroyImage(m_Device, m_ImportedImage, nullptr); + m_ImportedImage = VK_NULL_HANDLE; + } + } +} + +void MosisVulkanTexture::DestroyLocalResources() +{ + if (m_Device) + { + if (m_Sampler != VK_NULL_HANDLE) + { + vkDestroySampler(m_Device, m_Sampler, nullptr); + m_Sampler = VK_NULL_HANDLE; + } + if (m_LocalImageView != VK_NULL_HANDLE) + { + vkDestroyImageView(m_Device, m_LocalImageView, nullptr); + m_LocalImageView = VK_NULL_HANDLE; + } + if (m_LocalMemory != VK_NULL_HANDLE) + { + vkFreeMemory(m_Device, m_LocalMemory, nullptr); + m_LocalMemory = VK_NULL_HANDLE; + } + if (m_LocalImage != VK_NULL_HANDLE) + { + vkDestroyImage(m_Device, m_LocalImage, nullptr); + m_LocalImage = VK_NULL_HANDLE; + } + } +} + +#endif // PLATFORM_ANDROID diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisVulkanTexture.h b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisVulkanTexture.h new file mode 100644 index 0000000..7bc06ff --- /dev/null +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisVulkanTexture.h @@ -0,0 +1,87 @@ +#pragma once + +#if PLATFORM_ANDROID + +#include "CoreMinimal.h" +#include +#include +#include + +/** + * MosisVulkanTexture - Imports AHardwareBuffer as Vulkan texture for Unreal. + * Creates both imported and local copy images for safe rendering. + */ +class MosisVulkanTexture +{ +public: + MosisVulkanTexture() = default; + ~MosisVulkanTexture(); + + /** + * Initialize with Vulkan device objects. + * Must be called before Create(). + */ + bool Initialize(VkInstance instance, VkPhysicalDevice physDevice, VkDevice device, uint32_t queueFamilyIndex); + + /** + * Create texture from a hardware buffer. + * @param buffer The AHardwareBuffer from the Mosis service + * @return true if creation succeeded + */ + bool Create(AHardwareBuffer* buffer); + + /** + * Copy from imported image to local image. + * Call this each frame when a new frame is available. + */ + void CopyToLocal(VkCommandBuffer cmd); + + /** + * Destroy all resources. + */ + void Destroy(); + + // Accessors + VkImage GetLocalImage() const { return m_LocalImage; } + VkImageView GetLocalImageView() const { return m_LocalImageView; } + VkFormat GetFormat() const { return m_Format; } + uint32_t GetWidth() const { return m_Width; } + uint32_t GetHeight() const { return m_Height; } + bool IsValid() const { return m_Created; } + +private: + bool ImportHardwareBuffer(AHardwareBuffer* buffer); + bool CreateLocalImage(); + void DestroyImportedResources(); + void DestroyLocalResources(); + uint32_t FindMemoryType(uint32_t typeFilter, VkMemoryPropertyFlags properties); + + // Vulkan objects + VkInstance m_Instance = VK_NULL_HANDLE; + VkPhysicalDevice m_PhysicalDevice = VK_NULL_HANDLE; + VkDevice m_Device = VK_NULL_HANDLE; + uint32_t m_QueueFamilyIndex = 0; + + // Imported from HardwareBuffer + VkImage m_ImportedImage = VK_NULL_HANDLE; + VkDeviceMemory m_ImportedMemory = VK_NULL_HANDLE; + + // Local copy for safe rendering + VkImage m_LocalImage = VK_NULL_HANDLE; + VkDeviceMemory m_LocalMemory = VK_NULL_HANDLE; + VkImageView m_LocalImageView = VK_NULL_HANDLE; + VkSampler m_Sampler = VK_NULL_HANDLE; + + // Format info + VkFormat m_Format = VK_FORMAT_R8G8B8A8_UNORM; + uint32_t m_Width = 0; + uint32_t m_Height = 0; + + bool m_Initialized = false; + bool m_Created = false; + + // Extension function pointer + PFN_vkGetAndroidHardwareBufferPropertiesANDROID vkGetAndroidHardwareBufferPropertiesANDROID = nullptr; +}; + +#endif // PLATFORM_ANDROID diff --git a/Plugins/MosisSDK/Source/MosisSDK/Public/MosisPhoneActor.h b/Plugins/MosisSDK/Source/MosisSDK/Public/MosisPhoneActor.h new file mode 100644 index 0000000..50c5cc6 --- /dev/null +++ b/Plugins/MosisSDK/Source/MosisSDK/Public/MosisPhoneActor.h @@ -0,0 +1,72 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "GameFramework/Actor.h" +#include "MosisPhoneComponent.h" +#include "MosisPhoneActor.generated.h" + +class UStaticMeshComponent; +class UMaterialInstanceDynamic; + +/** + * AMosisPhoneActor - Pre-configured actor for displaying the Mosis phone. + * Contains mesh, material, and phone component ready for use. + */ +UCLASS(BlueprintType, Blueprintable) +class MOSISSDK_API AMosisPhoneActor : public AActor +{ + GENERATED_BODY() + +public: + AMosisPhoneActor(); + + virtual void BeginPlay() override; + + /** + * Send a touch event to the phone at world coordinates. + * Converts the hit location to UV coordinates automatically. + * @param HitLocation World location of the touch + * @param TouchType Type of touch event + * @return True if the touch was within phone bounds + */ + UFUNCTION(BlueprintCallable, Category = "Mosis") + bool SendTouchAtWorldLocation(FVector HitLocation, EMosisTouchType TouchType); + + /** + * Get the phone mesh component. + */ + UFUNCTION(BlueprintCallable, Category = "Mosis") + UStaticMeshComponent* GetPhoneMesh() const { return PhoneMesh; } + + /** + * Get the phone component. + */ + UFUNCTION(BlueprintCallable, Category = "Mosis") + UMosisPhoneComponent* GetPhoneComponent() const { return PhoneComponent; } + +protected: + /** Static mesh for the phone body */ + UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Mosis") + UStaticMeshComponent* PhoneMesh; + + /** Phone component for Mosis integration */ + UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Mosis") + UMosisPhoneComponent* PhoneComponent; + + /** Dynamic material instance for the screen */ + UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Mosis") + UMaterialInstanceDynamic* ScreenMaterial; + + /** + * Screen bounds in local space for UV mapping. + * X,Y = Min corner, Z,W = Max corner + */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis") + FVector4 ScreenBoundsLocal = FVector4(-50.0f, -100.0f, 50.0f, 100.0f); + +private: + /** Convert world hit to UV coordinates */ + bool WorldToPhoneUV(FVector WorldLocation, FVector2D& OutUV) const; +}; diff --git a/Plugins/MosisSDK/Source/MosisSDK/Public/MosisPhoneComponent.h b/Plugins/MosisSDK/Source/MosisSDK/Public/MosisPhoneComponent.h new file mode 100644 index 0000000..e640a7f --- /dev/null +++ b/Plugins/MosisSDK/Source/MosisSDK/Public/MosisPhoneComponent.h @@ -0,0 +1,109 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Components/ActorComponent.h" +#include "MosisPhoneComponent.generated.h" + +class UMaterialInstanceDynamic; +class UTexture2D; + +/** + * Touch event type for phone interactions. + */ +UENUM(BlueprintType) +enum class EMosisTouchType : uint8 +{ + Down, + Move, + Up +}; + +/** + * UMosisPhoneComponent - Component for displaying Mosis phone screen in UE5. + * Handles texture updates from the service and touch input forwarding. + */ +UCLASS(ClassGroup=(Mosis), meta=(BlueprintSpawnableComponent)) +class MOSISSDK_API UMosisPhoneComponent : public UActorComponent +{ + GENERATED_BODY() + +public: + UMosisPhoneComponent(); + + // UActorComponent interface + virtual void BeginPlay() override; + virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override; + virtual void TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override; + + /** + * Send a touch event to the phone. + * @param NormalizedUV Touch position in UV coordinates (0-1) + * @param TouchType Type of touch event (Down, Move, Up) + */ + UFUNCTION(BlueprintCallable, Category = "Mosis") + void SendTouch(FVector2D NormalizedUV, EMosisTouchType TouchType); + + /** + * Get the phone texture for rendering. + * @return The texture showing the phone screen, or nullptr if not ready + */ + UFUNCTION(BlueprintCallable, Category = "Mosis") + UTexture2D* GetPhoneTexture() const { return PhoneTexture; } + + /** + * Check if the phone is connected and ready. + */ + UFUNCTION(BlueprintCallable, Category = "Mosis") + bool IsConnected() const; + + /** + * Get the phone screen dimensions. + */ + UFUNCTION(BlueprintCallable, Category = "Mosis") + FIntPoint GetScreenSize() const { return FIntPoint(ScreenWidth, ScreenHeight); } + + /** + * Material instance to apply the phone texture to. + * Set this to the material on your phone mesh. + */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis") + UMaterialInstanceDynamic* PhoneMaterial; + + /** + * Parameter name in the material for the phone texture. + */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis") + FName TextureParameterName = TEXT("PhoneScreen"); + +protected: + /** Called when a new hardware buffer is received from the service. */ + void OnBufferAvailable(); + + /** Called when a new frame is available for display. */ + void OnFrameAvailable(); + + /** Update the texture on the render thread. */ + void UpdateTextureOnRenderThread(); + +private: + /** Phone screen texture */ + UPROPERTY() + UTexture2D* PhoneTexture; + + /** Screen dimensions from service */ + int32 ScreenWidth = 0; + int32 ScreenHeight = 0; + + /** Flag to track pending texture updates */ + bool bPendingTextureUpdate = false; + + /** Flag to track if we need to recreate the texture */ + bool bNeedsTextureCreate = false; + +#if PLATFORM_ANDROID + /** Vulkan texture wrapper */ + TSharedPtr VulkanTexture; +#endif +}; diff --git a/Plugins/MosisSDK/Source/MosisSDK/Public/MosisSDK.h b/Plugins/MosisSDK/Source/MosisSDK/Public/MosisSDK.h index 1bf380b..47df9a4 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Public/MosisSDK.h +++ b/Plugins/MosisSDK/Source/MosisSDK/Public/MosisSDK.h @@ -5,11 +5,22 @@ #include "CoreMinimal.h" #include "Modules/ModuleManager.h" -class FMosisSDKModule : public IModuleInterface +#if PLATFORM_ANDROID +class MosisClient; +#endif + +class MOSISSDK_API FMosisSDKModule : public IModuleInterface { public: - /** IModuleInterface implementation */ virtual void StartupModule() override; virtual void ShutdownModule() override; + +#if PLATFORM_ANDROID + /** + * Get the global MosisClient instance. + * @return Shared pointer to the client, or nullptr if not connected + */ + static std::shared_ptr GetClient(); +#endif };