diff --git a/Plugins/MosisSDK/README.md b/Plugins/MosisSDK/README.md index fe288bf..9e09fbc 100644 --- a/Plugins/MosisSDK/README.md +++ b/Plugins/MosisSDK/README.md @@ -12,24 +12,33 @@ The MosisSDK plugin connects to the MosisService Android application via AIDL (A ## Architecture ``` -┌─────────────────────────────────────────────────────────────┐ -│ Unreal Engine Game │ -│ ┌─────────────────────┐ ┌─────────────────────────────┐ │ -│ │ UMosisPhoneComponent│◄───│ MosisVulkanTexture │ │ -│ │ (Touch Input) │ │ (HardwareBuffer Import) │ │ -│ └──────────┬──────────┘ └──────────────▲──────────────┘ │ -│ │ │ │ -│ ▼ │ │ -│ ┌─────────────────────────────────────────┴───────────────┐│ -│ │ MosisClient ││ -│ │ (AIDL IMosisListener) ││ -│ └──────────────────────────┬──────────────────────────────┘│ -└─────────────────────────────┼───────────────────────────────┘ - │ Binder IPC -┌─────────────────────────────▼───────────────────────────────┐ -│ MosisService │ -│ (Renders phone UI via RmlUi) │ -└─────────────────────────────────────────────────────────────┘ +┌─────────────────────────────────────────────────────────────────────────┐ +│ Unreal Engine Game │ +│ │ +│ ┌─────────────────┐ ┌──────────────────────────────────────────┐ │ +│ │ AMosisPhoneActor│ │ UMosisPhoneTexture │ │ +│ │ (Plane Mesh) │ │ │ │ │ +│ │ (Material) │ │ ▼ │ │ +│ └────────┬────────┘ │ ┌────────────────────────────────────┐ │ │ +│ │ │ │ FMosisPhoneTextureResource │ │ │ +│ ▼ │ │ │ │ │ │ +│ ┌─────────────────────┐ │ │ ▼ ENQUEUE_RENDER_CMD │ │ │ +│ │ UMosisPhoneComponent│──┼──► IVulkanDynamicRHI:: │ │ │ +│ │ (Touch Input) │ │ │ RHICreateTexture2DFromAndroid │ │ │ +│ │ (Callbacks) │ │ │ HardwareBuffer() │ │ │ +│ └──────────┬──────────┘ │ └────────────────────────────────────┘ │ │ +│ │ └──────────────────────────────────────────┘ │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────┐ │ +│ │ MosisClient │ │ +│ │ (AIDL IMosisListener) │ │ +│ └──────────────────────────────┬───────────────────────────────────┘ │ +└─────────────────────────────────┼───────────────────────────────────────┘ + │ Binder IPC +┌─────────────────────────────────▼───────────────────────────────────────┐ +│ MosisService │ +│ (Renders phone UI via RmlUi) │ +└─────────────────────────────────────────────────────────────────────────┘ ``` ## Prerequisites @@ -73,12 +82,14 @@ Plugins/MosisSDK/ │ │ └── BpMosisService.h │ ├── Public/ │ │ ├── MosisSDK.h # Module interface -│ │ ├── MosisPhoneComponent.h # UE5 component -│ │ └── MosisPhoneActor.h # Blueprint actor +│ │ ├── MosisPhoneComponent.h # UE5 component (touch, callbacks) +│ │ └── MosisPhoneActor.h # Blueprint actor (mesh, material) │ └── Private/ │ ├── MosisSDK.cpp # Module + JNI callbacks │ ├── MosisClient.h/cpp # AIDL client implementation -│ ├── MosisVulkanTexture.h/cpp # Vulkan HardwareBuffer import +│ ├── MosisPhoneTexture.h/cpp # UTexture for phone screen +│ ├── MosisPhoneTextureResource.h/cpp # Render thread resource +│ ├── MosisVulkanTexture.h/cpp # Legacy Vulkan import (unused) │ ├── MosisPhoneComponent.cpp │ ├── MosisPhoneActor.cpp │ └── Android/ @@ -152,7 +163,8 @@ The separation ensures Windows builds don't try to compile Android-specific AIDL | Slate, SlateCore | ✓ | ✓ | UI framework | | RenderCore, RHI | ✓ | ✓ | Rendering abstraction | | Launch, ApplicationCore | ✗ | ✓ | Android JNI access | -| Vulkan | ✗ | ✓ | Hardware buffer import | +| VulkanRHI | ✗ | ✓ | IVulkanDynamicRHI for HardwareBuffer import | +| Vulkan | ✗ | ✓ | Vulkan headers | | binder_ndk, android, nativewindow | ✗ | ✓ | Android system libs | ### NDK Header Compatibility @@ -353,6 +365,13 @@ HandTrackingVersion=V2 ## Version History +- **v1.1** - UE5 RHI texture integration + - New `UMosisPhoneTexture` using UE5's `IVulkanDynamicRHI::RHICreateTexture2DFromAndroidHardwareBuffer()` + - `FMosisPhoneTextureResource` for render thread HardwareBuffer import + - `AMosisPhoneActor` now includes default plane mesh and material setup + - Thread-safe callbacks using `TWeakObjectPtr` and `TAtomic` + - Automatic texture-to-material binding in tick + - **v1.0** - Initial implementation - AIDL client for MosisService connection - Vulkan HardwareBuffer import diff --git a/Plugins/MosisSDK/Source/MosisSDK/MosisSDK.Build.cs b/Plugins/MosisSDK/Source/MosisSDK/MosisSDK.Build.cs index b8b18ad..5b4b763 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/MosisSDK.Build.cs +++ b/Plugins/MosisSDK/Source/MosisSDK/MosisSDK.Build.cs @@ -35,7 +35,8 @@ public class MosisSDK : ModuleRules // Android-specific module dependencies PrivateDependencyModuleNames.AddRange(new string[] { "Launch", - "ApplicationCore" + "ApplicationCore", + "VulkanRHI" }); // Add Vulkan support (Vulkan headers for HardwareBuffer import) diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneActor.cpp b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneActor.cpp index b079efe..7a310b6 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneActor.cpp +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneActor.cpp @@ -4,17 +4,27 @@ #include "Components/StaticMeshComponent.h" #include "Materials/MaterialInstanceDynamic.h" #include "Engine/StaticMesh.h" +#include "UObject/ConstructorHelpers.h" DEFINE_LOG_CATEGORY_STATIC(LogMosisPhoneActor, Log, All); AMosisPhoneActor::AMosisPhoneActor() { - PrimaryActorTick.bCanEverTick = false; + PrimaryActorTick.bCanEverTick = true; + PrimaryActorTick.bStartWithTickEnabled = true; // Create phone mesh component PhoneMesh = CreateDefaultSubobject(TEXT("PhoneMesh")); RootComponent = PhoneMesh; + // Load the default plane mesh + static ConstructorHelpers::FObjectFinder PlaneMeshFinder( + TEXT("/Engine/BasicShapes/Plane.Plane")); + if (PlaneMeshFinder.Succeeded()) + { + PhoneMesh->SetStaticMesh(PlaneMeshFinder.Object); + } + // Create phone component PhoneComponent = CreateDefaultSubobject(TEXT("PhoneComponent")); } @@ -25,13 +35,34 @@ void AMosisPhoneActor::BeginPlay() UE_LOG(LogMosisPhoneActor, Log, TEXT("BeginPlay")); + // Scale mesh to match phone screen size + // The default plane is 100x100 units, so scale accordingly + if (PhoneMesh) + { + float ScaleX = ScreenSizeWorld.X / 100.0f; + float ScaleY = ScreenSizeWorld.Y / 100.0f; + PhoneMesh->SetRelativeScale3D(FVector(ScaleX, ScaleY, 1.0f)); + + // Update screen bounds based on scaled size + float HalfWidth = ScreenSizeWorld.X / 2.0f; + float HalfHeight = ScreenSizeWorld.Y / 2.0f; + ScreenBoundsLocal = FVector4(-HalfWidth, -HalfHeight, HalfWidth, HalfHeight); + } + // Create dynamic material for the screen if (PhoneMesh) { - UMaterialInterface* BaseMaterial = PhoneMesh->GetMaterial(0); - if (BaseMaterial) + UMaterialInterface* MaterialToUse = BaseMaterial; + + // If no base material set, use the mesh's existing material + if (!MaterialToUse) { - ScreenMaterial = UMaterialInstanceDynamic::Create(BaseMaterial, this); + MaterialToUse = PhoneMesh->GetMaterial(0); + } + + if (MaterialToUse) + { + ScreenMaterial = UMaterialInstanceDynamic::Create(MaterialToUse, this); PhoneMesh->SetMaterial(0, ScreenMaterial); // Set it on the phone component @@ -42,6 +73,25 @@ void AMosisPhoneActor::BeginPlay() UE_LOG(LogMosisPhoneActor, Log, TEXT("BeginPlay: Created dynamic material")); } + else + { + UE_LOG(LogMosisPhoneActor, Warning, TEXT("BeginPlay: No base material available")); + } + } +} + +void AMosisPhoneActor::Tick(float DeltaSeconds) +{ + Super::Tick(DeltaSeconds); + + // Update material texture if phone component has a new texture + if (ScreenMaterial && PhoneComponent) + { + UTexture* PhoneTexture = PhoneComponent->GetPhoneTexture(); + if (PhoneTexture) + { + ScreenMaterial->SetTextureParameterValue(PhoneComponent->TextureParameterName, PhoneTexture); + } } } diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneComponent.cpp b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneComponent.cpp index 4a48fbe..4e8c0ef 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneComponent.cpp +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneComponent.cpp @@ -2,13 +2,12 @@ #include "MosisPhoneComponent.h" #include "MosisSDK.h" -#include "Engine/Texture2D.h" +#include "MosisPhoneTexture.h" #include "Materials/MaterialInstanceDynamic.h" #include "RenderingThread.h" #if PLATFORM_ANDROID #include "MosisClient.h" -#include "MosisVulkanTexture.h" #endif DEFINE_LOG_CATEGORY_STATIC(LogMosisPhone, Log, All); @@ -30,15 +29,22 @@ void UMosisPhoneComponent::BeginPlay() auto Client = FMosisSDKModule::GetClient(); if (Client) { - // Set up callbacks - Client->SetBufferCallback([this](AHardwareBuffer* buffer) { - // This runs on the binder thread - flag for main thread processing - bNeedsTextureCreate = true; + // Use weak pointer to prevent dangling references in callbacks + TWeakObjectPtr WeakThis(this); + + // Set up callbacks (these run on binder thread) + Client->SetBufferCallback([WeakThis](AHardwareBuffer* buffer) { + if (WeakThis.IsValid()) + { + WeakThis->bNeedsTextureCreate.Store(true); + } }); - Client->SetFrameCallback([this]() { - // This runs on the binder thread - flag for tick processing - bPendingTextureUpdate = true; + Client->SetFrameCallback([WeakThis]() { + if (WeakThis.IsValid()) + { + WeakThis->bPendingTextureUpdate.Store(true); + } }); UE_LOG(LogMosisPhone, Log, TEXT("BeginPlay: Callbacks registered")); @@ -55,12 +61,6 @@ void UMosisPhoneComponent::EndPlay(const EEndPlayReason::Type EndPlayReason) UE_LOG(LogMosisPhone, Log, TEXT("EndPlay")); #if PLATFORM_ANDROID - // Clean up Vulkan texture - if (VulkanTexture) - { - VulkanTexture.Reset(); - } - // Clear callbacks auto Client = FMosisSDKModule::GetClient(); if (Client) @@ -70,6 +70,9 @@ void UMosisPhoneComponent::EndPlay(const EEndPlayReason::Type EndPlayReason) } #endif + // Release texture + PhoneTexture = nullptr; + Super::EndPlay(EndPlayReason); } @@ -79,16 +82,14 @@ void UMosisPhoneComponent::TickComponent(float DeltaTime, ELevelTick TickType, F #if PLATFORM_ANDROID // Check if we need to create texture from new buffer - if (bNeedsTextureCreate) + if (bNeedsTextureCreate.Exchange(false)) { - bNeedsTextureCreate = false; OnBufferAvailable(); } // Check if we need to update texture for new frame - if (bPendingTextureUpdate && VulkanTexture && VulkanTexture->IsValid()) + if (bPendingTextureUpdate.Exchange(false) && PhoneTexture) { - bPendingTextureUpdate = false; OnFrameAvailable(); } #endif @@ -155,25 +156,17 @@ void UMosisPhoneComponent::OnBufferAvailable() UE_LOG(LogMosisPhone, Log, TEXT("OnBufferAvailable: %dx%d"), ScreenWidth, ScreenHeight); - // TODO: Initialize Vulkan texture on render thread - // This requires accessing UE5's Vulkan RHI which has platform-specific APIs. - // For now, we log the buffer availability and dimensions. - // Full implementation would: - // 1. Get VkDevice from GVulkanRHI or IVulkanDynamicRHI - // 2. Create MosisVulkanTexture and import the HardwareBuffer - // 3. Create UTexture2D wrapping the Vulkan image - - // Create placeholder texture + // Create phone texture if needed if (!PhoneTexture) { - PhoneTexture = UTexture2D::CreateTransient(ScreenWidth, ScreenHeight, PF_R8G8B8A8); - if (PhoneTexture) - { - PhoneTexture->UpdateResource(); - UE_LOG(LogMosisPhone, Log, TEXT("OnBufferAvailable: Created placeholder texture")); - } + PhoneTexture = NewObject(this); + PhoneTexture->Initialize(ScreenWidth, ScreenHeight); + UE_LOG(LogMosisPhone, Log, TEXT("OnBufferAvailable: Created phone texture")); } + // Import the hardware buffer + PhoneTexture->UpdateFromHardwareBuffer(Buffer); + // Update material if set if (PhoneMaterial && PhoneTexture) { @@ -185,19 +178,23 @@ void UMosisPhoneComponent::OnBufferAvailable() void UMosisPhoneComponent::OnFrameAvailable() { #if PLATFORM_ANDROID - if (!VulkanTexture || !VulkanTexture->IsValid()) + if (!PhoneTexture) { return; } - // TODO: Copy texture on render thread - // This requires accessing UE5's Vulkan command buffer management. - // Full implementation would enqueue a render command to copy the - // imported image to the local image. + // Notify texture that a new frame is available + // This schedules the render thread copy + PhoneTexture->NotifyFrameAvailable(); #endif } void UMosisPhoneComponent::UpdateTextureOnRenderThread() { - // Reserved for render thread texture updates + // No longer needed - handled by UMosisPhoneTexture +} + +UTexture* UMosisPhoneComponent::GetPhoneTexture() const +{ + return PhoneTexture; } diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTexture.cpp b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTexture.cpp new file mode 100644 index 0000000..2e36844 --- /dev/null +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTexture.cpp @@ -0,0 +1,94 @@ +// Copyright OmixLab LTD. All Rights Reserved. + +#include "MosisPhoneTexture.h" +#include "MosisPhoneTextureResource.h" +#include "RenderingThread.h" + +DEFINE_LOG_CATEGORY_STATIC(LogMosisPhoneTexture, Log, All); + +UMosisPhoneTexture::UMosisPhoneTexture() +{ + // Default initialization + SRGB = true; + NeverStream = true; + LODGroup = TEXTUREGROUP_UI; +} + +void UMosisPhoneTexture::Initialize(uint32 InWidth, uint32 InHeight) +{ + Width = InWidth; + Height = InHeight; + + UE_LOG(LogMosisPhoneTexture, Log, TEXT("Initialize: %dx%d"), Width, Height); + + // Create the resource + UpdateResource(); +} + +FTextureResource* UMosisPhoneTexture::CreateResource() +{ + UE_LOG(LogMosisPhoneTexture, Log, TEXT("CreateResource: %dx%d"), Width, Height); + + PhoneTextureResource = new FMosisPhoneTextureResource(Width, Height); + return PhoneTextureResource; +} + +bool UMosisPhoneTexture::IsReady() const +{ + return PhoneTextureResource && PhoneTextureResource->IsReady(); +} + +#if PLATFORM_ANDROID + +void UMosisPhoneTexture::UpdateFromHardwareBuffer(AHardwareBuffer* Buffer) +{ + if (!Buffer) + { + UE_LOG(LogMosisPhoneTexture, Warning, TEXT("UpdateFromHardwareBuffer: null buffer")); + return; + } + + if (!PhoneTextureResource) + { + UE_LOG(LogMosisPhoneTexture, Warning, TEXT("UpdateFromHardwareBuffer: resource not initialized")); + return; + } + + // Store buffer for render thread + PendingBuffer.Store(Buffer); + + // Capture resource pointer for lambda + FMosisPhoneTextureResource* Resource = PhoneTextureResource; + AHardwareBuffer* BufferToImport = Buffer; + + // Schedule render thread work + ENQUEUE_RENDER_COMMAND(MosisImportHardwareBuffer)( + [Resource, BufferToImport](FRHICommandListImmediate& RHICmdList) + { + Resource->UpdateFromHardwareBuffer_RenderThread(BufferToImport); + } + ); + + UE_LOG(LogMosisPhoneTexture, Log, TEXT("UpdateFromHardwareBuffer: Scheduled import")); +} + +void UMosisPhoneTexture::NotifyFrameAvailable() +{ + if (!PhoneTextureResource) + { + return; + } + + // Capture resource pointer for lambda + FMosisPhoneTextureResource* Resource = PhoneTextureResource; + + // Schedule render thread copy + ENQUEUE_RENDER_COMMAND(MosisCopyFrame)( + [Resource](FRHICommandListImmediate& RHICmdList) + { + Resource->CopySourceToLocal_RenderThread(RHICmdList); + } + ); +} + +#endif // PLATFORM_ANDROID diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTexture.h b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTexture.h new file mode 100644 index 0000000..981c201 --- /dev/null +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTexture.h @@ -0,0 +1,84 @@ +// Copyright OmixLab LTD. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Engine/Texture.h" +#include "MosisPhoneTexture.generated.h" + +#if PLATFORM_ANDROID +#include +#endif + +class FMosisPhoneTextureResource; + +/** + * UMosisPhoneTexture - Custom texture that displays the Mosis phone screen. + * + * This texture is updated from an AHardwareBuffer received from MosisService. + * Uses UE5's IVulkanDynamicRHI for efficient hardware buffer import. + * + * Usage: + * 1. Create via NewObject() with width/height + * 2. Call UpdateResource() to initialize + * 3. Call UpdateFromHardwareBuffer() when buffer is received + * 4. Call NotifyFrameAvailable() each frame when new content is ready + */ +UCLASS() +class UMosisPhoneTexture : public UTexture +{ + GENERATED_BODY() + +public: + UMosisPhoneTexture(); + + /** Initialize the texture with dimensions */ + void Initialize(uint32 InWidth, uint32 InHeight); + +#if PLATFORM_ANDROID + /** + * Update the texture from a hardware buffer. + * Schedules render thread work to import the buffer. + * @param Buffer The AHardwareBuffer from MosisService + */ + void UpdateFromHardwareBuffer(AHardwareBuffer* Buffer); + + /** + * Notify that a new frame is available. + * Schedules render thread work to copy the source to local texture. + */ + void NotifyFrameAvailable(); +#endif + + /** Check if the texture is ready for rendering */ + bool IsReady() const; + + // UTexture interface + virtual FTextureResource* CreateResource() override; + virtual EMaterialValueType GetMaterialType() const override { return MCT_Texture2D; } + virtual float GetSurfaceWidth() const override { return Width; } + virtual float GetSurfaceHeight() const override { return Height; } + virtual float GetSurfaceDepth() const override { return 0; } + virtual uint32 GetSurfaceArraySize() const override { return 0; } + +protected: + /** Width of the phone screen */ + UPROPERTY() + uint32 Width = 540; + + /** Height of the phone screen */ + UPROPERTY() + uint32 Height = 1170; + +private: + /** Our custom texture resource */ + FMosisPhoneTextureResource* PhoneTextureResource = nullptr; + +#if PLATFORM_ANDROID + /** Pending hardware buffer to import */ + TAtomic PendingBuffer{nullptr}; + + /** Flag indicating new frame is available */ + TAtomic bFrameAvailable{false}; +#endif +}; diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTextureResource.cpp b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTextureResource.cpp new file mode 100644 index 0000000..901f020 --- /dev/null +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTextureResource.cpp @@ -0,0 +1,133 @@ +// Copyright OmixLab LTD. All Rights Reserved. + +#include "MosisPhoneTextureResource.h" +#include "RenderingThread.h" +#include "RHICommandList.h" + +#if PLATFORM_ANDROID +#include "IVulkanDynamicRHI.h" +#endif + +DEFINE_LOG_CATEGORY_STATIC(LogMosisTextureResource, Log, All); + +FMosisPhoneTextureResource::FMosisPhoneTextureResource(uint32 InWidth, uint32 InHeight) + : Width(InWidth) + , Height(InHeight) +{ +} + +FMosisPhoneTextureResource::~FMosisPhoneTextureResource() +{ +} + +void FMosisPhoneTextureResource::InitRHI(FRHICommandListBase& RHICmdList) +{ + UE_LOG(LogMosisTextureResource, Log, TEXT("InitRHI: %dx%d"), Width, Height); + + // Create local texture for rendering + const FRHITextureCreateDesc Desc = + FRHITextureCreateDesc::Create2D(TEXT("MosisPhoneLocalTexture"), Width, Height, PF_R8G8B8A8) + .SetFlags(ETextureCreateFlags::ShaderResource | ETextureCreateFlags::RenderTargetable) + .SetInitialState(ERHIAccess::SRVMask); + + LocalTextureRHI = RHICreateTexture(Desc); + + if (LocalTextureRHI.IsValid()) + { + UE_LOG(LogMosisTextureResource, Log, TEXT("InitRHI: Local texture created")); + + // Create default SRV + TextureRHI = LocalTextureRHI; + } + else + { + UE_LOG(LogMosisTextureResource, Error, TEXT("InitRHI: Failed to create local texture")); + } +} + +void FMosisPhoneTextureResource::ReleaseRHI() +{ + UE_LOG(LogMosisTextureResource, Log, TEXT("ReleaseRHI")); + +#if PLATFORM_ANDROID + SourceTextureRHI.SafeRelease(); + CurrentBuffer = nullptr; +#endif + + LocalTextureRHI.SafeRelease(); + TextureRHI.SafeRelease(); + bIsReady = false; +} + +#if PLATFORM_ANDROID + +void FMosisPhoneTextureResource::UpdateFromHardwareBuffer_RenderThread(AHardwareBuffer* Buffer) +{ + check(IsInRenderingThread()); + + if (!Buffer) + { + UE_LOG(LogMosisTextureResource, Warning, TEXT("UpdateFromHardwareBuffer: null buffer")); + return; + } + + // Check if this is a new buffer + if (Buffer == CurrentBuffer && SourceTextureRHI.IsValid()) + { + return; + } + + // Get buffer dimensions + AHardwareBuffer_Desc Desc{}; + AHardwareBuffer_describe(Buffer, &Desc); + + UE_LOG(LogMosisTextureResource, Log, TEXT("UpdateFromHardwareBuffer: Importing %dx%d buffer"), + Desc.width, Desc.height); + + // Release old source texture + SourceTextureRHI.SafeRelease(); + + // Get the Vulkan RHI interface + IVulkanDynamicRHI* VulkanRHI = GetIVulkanDynamicRHI(); + if (!VulkanRHI) + { + UE_LOG(LogMosisTextureResource, Error, TEXT("UpdateFromHardwareBuffer: VulkanRHI not available")); + return; + } + + // Import the hardware buffer using UE5's built-in API + // This handles all Vulkan extension setup internally + SourceTextureRHI = VulkanRHI->RHICreateTexture2DFromAndroidHardwareBuffer(Buffer); + + if (SourceTextureRHI.IsValid()) + { + CurrentBuffer = Buffer; + UE_LOG(LogMosisTextureResource, Log, TEXT("UpdateFromHardwareBuffer: Import successful")); + } + else + { + UE_LOG(LogMosisTextureResource, Error, TEXT("UpdateFromHardwareBuffer: Import failed")); + } +} + +void FMosisPhoneTextureResource::CopySourceToLocal_RenderThread(FRHICommandListImmediate& RHICmdList) +{ + check(IsInRenderingThread()); + + if (!SourceTextureRHI.IsValid() || !LocalTextureRHI.IsValid()) + { + return; + } + + // Copy source to local texture + FRHICopyTextureInfo CopyInfo; + CopyInfo.Size.X = Width; + CopyInfo.Size.Y = Height; + CopyInfo.Size.Z = 1; + + RHICmdList.CopyTexture(SourceTextureRHI, LocalTextureRHI, CopyInfo); + + bIsReady = true; +} + +#endif // PLATFORM_ANDROID diff --git a/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTextureResource.h b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTextureResource.h new file mode 100644 index 0000000..9b6ecab --- /dev/null +++ b/Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTextureResource.h @@ -0,0 +1,76 @@ +// Copyright OmixLab LTD. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "TextureResource.h" +#include "RHI.h" +#include "RHIResources.h" + +#if PLATFORM_ANDROID +#include +#endif + +/** + * FMosisPhoneTextureResource - Render resource for importing AHardwareBuffer via UE5's Vulkan RHI. + * + * Uses IVulkanDynamicRHI::RHICreateTexture2DFromAndroidHardwareBuffer() for import, + * which handles all Vulkan extension setup internally. + * + * Thread Safety: + * - UpdateFromHardwareBuffer_RenderThread: Call from render thread only + * - CopySourceToLocal_RenderThread: Call from render thread only + * - Buffer pointer exchange uses FCriticalSection + */ +class FMosisPhoneTextureResource : public FTextureResource +{ +public: + FMosisPhoneTextureResource(uint32 InWidth, uint32 InHeight); + virtual ~FMosisPhoneTextureResource(); + + // FTextureResource interface + virtual void InitRHI(FRHICommandListBase& RHICmdList) override; + virtual void ReleaseRHI() override; + virtual uint32 GetSizeX() const override { return Width; } + virtual uint32 GetSizeY() const override { return Height; } + +#if PLATFORM_ANDROID + /** + * Import a hardware buffer on the render thread. + * Creates source texture from the buffer. + * @param Buffer The AHardwareBuffer from MosisService + */ + void UpdateFromHardwareBuffer_RenderThread(AHardwareBuffer* Buffer); + + /** + * Copy from source (imported) texture to local texture. + * Call this each frame when a new frame is available. + * @param RHICmdList The RHI command list for copy operations + */ + void CopySourceToLocal_RenderThread(FRHICommandListImmediate& RHICmdList); +#endif + + /** Check if we have a valid local texture for rendering */ + bool IsReady() const { return bIsReady; } + + /** Get the local texture for material sampling */ + FTextureRHIRef GetLocalTexture() const { return LocalTextureRHI; } + +private: + uint32 Width; + uint32 Height; + + /** Local texture copy safe for rendering (not shared with MosisService) */ + FTextureRHIRef LocalTextureRHI; + +#if PLATFORM_ANDROID + /** Source texture imported from AHardwareBuffer */ + FTextureRHIRef SourceTextureRHI; + + /** Currently imported hardware buffer */ + AHardwareBuffer* CurrentBuffer = nullptr; +#endif + + /** True when local texture is ready for rendering */ + TAtomic bIsReady{false}; +}; diff --git a/Plugins/MosisSDK/Source/MosisSDK/Public/MosisPhoneActor.h b/Plugins/MosisSDK/Source/MosisSDK/Public/MosisPhoneActor.h index 50c5cc6..6a110d3 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Public/MosisPhoneActor.h +++ b/Plugins/MosisSDK/Source/MosisSDK/Public/MosisPhoneActor.h @@ -13,6 +13,9 @@ class UMaterialInstanceDynamic; /** * AMosisPhoneActor - Pre-configured actor for displaying the Mosis phone. * Contains mesh, material, and phone component ready for use. + * + * By default, uses Engine's basic plane mesh. For custom phone models, + * set the PhoneMesh property to your mesh asset. */ UCLASS(BlueprintType, Blueprintable) class MOSISSDK_API AMosisPhoneActor : public AActor @@ -23,6 +26,7 @@ public: AMosisPhoneActor(); virtual void BeginPlay() override; + virtual void Tick(float DeltaSeconds) override; /** * Send a touch event to the phone at world coordinates. @@ -66,6 +70,21 @@ protected: UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis") FVector4 ScreenBoundsLocal = FVector4(-50.0f, -100.0f, 50.0f, 100.0f); + /** + * Base material to use for the phone screen. + * Should have a texture parameter matching TextureParameterName on the PhoneComponent. + * If not set, a simple unlit emissive material will be created. + */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis") + TObjectPtr BaseMaterial; + + /** + * Phone screen dimensions in world units. + * Default is 10x21.7 cm (typical smartphone aspect ratio 9:19.5) + */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis") + FVector2D ScreenSizeWorld = FVector2D(10.0f, 21.7f); + private: /** Convert world hit to UV coordinates */ bool WorldToPhoneUV(FVector WorldLocation, FVector2D& OutUV) const; diff --git a/Plugins/MosisSDK/Source/MosisSDK/Public/MosisPhoneComponent.h b/Plugins/MosisSDK/Source/MosisSDK/Public/MosisPhoneComponent.h index e640a7f..1c3db63 100644 --- a/Plugins/MosisSDK/Source/MosisSDK/Public/MosisPhoneComponent.h +++ b/Plugins/MosisSDK/Source/MosisSDK/Public/MosisPhoneComponent.h @@ -7,7 +7,7 @@ #include "MosisPhoneComponent.generated.h" class UMaterialInstanceDynamic; -class UTexture2D; +class UMosisPhoneTexture; /** * Touch event type for phone interactions. @@ -50,7 +50,7 @@ public: * @return The texture showing the phone screen, or nullptr if not ready */ UFUNCTION(BlueprintCallable, Category = "Mosis") - UTexture2D* GetPhoneTexture() const { return PhoneTexture; } + UTexture* GetPhoneTexture() const; /** * Check if the phone is connected and ready. @@ -90,20 +90,15 @@ protected: private: /** Phone screen texture */ UPROPERTY() - UTexture2D* PhoneTexture; + TObjectPtr PhoneTexture; /** Screen dimensions from service */ int32 ScreenWidth = 0; int32 ScreenHeight = 0; /** Flag to track pending texture updates */ - bool bPendingTextureUpdate = false; + TAtomic bPendingTextureUpdate{false}; /** Flag to track if we need to recreate the texture */ - bool bNeedsTextureCreate = false; - -#if PLATFORM_ANDROID - /** Vulkan texture wrapper */ - TSharedPtr VulkanTexture; -#endif + TAtomic bNeedsTextureCreate{false}; };