fix phone texture rendering using UTexture2DDynamic

This commit is contained in:
2026-01-17 18:20:08 +01:00
parent 554a946e60
commit dc1bd14ff0
7 changed files with 155 additions and 319 deletions

View File

@@ -17,16 +17,16 @@ The MosisSDK plugin connects to the MosisService Android application via AIDL (A
│ │ │ │
│ ┌─────────────────┐ ┌──────────────────────────────────────────┐ │ │ ┌─────────────────┐ ┌──────────────────────────────────────────┐ │
│ │ AMosisPhoneActor│ │ UMosisPhoneTexture │ │ │ │ AMosisPhoneActor│ │ UMosisPhoneTexture │ │
│ │ (Plane Mesh) │ │ │ │ │ │ (Plane Mesh) │ │ (extends UTexture2DDynamic) │ │
│ │ (Material) │ │ │ │ │ │ (Material) │ │ │ │
│ └────────┬────────┘ │ ┌────────────────────────────────────┐ │ │ │ └────────┬────────┘ │ ▼ CPU lock/copy │ │
│ │ │ FMosisPhoneTextureResource │ │ │ │ │ AHardwareBuffer_lock() │ │
│ ▼ │ │ │ │ ▼ │ │ │ │
│ ┌─────────────────────┐ │ENQUEUE_RENDER_CMD │ │ │ │ ┌─────────────────────┐ │ ▼ RHIUpdateTexture2D │ │
│ │ UMosisPhoneComponent│──┼──► IVulkanDynamicRHI:: │ │ │ │ UMosisPhoneComponent│──┼──► FTexture2DDynamicResource │ │
│ │ (Touch Input) │ │ RHICreateTexture2DFromAndroid │ │ │ │ │ (Touch Input) │ │ │ │
│ │ (Callbacks) │ │ HardwareBuffer() │ │ │ │ (Callbacks) │ │ │ │
│ └──────────┬──────────┘ │ └────────────────────────────────────┘ │ │ │ └──────────┬──────────┘ │ UMaterialInstanceDynamic │ │
│ │ └──────────────────────────────────────────┘ │ │ │ └──────────────────────────────────────────┘ │
│ ▼ │ │ ▼ │
│ ┌──────────────────────────────────────────────────────────────────┐ │ │ ┌──────────────────────────────────────────────────────────────────┐ │
@@ -87,8 +87,7 @@ Plugins/MosisSDK/
│ └── Private/ │ └── Private/
│ ├── MosisSDK.cpp # Module + JNI callbacks │ ├── MosisSDK.cpp # Module + JNI callbacks
│ ├── MosisClient.h/cpp # AIDL client implementation │ ├── MosisClient.h/cpp # AIDL client implementation
│ ├── MosisPhoneTexture.h/cpp # UTexture for phone screen │ ├── MosisPhoneTexture.h/cpp # UTexture2DDynamic subclass for phone screen
│ ├── MosisPhoneTextureResource.h/cpp # Render thread resource
│ ├── MosisPhoneComponent.cpp │ ├── MosisPhoneComponent.cpp
│ ├── MosisPhoneActor.cpp │ ├── MosisPhoneActor.cpp
│ └── Android/ │ └── Android/
@@ -364,6 +363,12 @@ HandTrackingVersion=V2
## Version History ## Version History
- **v1.2** - UTexture2DDynamic integration
- Switched `UMosisPhoneTexture` to extend `UTexture2DDynamic` for proper material system integration
- Removed custom `FMosisPhoneTextureResource` - using UE5's built-in `FTexture2DDynamicResource`
- CPU-based texture upload via `AHardwareBuffer_lock()` and `RHIUpdateTexture2D()`
- Fixed stride conversion (AHardwareBuffer stride is in pixels, RHI expects bytes)
- **v1.1** - UE5 RHI texture integration - **v1.1** - UE5 RHI texture integration
- New `UMosisPhoneTexture` using UE5's `IVulkanDynamicRHI::RHICreateTexture2DFromAndroidHardwareBuffer()` - New `UMosisPhoneTexture` using UE5's `IVulkanDynamicRHI::RHICreateTexture2DFromAndroidHardwareBuffer()`
- `FMosisPhoneTextureResource` for render thread HardwareBuffer import - `FMosisPhoneTextureResource` for render thread HardwareBuffer import

View File

@@ -43,18 +43,13 @@ void AMosisPhoneActor::BeginPlay()
UE_LOG(LogMosisPhoneActor, Log, TEXT("BeginPlay")); UE_LOG(LogMosisPhoneActor, Log, TEXT("BeginPlay"));
// Scale mesh to match phone screen size // Note: We no longer override the scale set in the editor.
// The default plane is 100x100 units, so scale accordingly // The user can scale the actor as desired.
// ScreenBoundsLocal uses the mesh's local bounds (default plane is 100x100 centered at origin)
if (PhoneMesh) if (PhoneMesh)
{ {
float ScaleX = ScreenSizeWorld.X / 100.0f; // Default plane is 100x100 units, so bounds are -50 to 50 in X and Y
float ScaleY = ScreenSizeWorld.Y / 100.0f; ScreenBoundsLocal = FVector4(-50.0f, -50.0f, 50.0f, 50.0f);
PhoneMesh->SetRelativeScale3D(FVector(ScaleX, ScaleY, 1.0f));
// Update screen bounds based on scaled size
float HalfWidth = ScreenSizeWorld.X / 2.0f;
float HalfHeight = ScreenSizeWorld.Y / 2.0f;
ScreenBoundsLocal = FVector4(-HalfWidth, -HalfHeight, HalfWidth, HalfHeight);
} }
// Create dynamic material for the screen // Create dynamic material for the screen

View File

@@ -48,6 +48,13 @@ void UMosisPhoneComponent::BeginPlay()
}); });
UE_LOG(LogMosisPhone, Log, TEXT("BeginPlay: Callbacks registered")); UE_LOG(LogMosisPhone, Log, TEXT("BeginPlay: Callbacks registered"));
// Check if buffer is already available (we may have missed the callback)
if (Client->GetHardwareBuffer() != nullptr)
{
UE_LOG(LogMosisPhone, Log, TEXT("BeginPlay: Buffer already available, triggering update"));
bNeedsTextureCreate.Store(true);
}
} }
else else
{ {
@@ -186,6 +193,13 @@ void UMosisPhoneComponent::OnFrameAvailable()
// Notify texture that a new frame is available // Notify texture that a new frame is available
// This schedules the render thread copy // This schedules the render thread copy
PhoneTexture->NotifyFrameAvailable(); PhoneTexture->NotifyFrameAvailable();
// Log occasionally to avoid spam
static int32 FrameCount = 0;
if (++FrameCount % 60 == 0)
{
UE_LOG(LogMosisPhone, Log, TEXT("OnFrameAvailable: Frame %d"), FrameCount);
}
#endif #endif
} }

View File

@@ -1,41 +1,25 @@
// Copyright OmixLab LTD. All Rights Reserved. // Copyright OmixLab LTD. All Rights Reserved.
#include "MosisPhoneTexture.h" #include "MosisPhoneTexture.h"
#include "MosisPhoneTextureResource.h"
#include "RenderingThread.h" #include "RenderingThread.h"
DEFINE_LOG_CATEGORY_STATIC(LogMosisPhoneTexture, Log, All); DEFINE_LOG_CATEGORY_STATIC(LogMosisPhoneTexture, Log, All);
UMosisPhoneTexture::UMosisPhoneTexture() UMosisPhoneTexture::UMosisPhoneTexture()
{ {
// Default initialization // UTexture2DDynamic defaults are fine
SRGB = true; SRGB = true;
NeverStream = true; Filter = TF_Bilinear;
LODGroup = TEXTUREGROUP_UI;
} }
void UMosisPhoneTexture::Initialize(uint32 InWidth, uint32 InHeight) void UMosisPhoneTexture::Initialize(uint32 InWidth, uint32 InHeight)
{ {
Width = InWidth; UE_LOG(LogMosisPhoneTexture, Log, TEXT("Initialize: %dx%d"), InWidth, InHeight);
Height = InHeight;
UE_LOG(LogMosisPhoneTexture, Log, TEXT("Initialize: %dx%d"), Width, Height); // Use UTexture2DDynamic's Init method
Init(InWidth, InHeight, PF_R8G8B8A8, false);
// Create the resource bIsReady = true;
UpdateResource();
}
FTextureResource* UMosisPhoneTexture::CreateResource()
{
UE_LOG(LogMosisPhoneTexture, Log, TEXT("CreateResource: %dx%d"), Width, Height);
PhoneTextureResource = new FMosisPhoneTextureResource(Width, Height);
return PhoneTextureResource;
}
bool UMosisPhoneTexture::IsReady() const
{
return PhoneTextureResource && PhoneTextureResource->IsReady();
} }
#if PLATFORM_ANDROID #if PLATFORM_ANDROID
@@ -48,47 +32,119 @@ void UMosisPhoneTexture::UpdateFromHardwareBuffer(AHardwareBuffer* Buffer)
return; return;
} }
if (!PhoneTextureResource) CurrentBuffer = Buffer;
// Get buffer dimensions
AHardwareBuffer_Desc Desc{};
AHardwareBuffer_describe(Buffer, &Desc);
UE_LOG(LogMosisPhoneTexture, Log, TEXT("UpdateFromHardwareBuffer: %dx%d"), Desc.width, Desc.height);
// Lock buffer and read data
void* LockedData = nullptr;
int32 Result = AHardwareBuffer_lock(Buffer, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN, -1, nullptr, &LockedData);
if (Result != 0 || !LockedData)
{ {
UE_LOG(LogMosisPhoneTexture, Warning, TEXT("UpdateFromHardwareBuffer: resource not initialized")); UE_LOG(LogMosisPhoneTexture, Error, TEXT("UpdateFromHardwareBuffer: Failed to lock buffer, result=%d"), Result);
return; return;
} }
// Store buffer for render thread uint8* PixelData = static_cast<uint8*>(LockedData);
PendingBuffer.Store(Buffer);
// Capture resource pointer for lambda // Copy data to a buffer for update (accounting for stride)
FMosisPhoneTextureResource* Resource = PhoneTextureResource; uint32 StridePixels = Desc.stride > 0 ? Desc.stride : Desc.width;
AHardwareBuffer* BufferToImport = Buffer; TArray<uint8> TextureData;
TextureData.SetNumUninitialized(Desc.width * Desc.height * 4);
// Schedule render thread work for (uint32 y = 0; y < Desc.height; y++)
ENQUEUE_RENDER_COMMAND(MosisImportHardwareBuffer)(
[Resource, BufferToImport](FRHICommandListImmediate& RHICmdList)
{ {
Resource->UpdateFromHardwareBuffer_RenderThread(BufferToImport); FMemory::Memcpy(
TextureData.GetData() + y * Desc.width * 4,
PixelData + y * StridePixels * 4,
Desc.width * 4
);
}
AHardwareBuffer_unlock(Buffer, nullptr);
// Update the texture using UTexture2DDynamic's update regions
FTexture2DDynamicResource* TextureResource = static_cast<FTexture2DDynamicResource*>(GetResource());
if (TextureResource)
{
ENQUEUE_RENDER_COMMAND(UpdateMosisTexture)(
[TextureResource, Data = MoveTemp(TextureData), Width = Desc.width, Height = Desc.height](FRHICommandListImmediate& RHICmdList)
{
FUpdateTextureRegion2D Region(0, 0, 0, 0, Width, Height);
RHIUpdateTexture2D(
TextureResource->GetTexture2DRHI(),
0,
Region,
Width * 4,
Data.GetData()
);
} }
); );
}
UE_LOG(LogMosisPhoneTexture, Log, TEXT("UpdateFromHardwareBuffer: Scheduled import"));
} }
void UMosisPhoneTexture::NotifyFrameAvailable() void UMosisPhoneTexture::NotifyFrameAvailable()
{ {
if (!PhoneTextureResource) if (!CurrentBuffer || !bIsReady)
{ {
return; return;
} }
// Capture resource pointer for lambda // Get buffer dimensions
FMosisPhoneTextureResource* Resource = PhoneTextureResource; AHardwareBuffer_Desc Desc{};
AHardwareBuffer_describe(CurrentBuffer, &Desc);
// Schedule render thread copy // Lock buffer and read data
ENQUEUE_RENDER_COMMAND(MosisCopyFrame)( void* LockedData = nullptr;
[Resource](FRHICommandListImmediate& RHICmdList) int32 Result = AHardwareBuffer_lock(CurrentBuffer, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN, -1, nullptr, &LockedData);
if (Result != 0 || !LockedData)
{ {
Resource->CopySourceToLocal_RenderThread(RHICmdList); return;
}
uint8* PixelData = static_cast<uint8*>(LockedData);
// Copy data to a buffer for update (accounting for stride)
uint32 StridePixels = Desc.stride > 0 ? Desc.stride : Desc.width;
TArray<uint8> TextureData;
TextureData.SetNumUninitialized(Desc.width * Desc.height * 4);
for (uint32 y = 0; y < Desc.height; y++)
{
FMemory::Memcpy(
TextureData.GetData() + y * Desc.width * 4,
PixelData + y * StridePixels * 4,
Desc.width * 4
);
}
AHardwareBuffer_unlock(CurrentBuffer, nullptr);
// Update the texture
FTexture2DDynamicResource* TextureResource = static_cast<FTexture2DDynamicResource*>(GetResource());
if (TextureResource)
{
ENQUEUE_RENDER_COMMAND(UpdateMosisTextureFrame)(
[TextureResource, Data = MoveTemp(TextureData), Width = Desc.width, Height = Desc.height](FRHICommandListImmediate& RHICmdList)
{
FUpdateTextureRegion2D Region(0, 0, 0, 0, Width, Height);
RHIUpdateTexture2D(
TextureResource->GetTexture2DRHI(),
0,
Region,
Width * 4,
Data.GetData()
);
} }
); );
}
} }
#endif // PLATFORM_ANDROID #endif // PLATFORM_ANDROID

View File

@@ -3,29 +3,21 @@
#pragma once #pragma once
#include "CoreMinimal.h" #include "CoreMinimal.h"
#include "Engine/Texture.h" #include "Engine/Texture2DDynamic.h"
#include "MosisPhoneTexture.generated.h" #include "MosisPhoneTexture.generated.h"
#if PLATFORM_ANDROID #if PLATFORM_ANDROID
#include <android/hardware_buffer.h> #include <android/hardware_buffer.h>
#endif #endif
class FMosisPhoneTextureResource;
/** /**
* UMosisPhoneTexture - Custom texture that displays the Mosis phone screen. * UMosisPhoneTexture - Dynamic texture that displays the Mosis phone screen.
* *
* This texture is updated from an AHardwareBuffer received from MosisService. * This texture is updated from an AHardwareBuffer received from MosisService.
* Uses UE5's IVulkanDynamicRHI for efficient hardware buffer import. * Inherits from UTexture2DDynamic for proper material integration.
*
* Usage:
* 1. Create via NewObject<UMosisPhoneTexture>() with width/height
* 2. Call UpdateResource() to initialize
* 3. Call UpdateFromHardwareBuffer() when buffer is received
* 4. Call NotifyFrameAvailable() each frame when new content is ready
*/ */
UCLASS() UCLASS()
class UMosisPhoneTexture : public UTexture class UMosisPhoneTexture : public UTexture2DDynamic
{ {
GENERATED_BODY() GENERATED_BODY()
@@ -38,47 +30,30 @@ public:
#if PLATFORM_ANDROID #if PLATFORM_ANDROID
/** /**
* Update the texture from a hardware buffer. * Update the texture from a hardware buffer.
* Schedules render thread work to import the buffer.
* @param Buffer The AHardwareBuffer from MosisService * @param Buffer The AHardwareBuffer from MosisService
*/ */
void UpdateFromHardwareBuffer(AHardwareBuffer* Buffer); void UpdateFromHardwareBuffer(AHardwareBuffer* Buffer);
/** /**
* Notify that a new frame is available. * Notify that a new frame is available.
* Schedules render thread work to copy the source to local texture. * Copies buffer data to the texture.
*/ */
void NotifyFrameAvailable(); void NotifyFrameAvailable();
/** Store the current hardware buffer reference */
void SetHardwareBuffer(AHardwareBuffer* Buffer) { CurrentBuffer = Buffer; }
AHardwareBuffer* GetHardwareBuffer() const { return CurrentBuffer; }
#endif #endif
/** Check if the texture is ready for rendering */ /** Check if the texture is ready for rendering */
bool IsReady() const; bool IsReady() const { return bIsReady; }
// UTexture interface
virtual FTextureResource* CreateResource() override;
virtual EMaterialValueType GetMaterialType() const override { return MCT_Texture2D; }
virtual float GetSurfaceWidth() const override { return Width; }
virtual float GetSurfaceHeight() const override { return Height; }
virtual float GetSurfaceDepth() const override { return 0; }
virtual uint32 GetSurfaceArraySize() const override { return 0; }
protected:
/** Width of the phone screen */
UPROPERTY()
uint32 Width = 540;
/** Height of the phone screen */
UPROPERTY()
uint32 Height = 1170;
private: private:
/** Our custom texture resource */
FMosisPhoneTextureResource* PhoneTextureResource = nullptr;
#if PLATFORM_ANDROID #if PLATFORM_ANDROID
/** Pending hardware buffer to import */ /** Current hardware buffer */
TAtomic<AHardwareBuffer*> PendingBuffer{nullptr}; AHardwareBuffer* CurrentBuffer = nullptr;
/** Flag indicating new frame is available */
TAtomic<bool> bFrameAvailable{false};
#endif #endif
/** True when texture has been initialized */
bool bIsReady = false;
}; };

View File

@@ -1,133 +0,0 @@
// Copyright OmixLab LTD. All Rights Reserved.
#include "MosisPhoneTextureResource.h"
#include "RenderingThread.h"
#include "RHICommandList.h"
#if PLATFORM_ANDROID
#include "IVulkanDynamicRHI.h"
#endif
DEFINE_LOG_CATEGORY_STATIC(LogMosisTextureResource, Log, All);
FMosisPhoneTextureResource::FMosisPhoneTextureResource(uint32 InWidth, uint32 InHeight)
: Width(InWidth)
, Height(InHeight)
{
}
FMosisPhoneTextureResource::~FMosisPhoneTextureResource()
{
}
void FMosisPhoneTextureResource::InitRHI(FRHICommandListBase& RHICmdList)
{
UE_LOG(LogMosisTextureResource, Log, TEXT("InitRHI: %dx%d"), Width, Height);
// Create local texture for rendering
const FRHITextureCreateDesc Desc =
FRHITextureCreateDesc::Create2D(TEXT("MosisPhoneLocalTexture"), Width, Height, PF_R8G8B8A8)
.SetFlags(ETextureCreateFlags::ShaderResource | ETextureCreateFlags::RenderTargetable)
.SetInitialState(ERHIAccess::SRVMask);
LocalTextureRHI = RHICreateTexture(Desc);
if (LocalTextureRHI.IsValid())
{
UE_LOG(LogMosisTextureResource, Log, TEXT("InitRHI: Local texture created"));
// Create default SRV
TextureRHI = LocalTextureRHI;
}
else
{
UE_LOG(LogMosisTextureResource, Error, TEXT("InitRHI: Failed to create local texture"));
}
}
void FMosisPhoneTextureResource::ReleaseRHI()
{
UE_LOG(LogMosisTextureResource, Log, TEXT("ReleaseRHI"));
#if PLATFORM_ANDROID
SourceTextureRHI.SafeRelease();
CurrentBuffer = nullptr;
#endif
LocalTextureRHI.SafeRelease();
TextureRHI.SafeRelease();
bIsReady = false;
}
#if PLATFORM_ANDROID
void FMosisPhoneTextureResource::UpdateFromHardwareBuffer_RenderThread(AHardwareBuffer* Buffer)
{
check(IsInRenderingThread());
if (!Buffer)
{
UE_LOG(LogMosisTextureResource, Warning, TEXT("UpdateFromHardwareBuffer: null buffer"));
return;
}
// Check if this is a new buffer
if (Buffer == CurrentBuffer && SourceTextureRHI.IsValid())
{
return;
}
// Get buffer dimensions
AHardwareBuffer_Desc Desc{};
AHardwareBuffer_describe(Buffer, &Desc);
UE_LOG(LogMosisTextureResource, Log, TEXT("UpdateFromHardwareBuffer: Importing %dx%d buffer"),
Desc.width, Desc.height);
// Release old source texture
SourceTextureRHI.SafeRelease();
// Get the Vulkan RHI interface
IVulkanDynamicRHI* VulkanRHI = GetIVulkanDynamicRHI();
if (!VulkanRHI)
{
UE_LOG(LogMosisTextureResource, Error, TEXT("UpdateFromHardwareBuffer: VulkanRHI not available"));
return;
}
// Import the hardware buffer using UE5's built-in API
// This handles all Vulkan extension setup internally
SourceTextureRHI = VulkanRHI->RHICreateTexture2DFromAndroidHardwareBuffer(Buffer);
if (SourceTextureRHI.IsValid())
{
CurrentBuffer = Buffer;
UE_LOG(LogMosisTextureResource, Log, TEXT("UpdateFromHardwareBuffer: Import successful"));
}
else
{
UE_LOG(LogMosisTextureResource, Error, TEXT("UpdateFromHardwareBuffer: Import failed"));
}
}
void FMosisPhoneTextureResource::CopySourceToLocal_RenderThread(FRHICommandListImmediate& RHICmdList)
{
check(IsInRenderingThread());
if (!SourceTextureRHI.IsValid() || !LocalTextureRHI.IsValid())
{
return;
}
// Copy source to local texture
FRHICopyTextureInfo CopyInfo;
CopyInfo.Size.X = Width;
CopyInfo.Size.Y = Height;
CopyInfo.Size.Z = 1;
RHICmdList.CopyTexture(SourceTextureRHI, LocalTextureRHI, CopyInfo);
bIsReady = true;
}
#endif // PLATFORM_ANDROID

View File

@@ -1,76 +0,0 @@
// Copyright OmixLab LTD. All Rights Reserved.
#pragma once
#include "CoreMinimal.h"
#include "TextureResource.h"
#include "RHI.h"
#include "RHIResources.h"
#if PLATFORM_ANDROID
#include <android/hardware_buffer.h>
#endif
/**
* FMosisPhoneTextureResource - Render resource for importing AHardwareBuffer via UE5's Vulkan RHI.
*
* Uses IVulkanDynamicRHI::RHICreateTexture2DFromAndroidHardwareBuffer() for import,
* which handles all Vulkan extension setup internally.
*
* Thread Safety:
* - UpdateFromHardwareBuffer_RenderThread: Call from render thread only
* - CopySourceToLocal_RenderThread: Call from render thread only
* - Buffer pointer exchange uses FCriticalSection
*/
class FMosisPhoneTextureResource : public FTextureResource
{
public:
FMosisPhoneTextureResource(uint32 InWidth, uint32 InHeight);
virtual ~FMosisPhoneTextureResource();
// FTextureResource interface
virtual void InitRHI(FRHICommandListBase& RHICmdList) override;
virtual void ReleaseRHI() override;
virtual uint32 GetSizeX() const override { return Width; }
virtual uint32 GetSizeY() const override { return Height; }
#if PLATFORM_ANDROID
/**
* Import a hardware buffer on the render thread.
* Creates source texture from the buffer.
* @param Buffer The AHardwareBuffer from MosisService
*/
void UpdateFromHardwareBuffer_RenderThread(AHardwareBuffer* Buffer);
/**
* Copy from source (imported) texture to local texture.
* Call this each frame when a new frame is available.
* @param RHICmdList The RHI command list for copy operations
*/
void CopySourceToLocal_RenderThread(FRHICommandListImmediate& RHICmdList);
#endif
/** Check if we have a valid local texture for rendering */
bool IsReady() const { return bIsReady; }
/** Get the local texture for material sampling */
FTextureRHIRef GetLocalTexture() const { return LocalTextureRHI; }
private:
uint32 Width;
uint32 Height;
/** Local texture copy safe for rendering (not shared with MosisService) */
FTextureRHIRef LocalTextureRHI;
#if PLATFORM_ANDROID
/** Source texture imported from AHardwareBuffer */
FTextureRHIRef SourceTextureRHI;
/** Currently imported hardware buffer */
AHardwareBuffer* CurrentBuffer = nullptr;
#endif
/** True when local texture is ready for rendering */
TAtomic<bool> bIsReady{false};
};