omar/MosisUnreal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix phone texture rendering using UTexture2DDynamic

Browse Source
This commit is contained in:
omigamedev
2026-01-17 18:20:08 +01:00
parent 554a946e60
commit dc1bd14ff0
7 changed files with 155 additions and 319 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
Plugins/MosisSDK/README.md
View File

@@ -17,16 +17,16 @@ The MosisSDK plugin connects to the MosisService Android application via AIDL (A
│ │
│ ┌─────────────────┐ ┌──────────────────────────────────────────┐ │
│ │ AMosisPhoneActor│ │ UMosisPhoneTexture │ │
│ │ (Plane Mesh) │ │ │ │ │
│ │ (Material) │ │ ▼ │ │
│ └────────┬────────┘ │ ┌────────────────────────────────────┐ │ │
│ │ │ │ FMosisPhoneTextureResource │ │ │
│ ▼ │ │ │ │ │ │
│ ┌─────────────────────┐ │ │ ▼ ENQUEUE_RENDER_CMD │ │ │
│ │ UMosisPhoneComponent│──┼──► IVulkanDynamicRHI:: │ │ │
│ │ (Touch Input) │ │ │ RHICreateTexture2DFromAndroid │ │ │
│ │ (Callbacks) │ │ │ HardwareBuffer() │ │ │
│ └──────────┬──────────┘ │ └────────────────────────────────────┘ │ │
│ │ (Plane Mesh) │ │ (extends UTexture2DDynamic) │ │
│ │ (Material) │ │ │ │ │
│ └────────┬────────┘ │ ▼ CPU lock/copy │ │
│ │ │ AHardwareBuffer_lock() │ │
│ ▼ │ │ │ │
│ ┌─────────────────────┐ │ ▼ RHIUpdateTexture2D │ │
│ │ UMosisPhoneComponent│──┼──► FTexture2DDynamicResource │ │
│ │ (Touch Input) │ │ │ │ │
│ │ (Callbacks) │ │ ▼ │ │
│ └──────────┬──────────┘ │ UMaterialInstanceDynamic │ │
│ │ └──────────────────────────────────────────┘ │
│ ▼ │
│ ┌──────────────────────────────────────────────────────────────────┐ │
@@ -87,8 +87,7 @@ Plugins/MosisSDK/
│ └── Private/
│ ├── MosisSDK.cpp # Module + JNI callbacks
│ ├── MosisClient.h/cpp # AIDL client implementation
│ ├── MosisPhoneTexture.h/cpp # UTexture for phone screen
│ ├── MosisPhoneTextureResource.h/cpp # Render thread resource
│ ├── MosisPhoneTexture.h/cpp # UTexture2DDynamic subclass for phone screen
│ ├── MosisPhoneComponent.cpp
│ ├── MosisPhoneActor.cpp
│ └── Android/
@@ -364,6 +363,12 @@ HandTrackingVersion=V2
## Version History
- **v1.2** - UTexture2DDynamic integration
- Switched `UMosisPhoneTexture` to extend `UTexture2DDynamic` for proper material system integration
- Removed custom `FMosisPhoneTextureResource` - using UE5's built-in `FTexture2DDynamicResource`
- CPU-based texture upload via `AHardwareBuffer_lock()` and `RHIUpdateTexture2D()`
- Fixed stride conversion (AHardwareBuffer stride is in pixels, RHI expects bytes)
- **v1.1** - UE5 RHI texture integration
- New `UMosisPhoneTexture` using UE5's `IVulkanDynamicRHI::RHICreateTexture2DFromAndroidHardwareBuffer()`
- `FMosisPhoneTextureResource` for render thread HardwareBuffer import

15
Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneActor.cpp
View File

@@ -43,18 +43,13 @@ void AMosisPhoneActor::BeginPlay()
UE_LOG(LogMosisPhoneActor, Log, TEXT("BeginPlay"));
// Scale mesh to match phone screen size
// The default plane is 100x100 units, so scale accordingly
// Note: We no longer override the scale set in the editor.
// The user can scale the actor as desired.
// ScreenBoundsLocal uses the mesh's local bounds (default plane is 100x100 centered at origin)
if (PhoneMesh)
{
float ScaleX = ScreenSizeWorld.X / 100.0f;
float ScaleY = ScreenSizeWorld.Y / 100.0f;
PhoneMesh->SetRelativeScale3D(FVector(ScaleX, ScaleY, 1.0f));
// Update screen bounds based on scaled size
float HalfWidth = ScreenSizeWorld.X / 2.0f;
float HalfHeight = ScreenSizeWorld.Y / 2.0f;
ScreenBoundsLocal = FVector4(-HalfWidth, -HalfHeight, HalfWidth, HalfHeight);
// Default plane is 100x100 units, so bounds are -50 to 50 in X and Y
ScreenBoundsLocal = FVector4(-50.0f, -50.0f, 50.0f, 50.0f);
}
// Create dynamic material for the screen

14
Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneComponent.cpp
View File

@@ -48,6 +48,13 @@ void UMosisPhoneComponent::BeginPlay()
});
UE_LOG(LogMosisPhone, Log, TEXT("BeginPlay: Callbacks registered"));
// Check if buffer is already available (we may have missed the callback)
if (Client->GetHardwareBuffer() != nullptr)
{
UE_LOG(LogMosisPhone, Log, TEXT("BeginPlay: Buffer already available, triggering update"));
bNeedsTextureCreate.Store(true);
}
}
else
{
@@ -186,6 +193,13 @@ void UMosisPhoneComponent::OnFrameAvailable()
// Notify texture that a new frame is available
// This schedules the render thread copy
PhoneTexture->NotifyFrameAvailable();
// Log occasionally to avoid spam
static int32 FrameCount = 0;
if (++FrameCount % 60 == 0)
{
UE_LOG(LogMosisPhone, Log, TEXT("OnFrameAvailable: Frame %d"), FrameCount);
}
#endif
}

138
Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTexture.cpp
View File

@@ -1,41 +1,25 @@
// Copyright OmixLab LTD. All Rights Reserved.
#include "MosisPhoneTexture.h"
#include "MosisPhoneTextureResource.h"
#include "RenderingThread.h"
DEFINE_LOG_CATEGORY_STATIC(LogMosisPhoneTexture, Log, All);
UMosisPhoneTexture::UMosisPhoneTexture()
{
// Default initialization
// UTexture2DDynamic defaults are fine
SRGB = true;
NeverStream = true;
LODGroup = TEXTUREGROUP_UI;
Filter = TF_Bilinear;
}
void UMosisPhoneTexture::Initialize(uint32 InWidth, uint32 InHeight)
{
Width = InWidth;
Height = InHeight;
UE_LOG(LogMosisPhoneTexture, Log, TEXT("Initialize: %dx%d"), InWidth, InHeight);
UE_LOG(LogMosisPhoneTexture, Log, TEXT("Initialize: %dx%d"), Width, Height);
// Use UTexture2DDynamic's Init method
Init(InWidth, InHeight, PF_R8G8B8A8, false);
// Create the resource
UpdateResource();
}
FTextureResource* UMosisPhoneTexture::CreateResource()
{
UE_LOG(LogMosisPhoneTexture, Log, TEXT("CreateResource: %dx%d"), Width, Height);
PhoneTextureResource = new FMosisPhoneTextureResource(Width, Height);
return PhoneTextureResource;
}
bool UMosisPhoneTexture::IsReady() const
{
return PhoneTextureResource && PhoneTextureResource->IsReady();
bIsReady = true;
}
#if PLATFORM_ANDROID
@@ -48,47 +32,119 @@ void UMosisPhoneTexture::UpdateFromHardwareBuffer(AHardwareBuffer* Buffer)
return;
}
if (!PhoneTextureResource)
CurrentBuffer = Buffer;
// Get buffer dimensions
AHardwareBuffer_Desc Desc{};
AHardwareBuffer_describe(Buffer, &Desc);
UE_LOG(LogMosisPhoneTexture, Log, TEXT("UpdateFromHardwareBuffer: %dx%d"), Desc.width, Desc.height);
// Lock buffer and read data
void* LockedData = nullptr;
int32 Result = AHardwareBuffer_lock(Buffer, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN, -1, nullptr, &LockedData);
if (Result != 0 || !LockedData)
{
UE_LOG(LogMosisPhoneTexture, Warning, TEXT("UpdateFromHardwareBuffer: resource not initialized"));
UE_LOG(LogMosisPhoneTexture, Error, TEXT("UpdateFromHardwareBuffer: Failed to lock buffer, result=%d"), Result);
return;
}
// Store buffer for render thread
PendingBuffer.Store(Buffer);
uint8* PixelData = static_cast<uint8*>(LockedData);
// Capture resource pointer for lambda
FMosisPhoneTextureResource* Resource = PhoneTextureResource;
AHardwareBuffer* BufferToImport = Buffer;
// Copy data to a buffer for update (accounting for stride)
uint32 StridePixels = Desc.stride > 0 ? Desc.stride : Desc.width;
TArray<uint8> TextureData;
TextureData.SetNumUninitialized(Desc.width * Desc.height * 4);
// Schedule render thread work
ENQUEUE_RENDER_COMMAND(MosisImportHardwareBuffer)(
[Resource, BufferToImport](FRHICommandListImmediate& RHICmdList)
for (uint32 y = 0; y < Desc.height; y++)
{
Resource->UpdateFromHardwareBuffer_RenderThread(BufferToImport);
FMemory::Memcpy(
TextureData.GetData() + y * Desc.width * 4,
PixelData + y * StridePixels * 4,
Desc.width * 4
);
}
AHardwareBuffer_unlock(Buffer, nullptr);
// Update the texture using UTexture2DDynamic's update regions
FTexture2DDynamicResource* TextureResource = static_cast<FTexture2DDynamicResource*>(GetResource());
if (TextureResource)
{
ENQUEUE_RENDER_COMMAND(UpdateMosisTexture)(
[TextureResource, Data = MoveTemp(TextureData), Width = Desc.width, Height = Desc.height](FRHICommandListImmediate& RHICmdList)
{
FUpdateTextureRegion2D Region(0, 0, 0, 0, Width, Height);
RHIUpdateTexture2D(
TextureResource->GetTexture2DRHI(),
0,
Region,
Width * 4,
Data.GetData()
);
}
);
}
UE_LOG(LogMosisPhoneTexture, Log, TEXT("UpdateFromHardwareBuffer: Scheduled import"));
}
void UMosisPhoneTexture::NotifyFrameAvailable()
{
if (!PhoneTextureResource)
if (!CurrentBuffer || !bIsReady)
{
return;
}
// Capture resource pointer for lambda
FMosisPhoneTextureResource* Resource = PhoneTextureResource;
// Get buffer dimensions
AHardwareBuffer_Desc Desc{};
AHardwareBuffer_describe(CurrentBuffer, &Desc);
// Schedule render thread copy
ENQUEUE_RENDER_COMMAND(MosisCopyFrame)(
[Resource](FRHICommandListImmediate& RHICmdList)
// Lock buffer and read data
void* LockedData = nullptr;
int32 Result = AHardwareBuffer_lock(CurrentBuffer, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN, -1, nullptr, &LockedData);
if (Result != 0 || !LockedData)
{
Resource->CopySourceToLocal_RenderThread(RHICmdList);
return;
}
uint8* PixelData = static_cast<uint8*>(LockedData);
// Copy data to a buffer for update (accounting for stride)
uint32 StridePixels = Desc.stride > 0 ? Desc.stride : Desc.width;
TArray<uint8> TextureData;
TextureData.SetNumUninitialized(Desc.width * Desc.height * 4);
for (uint32 y = 0; y < Desc.height; y++)
{
FMemory::Memcpy(
TextureData.GetData() + y * Desc.width * 4,
PixelData + y * StridePixels * 4,
Desc.width * 4
);
}
AHardwareBuffer_unlock(CurrentBuffer, nullptr);
// Update the texture
FTexture2DDynamicResource* TextureResource = static_cast<FTexture2DDynamicResource*>(GetResource());
if (TextureResource)
{
ENQUEUE_RENDER_COMMAND(UpdateMosisTextureFrame)(
[TextureResource, Data = MoveTemp(TextureData), Width = Desc.width, Height = Desc.height](FRHICommandListImmediate& RHICmdList)
{
FUpdateTextureRegion2D Region(0, 0, 0, 0, Width, Height);
RHIUpdateTexture2D(
TextureResource->GetTexture2DRHI(),
0,
Region,
Width * 4,
Data.GetData()
);
}
);
}
}
#endif // PLATFORM_ANDROID

55
Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTexture.h
View File

@@ -3,29 +3,21 @@
#pragma once
#include "CoreMinimal.h"
#include "Engine/Texture.h"
#include "Engine/Texture2DDynamic.h"
#include "MosisPhoneTexture.generated.h"
#if PLATFORM_ANDROID
#include <android/hardware_buffer.h>
#endif
class FMosisPhoneTextureResource;
/**
* UMosisPhoneTexture - Custom texture that displays the Mosis phone screen.
* UMosisPhoneTexture - Dynamic texture that displays the Mosis phone screen.
*
* This texture is updated from an AHardwareBuffer received from MosisService.
* Uses UE5's IVulkanDynamicRHI for efficient hardware buffer import.
*
* Usage:
* 1. Create via NewObject<UMosisPhoneTexture>() with width/height
* 2. Call UpdateResource() to initialize
* 3. Call UpdateFromHardwareBuffer() when buffer is received
* 4. Call NotifyFrameAvailable() each frame when new content is ready
* Inherits from UTexture2DDynamic for proper material integration.
*/
UCLASS()
class UMosisPhoneTexture : public UTexture
class UMosisPhoneTexture : public UTexture2DDynamic
{
GENERATED_BODY()
@@ -38,47 +30,30 @@ public:
#if PLATFORM_ANDROID
/**
* Update the texture from a hardware buffer.
* Schedules render thread work to import the buffer.
* @param Buffer The AHardwareBuffer from MosisService
*/
void UpdateFromHardwareBuffer(AHardwareBuffer* Buffer);
/**
* Notify that a new frame is available.
* Schedules render thread work to copy the source to local texture.
* Copies buffer data to the texture.
*/
void NotifyFrameAvailable();
/** Store the current hardware buffer reference */
void SetHardwareBuffer(AHardwareBuffer* Buffer) { CurrentBuffer = Buffer; }
AHardwareBuffer* GetHardwareBuffer() const { return CurrentBuffer; }
#endif
/** Check if the texture is ready for rendering */
bool IsReady() const;
// UTexture interface
virtual FTextureResource* CreateResource() override;
virtual EMaterialValueType GetMaterialType() const override { return MCT_Texture2D; }
virtual float GetSurfaceWidth() const override { return Width; }
virtual float GetSurfaceHeight() const override { return Height; }
virtual float GetSurfaceDepth() const override { return 0; }
virtual uint32 GetSurfaceArraySize() const override { return 0; }
protected:
/** Width of the phone screen */
UPROPERTY()
uint32 Width = 540;
/** Height of the phone screen */
UPROPERTY()
uint32 Height = 1170;
bool IsReady() const { return bIsReady; }
private:
/** Our custom texture resource */
FMosisPhoneTextureResource* PhoneTextureResource = nullptr;
#if PLATFORM_ANDROID
/** Pending hardware buffer to import */
TAtomic<AHardwareBuffer*> PendingBuffer{nullptr};
/** Flag indicating new frame is available */
TAtomic<bool> bFrameAvailable{false};
/** Current hardware buffer */
AHardwareBuffer* CurrentBuffer = nullptr;
#endif
/** True when texture has been initialized */
bool bIsReady = false;
};

133
Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTextureResource.cpp
View File

@@ -1,133 +0,0 @@
// Copyright OmixLab LTD. All Rights Reserved.
#include "MosisPhoneTextureResource.h"
#include "RenderingThread.h"
#include "RHICommandList.h"
#if PLATFORM_ANDROID
#include "IVulkanDynamicRHI.h"
#endif
DEFINE_LOG_CATEGORY_STATIC(LogMosisTextureResource, Log, All);
FMosisPhoneTextureResource::FMosisPhoneTextureResource(uint32 InWidth, uint32 InHeight)
: Width(InWidth)
, Height(InHeight)
{
}
FMosisPhoneTextureResource::~FMosisPhoneTextureResource()
{
}
void FMosisPhoneTextureResource::InitRHI(FRHICommandListBase& RHICmdList)
{
UE_LOG(LogMosisTextureResource, Log, TEXT("InitRHI: %dx%d"), Width, Height);
// Create local texture for rendering
const FRHITextureCreateDesc Desc =
FRHITextureCreateDesc::Create2D(TEXT("MosisPhoneLocalTexture"), Width, Height, PF_R8G8B8A8)
.SetFlags(ETextureCreateFlags::ShaderResource | ETextureCreateFlags::RenderTargetable)
.SetInitialState(ERHIAccess::SRVMask);
LocalTextureRHI = RHICreateTexture(Desc);
if (LocalTextureRHI.IsValid())
{
UE_LOG(LogMosisTextureResource, Log, TEXT("InitRHI: Local texture created"));
// Create default SRV
TextureRHI = LocalTextureRHI;
}
else
{
UE_LOG(LogMosisTextureResource, Error, TEXT("InitRHI: Failed to create local texture"));
}
}
void FMosisPhoneTextureResource::ReleaseRHI()
{
UE_LOG(LogMosisTextureResource, Log, TEXT("ReleaseRHI"));
#if PLATFORM_ANDROID
SourceTextureRHI.SafeRelease();
CurrentBuffer = nullptr;
#endif
LocalTextureRHI.SafeRelease();
TextureRHI.SafeRelease();
bIsReady = false;
}
#if PLATFORM_ANDROID
void FMosisPhoneTextureResource::UpdateFromHardwareBuffer_RenderThread(AHardwareBuffer* Buffer)
{
check(IsInRenderingThread());
if (!Buffer)
{
UE_LOG(LogMosisTextureResource, Warning, TEXT("UpdateFromHardwareBuffer: null buffer"));
return;
}
// Check if this is a new buffer
if (Buffer == CurrentBuffer && SourceTextureRHI.IsValid())
{
return;
}
// Get buffer dimensions
AHardwareBuffer_Desc Desc{};
AHardwareBuffer_describe(Buffer, &Desc);
UE_LOG(LogMosisTextureResource, Log, TEXT("UpdateFromHardwareBuffer: Importing %dx%d buffer"),
Desc.width, Desc.height);
// Release old source texture
SourceTextureRHI.SafeRelease();
// Get the Vulkan RHI interface
IVulkanDynamicRHI* VulkanRHI = GetIVulkanDynamicRHI();
if (!VulkanRHI)
{
UE_LOG(LogMosisTextureResource, Error, TEXT("UpdateFromHardwareBuffer: VulkanRHI not available"));
return;
}
// Import the hardware buffer using UE5's built-in API
// This handles all Vulkan extension setup internally
SourceTextureRHI = VulkanRHI->RHICreateTexture2DFromAndroidHardwareBuffer(Buffer);
if (SourceTextureRHI.IsValid())
{
CurrentBuffer = Buffer;
UE_LOG(LogMosisTextureResource, Log, TEXT("UpdateFromHardwareBuffer: Import successful"));
}
else
{
UE_LOG(LogMosisTextureResource, Error, TEXT("UpdateFromHardwareBuffer: Import failed"));
}
}
void FMosisPhoneTextureResource::CopySourceToLocal_RenderThread(FRHICommandListImmediate& RHICmdList)
{
check(IsInRenderingThread());
if (!SourceTextureRHI.IsValid() || !LocalTextureRHI.IsValid())
{
return;
}
// Copy source to local texture
FRHICopyTextureInfo CopyInfo;
CopyInfo.Size.X = Width;
CopyInfo.Size.Y = Height;
CopyInfo.Size.Z = 1;
RHICmdList.CopyTexture(SourceTextureRHI, LocalTextureRHI, CopyInfo);
bIsReady = true;
}
#endif // PLATFORM_ANDROID

76
Plugins/MosisSDK/Source/MosisSDK/Private/MosisPhoneTextureResource.h
View File

@@ -1,76 +0,0 @@
// Copyright OmixLab LTD. All Rights Reserved.
#pragma once
#include "CoreMinimal.h"
#include "TextureResource.h"
#include "RHI.h"
#include "RHIResources.h"
#if PLATFORM_ANDROID
#include <android/hardware_buffer.h>
#endif
/**
* FMosisPhoneTextureResource - Render resource for importing AHardwareBuffer via UE5's Vulkan RHI.
*
* Uses IVulkanDynamicRHI::RHICreateTexture2DFromAndroidHardwareBuffer() for import,
* which handles all Vulkan extension setup internally.
*
* Thread Safety:
* - UpdateFromHardwareBuffer_RenderThread: Call from render thread only
* - CopySourceToLocal_RenderThread: Call from render thread only
* - Buffer pointer exchange uses FCriticalSection
*/
class FMosisPhoneTextureResource : public FTextureResource
{
public:
FMosisPhoneTextureResource(uint32 InWidth, uint32 InHeight);
virtual ~FMosisPhoneTextureResource();
// FTextureResource interface
virtual void InitRHI(FRHICommandListBase& RHICmdList) override;
virtual void ReleaseRHI() override;
virtual uint32 GetSizeX() const override { return Width; }
virtual uint32 GetSizeY() const override { return Height; }
#if PLATFORM_ANDROID
/**
* Import a hardware buffer on the render thread.
* Creates source texture from the buffer.
* @param Buffer The AHardwareBuffer from MosisService
*/
void UpdateFromHardwareBuffer_RenderThread(AHardwareBuffer* Buffer);
/**
* Copy from source (imported) texture to local texture.
* Call this each frame when a new frame is available.
* @param RHICmdList The RHI command list for copy operations
*/
void CopySourceToLocal_RenderThread(FRHICommandListImmediate& RHICmdList);
#endif
/** Check if we have a valid local texture for rendering */
bool IsReady() const { return bIsReady; }
/** Get the local texture for material sampling */
FTextureRHIRef GetLocalTexture() const { return LocalTextureRHI; }
private:
uint32 Width;
uint32 Height;
/** Local texture copy safe for rendering (not shared with MosisService) */
FTextureRHIRef LocalTextureRHI;
#if PLATFORM_ANDROID
/** Source texture imported from AHardwareBuffer */
FTextureRHIRef SourceTextureRHI;
/** Currently imported hardware buffer */
AHardwareBuffer* CurrentBuffer = nullptr;
#endif
/** True when local texture is ready for rendering */
TAtomic<bool> bIsReady{false};
};