Compare commits

..

3 Commits

13 changed files with 461 additions and 585 deletions

View File

@@ -2,3 +2,6 @@
ProjectID=DD810CA045CA9288C9C53E8638A45978
bStartInVR=True
[/Script/UnrealEd.ProjectPackagingSettings]
+DirectoriesToAlwaysCook=(Path="/Game/Mosis")

Binary file not shown.

View File

@@ -12,24 +12,33 @@ The MosisSDK plugin connects to the MosisService Android application via AIDL (A
## Architecture
```
┌─────────────────────────────────────────────────────────────┐
│ Unreal Engine Game │
┌─────────────────────┐ ┌─────────────────────────────┐
│ UMosisPhoneComponent│◄───│ MosisVulkanTexture │
│ │ (Touch Input) │ │ (HardwareBuffer Import) │ │
└──────────┬──────────┘ └──────────────▲──────────────┘
│ │
┌─────────────────────────────────────────┴───────────────┐
│ MosisClient ││
│ (AIDL IMosisListener)
└──────────────────────────┬──────────────────────────────
└─────────────────────────────┼───────────────────────────────┘
Binder IPC
─────────────────────────────▼───────────────────────────────┐
MosisService
(Renders phone UI via RmlUi)
─────────────────────────────────────────────────────────────
┌─────────────────────────────────────────────────────────────────────────
Unreal Engine Game
┌─────────────────┐ ┌──────────────────────────────────────────┐
│ │ AMosisPhoneActor│ │ UMosisPhoneTexture
│ (Plane Mesh) │ │ (extends UTexture2DDynamic) │
│ (Material) │ │
└────────┬────────┘ ▼ CPU lock/copy │
│ │ AHardwareBuffer_lock() │
▼ │ │
┌─────────────────────┐ │ ▼ RHIUpdateTexture2D
│ UMosisPhoneComponent│────► FTexture2DDynamicResource │
│ │ (Touch Input) │ │ │ │ │
(Callbacks) │ │ ▼
│ └────────────────────┘ │ UMaterialInstanceDynamic │ │
└──────────────────────────────────────────┘
│ ┌──────────────────────────────────────────────────────────────────┐ │
│ │ MosisClient │ │
│ │ (AIDL IMosisListener) │ │
│ └──────────────────────────────┬───────────────────────────────────┘ │
└─────────────────────────────────┼───────────────────────────────────────┘
│ Binder IPC
┌─────────────────────────────────▼───────────────────────────────────────┐
│ MosisService │
│ (Renders phone UI via RmlUi) │
└─────────────────────────────────────────────────────────────────────────┘
```
## Prerequisites
@@ -73,12 +82,12 @@ Plugins/MosisSDK/
│ │ └── BpMosisService.h
│ ├── Public/
│ │ ├── MosisSDK.h # Module interface
│ │ ├── MosisPhoneComponent.h # UE5 component
│ │ └── MosisPhoneActor.h # Blueprint actor
│ │ ├── MosisPhoneComponent.h # UE5 component (touch, callbacks)
│ │ └── MosisPhoneActor.h # Blueprint actor (mesh, material)
│ └── Private/
│ ├── MosisSDK.cpp # Module + JNI callbacks
│ ├── MosisClient.h/cpp # AIDL client implementation
│ ├── MosisVulkanTexture.h/cpp # Vulkan HardwareBuffer import
│ ├── MosisPhoneTexture.h/cpp # UTexture2DDynamic subclass for phone screen
│ ├── MosisPhoneComponent.cpp
│ ├── MosisPhoneActor.cpp
│ └── Android/
@@ -152,7 +161,8 @@ The separation ensures Windows builds don't try to compile Android-specific AIDL
| Slate, SlateCore | ✓ | ✓ | UI framework |
| RenderCore, RHI | ✓ | ✓ | Rendering abstraction |
| Launch, ApplicationCore | ✗ | ✓ | Android JNI access |
| Vulkan | ✗ | ✓ | Hardware buffer import |
| VulkanRHI | ✗ | ✓ | IVulkanDynamicRHI for HardwareBuffer import |
| Vulkan | ✗ | ✓ | Vulkan headers |
| binder_ndk, android, nativewindow | ✗ | ✓ | Android system libs |
### NDK Header Compatibility
@@ -353,6 +363,19 @@ HandTrackingVersion=V2
## Version History
- **v1.2** - UTexture2DDynamic integration
- Switched `UMosisPhoneTexture` to extend `UTexture2DDynamic` for proper material system integration
- Removed custom `FMosisPhoneTextureResource` - using UE5's built-in `FTexture2DDynamicResource`
- CPU-based texture upload via `AHardwareBuffer_lock()` and `RHIUpdateTexture2D()`
- Fixed stride conversion (AHardwareBuffer stride is in pixels, RHI expects bytes)
- **v1.1** - UE5 RHI texture integration
- New `UMosisPhoneTexture` using UE5's `IVulkanDynamicRHI::RHICreateTexture2DFromAndroidHardwareBuffer()`
- `FMosisPhoneTextureResource` for render thread HardwareBuffer import
- `AMosisPhoneActor` now includes default plane mesh and material setup
- Thread-safe callbacks using `TWeakObjectPtr` and `TAtomic`
- Automatic texture-to-material binding in tick
- **v1.0** - Initial implementation
- AIDL client for MosisService connection
- Vulkan HardwareBuffer import

View File

@@ -35,7 +35,8 @@ public class MosisSDK : ModuleRules
// Android-specific module dependencies
PrivateDependencyModuleNames.AddRange(new string[] {
"Launch",
"ApplicationCore"
"ApplicationCore",
"VulkanRHI"
});
// Add Vulkan support (Vulkan headers for HardwareBuffer import)

View File

@@ -4,17 +4,35 @@
#include "Components/StaticMeshComponent.h"
#include "Materials/MaterialInstanceDynamic.h"
#include "Engine/StaticMesh.h"
#include "UObject/ConstructorHelpers.h"
DEFINE_LOG_CATEGORY_STATIC(LogMosisPhoneActor, Log, All);
AMosisPhoneActor::AMosisPhoneActor()
{
PrimaryActorTick.bCanEverTick = false;
PrimaryActorTick.bCanEverTick = true;
PrimaryActorTick.bStartWithTickEnabled = true;
// Create phone mesh component
PhoneMesh = CreateDefaultSubobject<UStaticMeshComponent>(TEXT("PhoneMesh"));
RootComponent = PhoneMesh;
// Load the default plane mesh
static ConstructorHelpers::FObjectFinder<UStaticMesh> PlaneMeshFinder(
TEXT("/Engine/BasicShapes/Plane.Plane"));
if (PlaneMeshFinder.Succeeded())
{
PhoneMesh->SetStaticMesh(PlaneMeshFinder.Object);
}
// Load the phone screen material
static ConstructorHelpers::FObjectFinder<UMaterialInterface> PhoneMaterialFinder(
TEXT("/Game/Mosis/Materials/M_PhoneScreen.M_PhoneScreen"));
if (PhoneMaterialFinder.Succeeded())
{
BaseMaterial = PhoneMaterialFinder.Object;
}
// Create phone component
PhoneComponent = CreateDefaultSubobject<UMosisPhoneComponent>(TEXT("PhoneComponent"));
}
@@ -25,13 +43,29 @@ void AMosisPhoneActor::BeginPlay()
UE_LOG(LogMosisPhoneActor, Log, TEXT("BeginPlay"));
// Note: We no longer override the scale set in the editor.
// The user can scale the actor as desired.
// ScreenBoundsLocal uses the mesh's local bounds (default plane is 100x100 centered at origin)
if (PhoneMesh)
{
// Default plane is 100x100 units, so bounds are -50 to 50 in X and Y
ScreenBoundsLocal = FVector4(-50.0f, -50.0f, 50.0f, 50.0f);
}
// Create dynamic material for the screen
if (PhoneMesh)
{
UMaterialInterface* BaseMaterial = PhoneMesh->GetMaterial(0);
if (BaseMaterial)
UMaterialInterface* MaterialToUse = BaseMaterial;
// If no base material set, use the mesh's existing material
if (!MaterialToUse)
{
ScreenMaterial = UMaterialInstanceDynamic::Create(BaseMaterial, this);
MaterialToUse = PhoneMesh->GetMaterial(0);
}
if (MaterialToUse)
{
ScreenMaterial = UMaterialInstanceDynamic::Create(MaterialToUse, this);
PhoneMesh->SetMaterial(0, ScreenMaterial);
// Set it on the phone component
@@ -42,6 +76,25 @@ void AMosisPhoneActor::BeginPlay()
UE_LOG(LogMosisPhoneActor, Log, TEXT("BeginPlay: Created dynamic material"));
}
else
{
UE_LOG(LogMosisPhoneActor, Warning, TEXT("BeginPlay: No base material available"));
}
}
}
void AMosisPhoneActor::Tick(float DeltaSeconds)
{
Super::Tick(DeltaSeconds);
// Update material texture if phone component has a new texture
if (ScreenMaterial && PhoneComponent)
{
UTexture* PhoneTexture = PhoneComponent->GetPhoneTexture();
if (PhoneTexture)
{
ScreenMaterial->SetTextureParameterValue(PhoneComponent->TextureParameterName, PhoneTexture);
}
}
}

View File

@@ -2,13 +2,12 @@
#include "MosisPhoneComponent.h"
#include "MosisSDK.h"
#include "Engine/Texture2D.h"
#include "MosisPhoneTexture.h"
#include "Materials/MaterialInstanceDynamic.h"
#include "RenderingThread.h"
#if PLATFORM_ANDROID
#include "MosisClient.h"
#include "MosisVulkanTexture.h"
#endif
DEFINE_LOG_CATEGORY_STATIC(LogMosisPhone, Log, All);
@@ -30,18 +29,32 @@ void UMosisPhoneComponent::BeginPlay()
auto Client = FMosisSDKModule::GetClient();
if (Client)
{
// Set up callbacks
Client->SetBufferCallback([this](AHardwareBuffer* buffer) {
// This runs on the binder thread - flag for main thread processing
bNeedsTextureCreate = true;
// Use weak pointer to prevent dangling references in callbacks
TWeakObjectPtr<UMosisPhoneComponent> WeakThis(this);
// Set up callbacks (these run on binder thread)
Client->SetBufferCallback([WeakThis](AHardwareBuffer* buffer) {
if (WeakThis.IsValid())
{
WeakThis->bNeedsTextureCreate.Store(true);
}
});
Client->SetFrameCallback([this]() {
// This runs on the binder thread - flag for tick processing
bPendingTextureUpdate = true;
Client->SetFrameCallback([WeakThis]() {
if (WeakThis.IsValid())
{
WeakThis->bPendingTextureUpdate.Store(true);
}
});
UE_LOG(LogMosisPhone, Log, TEXT("BeginPlay: Callbacks registered"));
// Check if buffer is already available (we may have missed the callback)
if (Client->GetHardwareBuffer() != nullptr)
{
UE_LOG(LogMosisPhone, Log, TEXT("BeginPlay: Buffer already available, triggering update"));
bNeedsTextureCreate.Store(true);
}
}
else
{
@@ -55,12 +68,6 @@ void UMosisPhoneComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
UE_LOG(LogMosisPhone, Log, TEXT("EndPlay"));
#if PLATFORM_ANDROID
// Clean up Vulkan texture
if (VulkanTexture)
{
VulkanTexture.Reset();
}
// Clear callbacks
auto Client = FMosisSDKModule::GetClient();
if (Client)
@@ -70,6 +77,9 @@ void UMosisPhoneComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
}
#endif
// Release texture
PhoneTexture = nullptr;
Super::EndPlay(EndPlayReason);
}
@@ -79,16 +89,14 @@ void UMosisPhoneComponent::TickComponent(float DeltaTime, ELevelTick TickType, F
#if PLATFORM_ANDROID
// Check if we need to create texture from new buffer
if (bNeedsTextureCreate)
if (bNeedsTextureCreate.Exchange(false))
{
bNeedsTextureCreate = false;
OnBufferAvailable();
}
// Check if we need to update texture for new frame
if (bPendingTextureUpdate && VulkanTexture && VulkanTexture->IsValid())
if (bPendingTextureUpdate.Exchange(false) && PhoneTexture)
{
bPendingTextureUpdate = false;
OnFrameAvailable();
}
#endif
@@ -155,25 +163,17 @@ void UMosisPhoneComponent::OnBufferAvailable()
UE_LOG(LogMosisPhone, Log, TEXT("OnBufferAvailable: %dx%d"), ScreenWidth, ScreenHeight);
// TODO: Initialize Vulkan texture on render thread
// This requires accessing UE5's Vulkan RHI which has platform-specific APIs.
// For now, we log the buffer availability and dimensions.
// Full implementation would:
// 1. Get VkDevice from GVulkanRHI or IVulkanDynamicRHI
// 2. Create MosisVulkanTexture and import the HardwareBuffer
// 3. Create UTexture2D wrapping the Vulkan image
// Create placeholder texture
// Create phone texture if needed
if (!PhoneTexture)
{
PhoneTexture = UTexture2D::CreateTransient(ScreenWidth, ScreenHeight, PF_R8G8B8A8);
if (PhoneTexture)
{
PhoneTexture->UpdateResource();
UE_LOG(LogMosisPhone, Log, TEXT("OnBufferAvailable: Created placeholder texture"));
}
PhoneTexture = NewObject<UMosisPhoneTexture>(this);
PhoneTexture->Initialize(ScreenWidth, ScreenHeight);
UE_LOG(LogMosisPhone, Log, TEXT("OnBufferAvailable: Created phone texture"));
}
// Import the hardware buffer
PhoneTexture->UpdateFromHardwareBuffer(Buffer);
// Update material if set
if (PhoneMaterial && PhoneTexture)
{
@@ -185,19 +185,30 @@ void UMosisPhoneComponent::OnBufferAvailable()
void UMosisPhoneComponent::OnFrameAvailable()
{
#if PLATFORM_ANDROID
if (!VulkanTexture || !VulkanTexture->IsValid())
if (!PhoneTexture)
{
return;
}
// TODO: Copy texture on render thread
// This requires accessing UE5's Vulkan command buffer management.
// Full implementation would enqueue a render command to copy the
// imported image to the local image.
// Notify texture that a new frame is available
// This schedules the render thread copy
PhoneTexture->NotifyFrameAvailable();
// Log occasionally to avoid spam
static int32 FrameCount = 0;
if (++FrameCount % 60 == 0)
{
UE_LOG(LogMosisPhone, Log, TEXT("OnFrameAvailable: Frame %d"), FrameCount);
}
#endif
}
void UMosisPhoneComponent::UpdateTextureOnRenderThread()
{
// Reserved for render thread texture updates
// No longer needed - handled by UMosisPhoneTexture
}
UTexture* UMosisPhoneComponent::GetPhoneTexture() const
{
return PhoneTexture;
}

View File

@@ -0,0 +1,150 @@
// Copyright OmixLab LTD. All Rights Reserved.
#include "MosisPhoneTexture.h"
#include "RenderingThread.h"
DEFINE_LOG_CATEGORY_STATIC(LogMosisPhoneTexture, Log, All);
UMosisPhoneTexture::UMosisPhoneTexture()
{
// UTexture2DDynamic defaults are fine
SRGB = true;
Filter = TF_Bilinear;
}
void UMosisPhoneTexture::Initialize(uint32 InWidth, uint32 InHeight)
{
UE_LOG(LogMosisPhoneTexture, Log, TEXT("Initialize: %dx%d"), InWidth, InHeight);
// Use UTexture2DDynamic's Init method
Init(InWidth, InHeight, PF_R8G8B8A8, false);
bIsReady = true;
}
#if PLATFORM_ANDROID
void UMosisPhoneTexture::UpdateFromHardwareBuffer(AHardwareBuffer* Buffer)
{
if (!Buffer)
{
UE_LOG(LogMosisPhoneTexture, Warning, TEXT("UpdateFromHardwareBuffer: null buffer"));
return;
}
CurrentBuffer = Buffer;
// Get buffer dimensions
AHardwareBuffer_Desc Desc{};
AHardwareBuffer_describe(Buffer, &Desc);
UE_LOG(LogMosisPhoneTexture, Log, TEXT("UpdateFromHardwareBuffer: %dx%d"), Desc.width, Desc.height);
// Lock buffer and read data
void* LockedData = nullptr;
int32 Result = AHardwareBuffer_lock(Buffer, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN, -1, nullptr, &LockedData);
if (Result != 0 || !LockedData)
{
UE_LOG(LogMosisPhoneTexture, Error, TEXT("UpdateFromHardwareBuffer: Failed to lock buffer, result=%d"), Result);
return;
}
uint8* PixelData = static_cast<uint8*>(LockedData);
// Copy data to a buffer for update (accounting for stride)
uint32 StridePixels = Desc.stride > 0 ? Desc.stride : Desc.width;
TArray<uint8> TextureData;
TextureData.SetNumUninitialized(Desc.width * Desc.height * 4);
for (uint32 y = 0; y < Desc.height; y++)
{
FMemory::Memcpy(
TextureData.GetData() + y * Desc.width * 4,
PixelData + y * StridePixels * 4,
Desc.width * 4
);
}
AHardwareBuffer_unlock(Buffer, nullptr);
// Update the texture using UTexture2DDynamic's update regions
FTexture2DDynamicResource* TextureResource = static_cast<FTexture2DDynamicResource*>(GetResource());
if (TextureResource)
{
ENQUEUE_RENDER_COMMAND(UpdateMosisTexture)(
[TextureResource, Data = MoveTemp(TextureData), Width = Desc.width, Height = Desc.height](FRHICommandListImmediate& RHICmdList)
{
FUpdateTextureRegion2D Region(0, 0, 0, 0, Width, Height);
RHIUpdateTexture2D(
TextureResource->GetTexture2DRHI(),
0,
Region,
Width * 4,
Data.GetData()
);
}
);
}
}
void UMosisPhoneTexture::NotifyFrameAvailable()
{
if (!CurrentBuffer || !bIsReady)
{
return;
}
// Get buffer dimensions
AHardwareBuffer_Desc Desc{};
AHardwareBuffer_describe(CurrentBuffer, &Desc);
// Lock buffer and read data
void* LockedData = nullptr;
int32 Result = AHardwareBuffer_lock(CurrentBuffer, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN, -1, nullptr, &LockedData);
if (Result != 0 || !LockedData)
{
return;
}
uint8* PixelData = static_cast<uint8*>(LockedData);
// Copy data to a buffer for update (accounting for stride)
uint32 StridePixels = Desc.stride > 0 ? Desc.stride : Desc.width;
TArray<uint8> TextureData;
TextureData.SetNumUninitialized(Desc.width * Desc.height * 4);
for (uint32 y = 0; y < Desc.height; y++)
{
FMemory::Memcpy(
TextureData.GetData() + y * Desc.width * 4,
PixelData + y * StridePixels * 4,
Desc.width * 4
);
}
AHardwareBuffer_unlock(CurrentBuffer, nullptr);
// Update the texture
FTexture2DDynamicResource* TextureResource = static_cast<FTexture2DDynamicResource*>(GetResource());
if (TextureResource)
{
ENQUEUE_RENDER_COMMAND(UpdateMosisTextureFrame)(
[TextureResource, Data = MoveTemp(TextureData), Width = Desc.width, Height = Desc.height](FRHICommandListImmediate& RHICmdList)
{
FUpdateTextureRegion2D Region(0, 0, 0, 0, Width, Height);
RHIUpdateTexture2D(
TextureResource->GetTexture2DRHI(),
0,
Region,
Width * 4,
Data.GetData()
);
}
);
}
}
#endif // PLATFORM_ANDROID

View File

@@ -0,0 +1,59 @@
// Copyright OmixLab LTD. All Rights Reserved.
#pragma once
#include "CoreMinimal.h"
#include "Engine/Texture2DDynamic.h"
#include "MosisPhoneTexture.generated.h"
#if PLATFORM_ANDROID
#include <android/hardware_buffer.h>
#endif
/**
* UMosisPhoneTexture - Dynamic texture that displays the Mosis phone screen.
*
* This texture is updated from an AHardwareBuffer received from MosisService.
* Inherits from UTexture2DDynamic for proper material integration.
*/
UCLASS()
class UMosisPhoneTexture : public UTexture2DDynamic
{
GENERATED_BODY()
public:
UMosisPhoneTexture();
/** Initialize the texture with dimensions */
void Initialize(uint32 InWidth, uint32 InHeight);
#if PLATFORM_ANDROID
/**
* Update the texture from a hardware buffer.
* @param Buffer The AHardwareBuffer from MosisService
*/
void UpdateFromHardwareBuffer(AHardwareBuffer* Buffer);
/**
* Notify that a new frame is available.
* Copies buffer data to the texture.
*/
void NotifyFrameAvailable();
/** Store the current hardware buffer reference */
void SetHardwareBuffer(AHardwareBuffer* Buffer) { CurrentBuffer = Buffer; }
AHardwareBuffer* GetHardwareBuffer() const { return CurrentBuffer; }
#endif
/** Check if the texture is ready for rendering */
bool IsReady() const { return bIsReady; }
private:
#if PLATFORM_ANDROID
/** Current hardware buffer */
AHardwareBuffer* CurrentBuffer = nullptr;
#endif
/** True when texture has been initialized */
bool bIsReady = false;
};

View File

@@ -1,421 +0,0 @@
#include "MosisVulkanTexture.h"
#if PLATFORM_ANDROID
#include "Logging/LogMacros.h"
DEFINE_LOG_CATEGORY_STATIC(LogMosisVulkan, Log, All);
MosisVulkanTexture::~MosisVulkanTexture()
{
Destroy();
}
bool MosisVulkanTexture::Initialize(VkInstance instance, VkPhysicalDevice physDevice, VkDevice device, uint32_t queueFamilyIndex)
{
if (m_Initialized)
{
return true;
}
m_Instance = instance;
m_PhysicalDevice = physDevice;
m_Device = device;
m_QueueFamilyIndex = queueFamilyIndex;
// Load extension function
vkGetAndroidHardwareBufferPropertiesANDROID =
reinterpret_cast<PFN_vkGetAndroidHardwareBufferPropertiesANDROID>(
vkGetInstanceProcAddr(m_Instance, "vkGetAndroidHardwareBufferPropertiesANDROID")
);
if (!vkGetAndroidHardwareBufferPropertiesANDROID)
{
UE_LOG(LogMosisVulkan, Error, TEXT("Initialize: vkGetAndroidHardwareBufferPropertiesANDROID not available"));
return false;
}
m_Initialized = true;
UE_LOG(LogMosisVulkan, Log, TEXT("Initialize: Success"));
return true;
}
bool MosisVulkanTexture::Create(AHardwareBuffer* buffer)
{
if (!m_Initialized)
{
UE_LOG(LogMosisVulkan, Error, TEXT("Create: Not initialized"));
return false;
}
if (m_Created)
{
Destroy();
}
// Get buffer dimensions
AHardwareBuffer_Desc desc{};
AHardwareBuffer_describe(buffer, &desc);
m_Width = desc.width;
m_Height = desc.height;
UE_LOG(LogMosisVulkan, Log, TEXT("Create: %dx%d texture"), m_Width, m_Height);
if (!ImportHardwareBuffer(buffer))
{
UE_LOG(LogMosisVulkan, Error, TEXT("Create: Failed to import hardware buffer"));
return false;
}
if (!CreateLocalImage())
{
UE_LOG(LogMosisVulkan, Error, TEXT("Create: Failed to create local image"));
DestroyImportedResources();
return false;
}
m_Created = true;
UE_LOG(LogMosisVulkan, Log, TEXT("Create: Success"));
return true;
}
bool MosisVulkanTexture::ImportHardwareBuffer(AHardwareBuffer* buffer)
{
// Query hardware buffer properties
VkAndroidHardwareBufferFormatPropertiesANDROID formatProps{};
formatProps.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_FORMAT_PROPERTIES_ANDROID;
VkAndroidHardwareBufferPropertiesANDROID props{};
props.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID;
props.pNext = &formatProps;
if (vkGetAndroidHardwareBufferPropertiesANDROID(m_Device, buffer, &props) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("ImportHardwareBuffer: Failed to get properties"));
return false;
}
m_Format = formatProps.format;
UE_LOG(LogMosisVulkan, Log, TEXT("ImportHardwareBuffer: Format=%d"), static_cast<int>(m_Format));
// Create VkImage with external memory
VkExternalMemoryImageCreateInfo extMemImageInfo{};
extMemImageInfo.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO;
extMemImageInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID;
VkImageCreateInfo imageInfo{};
imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
imageInfo.pNext = &extMemImageInfo;
imageInfo.imageType = VK_IMAGE_TYPE_2D;
imageInfo.format = m_Format;
imageInfo.extent = {m_Width, m_Height, 1};
imageInfo.mipLevels = 1;
imageInfo.arrayLayers = 1;
imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
imageInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
if (vkCreateImage(m_Device, &imageInfo, nullptr, &m_ImportedImage) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("ImportHardwareBuffer: Failed to create image"));
return false;
}
// Import memory from hardware buffer
VkImportAndroidHardwareBufferInfoANDROID importInfo{};
importInfo.sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID;
importInfo.buffer = buffer;
VkMemoryDedicatedAllocateInfo dedicatedInfo{};
dedicatedInfo.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO;
dedicatedInfo.pNext = &importInfo;
dedicatedInfo.image = m_ImportedImage;
VkMemoryAllocateInfo allocInfo{};
allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
allocInfo.pNext = &dedicatedInfo;
allocInfo.allocationSize = props.allocationSize;
allocInfo.memoryTypeIndex = FindMemoryType(props.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
if (vkAllocateMemory(m_Device, &allocInfo, nullptr, &m_ImportedMemory) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("ImportHardwareBuffer: Failed to allocate memory"));
vkDestroyImage(m_Device, m_ImportedImage, nullptr);
m_ImportedImage = VK_NULL_HANDLE;
return false;
}
if (vkBindImageMemory(m_Device, m_ImportedImage, m_ImportedMemory, 0) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("ImportHardwareBuffer: Failed to bind memory"));
DestroyImportedResources();
return false;
}
UE_LOG(LogMosisVulkan, Log, TEXT("ImportHardwareBuffer: Success"));
return true;
}
bool MosisVulkanTexture::CreateLocalImage()
{
// Create local VkImage
VkImageCreateInfo imageInfo{};
imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
imageInfo.imageType = VK_IMAGE_TYPE_2D;
imageInfo.format = m_Format;
imageInfo.extent = {m_Width, m_Height, 1};
imageInfo.mipLevels = 1;
imageInfo.arrayLayers = 1;
imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
imageInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
if (vkCreateImage(m_Device, &imageInfo, nullptr, &m_LocalImage) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to create image"));
return false;
}
// Allocate memory
VkMemoryRequirements memReqs;
vkGetImageMemoryRequirements(m_Device, m_LocalImage, &memReqs);
VkMemoryAllocateInfo allocInfo{};
allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
allocInfo.allocationSize = memReqs.size;
allocInfo.memoryTypeIndex = FindMemoryType(memReqs.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
if (vkAllocateMemory(m_Device, &allocInfo, nullptr, &m_LocalMemory) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to allocate memory"));
vkDestroyImage(m_Device, m_LocalImage, nullptr);
m_LocalImage = VK_NULL_HANDLE;
return false;
}
if (vkBindImageMemory(m_Device, m_LocalImage, m_LocalMemory, 0) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to bind memory"));
DestroyLocalResources();
return false;
}
// Create image view
VkImageViewCreateInfo viewInfo{};
viewInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
viewInfo.image = m_LocalImage;
viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D;
viewInfo.format = m_Format;
viewInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
viewInfo.subresourceRange.baseMipLevel = 0;
viewInfo.subresourceRange.levelCount = 1;
viewInfo.subresourceRange.baseArrayLayer = 0;
viewInfo.subresourceRange.layerCount = 1;
if (vkCreateImageView(m_Device, &viewInfo, nullptr, &m_LocalImageView) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to create image view"));
DestroyLocalResources();
return false;
}
// Create sampler
VkSamplerCreateInfo samplerInfo{};
samplerInfo.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO;
samplerInfo.magFilter = VK_FILTER_LINEAR;
samplerInfo.minFilter = VK_FILTER_LINEAR;
samplerInfo.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
samplerInfo.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
samplerInfo.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
samplerInfo.anisotropyEnable = VK_FALSE;
samplerInfo.maxAnisotropy = 1.0f;
samplerInfo.borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK;
samplerInfo.unnormalizedCoordinates = VK_FALSE;
samplerInfo.compareEnable = VK_FALSE;
samplerInfo.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR;
if (vkCreateSampler(m_Device, &samplerInfo, nullptr, &m_Sampler) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to create sampler"));
DestroyLocalResources();
return false;
}
UE_LOG(LogMosisVulkan, Log, TEXT("CreateLocalImage: Success"));
return true;
}
void MosisVulkanTexture::CopyToLocal(VkCommandBuffer cmd)
{
if (!m_Created)
{
return;
}
// Transition imported image to transfer src
VkImageMemoryBarrier srcBarrier{};
srcBarrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
srcBarrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
srcBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
srcBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
srcBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
srcBarrier.image = m_ImportedImage;
srcBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
srcBarrier.subresourceRange.baseMipLevel = 0;
srcBarrier.subresourceRange.levelCount = 1;
srcBarrier.subresourceRange.baseArrayLayer = 0;
srcBarrier.subresourceRange.layerCount = 1;
srcBarrier.srcAccessMask = 0;
srcBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
vkCmdPipelineBarrier(
cmd,
VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT,
0, 0, nullptr, 0, nullptr, 1, &srcBarrier
);
// Transition local image to transfer dst
VkImageMemoryBarrier dstBarrier{};
dstBarrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
dstBarrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
dstBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
dstBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
dstBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
dstBarrier.image = m_LocalImage;
dstBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
dstBarrier.subresourceRange.baseMipLevel = 0;
dstBarrier.subresourceRange.levelCount = 1;
dstBarrier.subresourceRange.baseArrayLayer = 0;
dstBarrier.subresourceRange.layerCount = 1;
dstBarrier.srcAccessMask = 0;
dstBarrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
vkCmdPipelineBarrier(
cmd,
VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT,
0, 0, nullptr, 0, nullptr, 1, &dstBarrier
);
// Copy image
VkImageCopy copyRegion{};
copyRegion.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
copyRegion.srcSubresource.layerCount = 1;
copyRegion.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
copyRegion.dstSubresource.layerCount = 1;
copyRegion.extent = {m_Width, m_Height, 1};
vkCmdCopyImage(
cmd,
m_ImportedImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
m_LocalImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
1, &copyRegion
);
// Transition local image to shader read
VkImageMemoryBarrier shaderBarrier{};
shaderBarrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
shaderBarrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
shaderBarrier.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
shaderBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
shaderBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
shaderBarrier.image = m_LocalImage;
shaderBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
shaderBarrier.subresourceRange.baseMipLevel = 0;
shaderBarrier.subresourceRange.levelCount = 1;
shaderBarrier.subresourceRange.baseArrayLayer = 0;
shaderBarrier.subresourceRange.layerCount = 1;
shaderBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
shaderBarrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
vkCmdPipelineBarrier(
cmd,
VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
0, 0, nullptr, 0, nullptr, 1, &shaderBarrier
);
}
uint32_t MosisVulkanTexture::FindMemoryType(uint32_t typeFilter, VkMemoryPropertyFlags properties)
{
VkPhysicalDeviceMemoryProperties memProps;
vkGetPhysicalDeviceMemoryProperties(m_PhysicalDevice, &memProps);
for (uint32_t i = 0; i < memProps.memoryTypeCount; i++)
{
if ((typeFilter & (1 << i)) &&
(memProps.memoryTypes[i].propertyFlags & properties) == properties)
{
return i;
}
}
UE_LOG(LogMosisVulkan, Error, TEXT("FindMemoryType: Failed to find suitable memory type"));
return 0;
}
void MosisVulkanTexture::Destroy()
{
if (m_Device && m_Created)
{
vkDeviceWaitIdle(m_Device);
}
DestroyLocalResources();
DestroyImportedResources();
m_Width = 0;
m_Height = 0;
m_Created = false;
}
void MosisVulkanTexture::DestroyImportedResources()
{
if (m_Device)
{
if (m_ImportedMemory != VK_NULL_HANDLE)
{
vkFreeMemory(m_Device, m_ImportedMemory, nullptr);
m_ImportedMemory = VK_NULL_HANDLE;
}
if (m_ImportedImage != VK_NULL_HANDLE)
{
vkDestroyImage(m_Device, m_ImportedImage, nullptr);
m_ImportedImage = VK_NULL_HANDLE;
}
}
}
void MosisVulkanTexture::DestroyLocalResources()
{
if (m_Device)
{
if (m_Sampler != VK_NULL_HANDLE)
{
vkDestroySampler(m_Device, m_Sampler, nullptr);
m_Sampler = VK_NULL_HANDLE;
}
if (m_LocalImageView != VK_NULL_HANDLE)
{
vkDestroyImageView(m_Device, m_LocalImageView, nullptr);
m_LocalImageView = VK_NULL_HANDLE;
}
if (m_LocalMemory != VK_NULL_HANDLE)
{
vkFreeMemory(m_Device, m_LocalMemory, nullptr);
m_LocalMemory = VK_NULL_HANDLE;
}
if (m_LocalImage != VK_NULL_HANDLE)
{
vkDestroyImage(m_Device, m_LocalImage, nullptr);
m_LocalImage = VK_NULL_HANDLE;
}
}
}
#endif // PLATFORM_ANDROID

View File

@@ -1,87 +0,0 @@
#pragma once
#if PLATFORM_ANDROID
#include "CoreMinimal.h"
#include <vulkan/vulkan.h>
#include <vulkan/vulkan_android.h>
#include <android/hardware_buffer.h>
/**
* MosisVulkanTexture - Imports AHardwareBuffer as Vulkan texture for Unreal.
* Creates both imported and local copy images for safe rendering.
*/
class MosisVulkanTexture
{
public:
MosisVulkanTexture() = default;
~MosisVulkanTexture();
/**
* Initialize with Vulkan device objects.
* Must be called before Create().
*/
bool Initialize(VkInstance instance, VkPhysicalDevice physDevice, VkDevice device, uint32_t queueFamilyIndex);
/**
* Create texture from a hardware buffer.
* @param buffer The AHardwareBuffer from the Mosis service
* @return true if creation succeeded
*/
bool Create(AHardwareBuffer* buffer);
/**
* Copy from imported image to local image.
* Call this each frame when a new frame is available.
*/
void CopyToLocal(VkCommandBuffer cmd);
/**
* Destroy all resources.
*/
void Destroy();
// Accessors
VkImage GetLocalImage() const { return m_LocalImage; }
VkImageView GetLocalImageView() const { return m_LocalImageView; }
VkFormat GetFormat() const { return m_Format; }
uint32_t GetWidth() const { return m_Width; }
uint32_t GetHeight() const { return m_Height; }
bool IsValid() const { return m_Created; }
private:
bool ImportHardwareBuffer(AHardwareBuffer* buffer);
bool CreateLocalImage();
void DestroyImportedResources();
void DestroyLocalResources();
uint32_t FindMemoryType(uint32_t typeFilter, VkMemoryPropertyFlags properties);
// Vulkan objects
VkInstance m_Instance = VK_NULL_HANDLE;
VkPhysicalDevice m_PhysicalDevice = VK_NULL_HANDLE;
VkDevice m_Device = VK_NULL_HANDLE;
uint32_t m_QueueFamilyIndex = 0;
// Imported from HardwareBuffer
VkImage m_ImportedImage = VK_NULL_HANDLE;
VkDeviceMemory m_ImportedMemory = VK_NULL_HANDLE;
// Local copy for safe rendering
VkImage m_LocalImage = VK_NULL_HANDLE;
VkDeviceMemory m_LocalMemory = VK_NULL_HANDLE;
VkImageView m_LocalImageView = VK_NULL_HANDLE;
VkSampler m_Sampler = VK_NULL_HANDLE;
// Format info
VkFormat m_Format = VK_FORMAT_R8G8B8A8_UNORM;
uint32_t m_Width = 0;
uint32_t m_Height = 0;
bool m_Initialized = false;
bool m_Created = false;
// Extension function pointer
PFN_vkGetAndroidHardwareBufferPropertiesANDROID vkGetAndroidHardwareBufferPropertiesANDROID = nullptr;
};
#endif // PLATFORM_ANDROID

View File

@@ -13,6 +13,9 @@ class UMaterialInstanceDynamic;
/**
* AMosisPhoneActor - Pre-configured actor for displaying the Mosis phone.
* Contains mesh, material, and phone component ready for use.
*
* By default, uses Engine's basic plane mesh. For custom phone models,
* set the PhoneMesh property to your mesh asset.
*/
UCLASS(BlueprintType, Blueprintable)
class MOSISSDK_API AMosisPhoneActor : public AActor
@@ -23,6 +26,7 @@ public:
AMosisPhoneActor();
virtual void BeginPlay() override;
virtual void Tick(float DeltaSeconds) override;
/**
* Send a touch event to the phone at world coordinates.
@@ -66,6 +70,21 @@ protected:
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis")
FVector4 ScreenBoundsLocal = FVector4(-50.0f, -100.0f, 50.0f, 100.0f);
/**
* Base material to use for the phone screen.
* Should have a texture parameter matching TextureParameterName on the PhoneComponent.
* If not set, a simple unlit emissive material will be created.
*/
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis")
TObjectPtr<UMaterialInterface> BaseMaterial;
/**
* Phone screen dimensions in world units.
* Default is 10x21.7 cm (typical smartphone aspect ratio 9:19.5)
*/
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis")
FVector2D ScreenSizeWorld = FVector2D(10.0f, 21.7f);
private:
/** Convert world hit to UV coordinates */
bool WorldToPhoneUV(FVector WorldLocation, FVector2D& OutUV) const;

View File

@@ -7,7 +7,7 @@
#include "MosisPhoneComponent.generated.h"
class UMaterialInstanceDynamic;
class UTexture2D;
class UMosisPhoneTexture;
/**
* Touch event type for phone interactions.
@@ -50,7 +50,7 @@ public:
* @return The texture showing the phone screen, or nullptr if not ready
*/
UFUNCTION(BlueprintCallable, Category = "Mosis")
UTexture2D* GetPhoneTexture() const { return PhoneTexture; }
UTexture* GetPhoneTexture() const;
/**
* Check if the phone is connected and ready.
@@ -90,20 +90,15 @@ protected:
private:
/** Phone screen texture */
UPROPERTY()
UTexture2D* PhoneTexture;
TObjectPtr<UMosisPhoneTexture> PhoneTexture;
/** Screen dimensions from service */
int32 ScreenWidth = 0;
int32 ScreenHeight = 0;
/** Flag to track pending texture updates */
bool bPendingTextureUpdate = false;
TAtomic<bool> bPendingTextureUpdate{false};
/** Flag to track if we need to recreate the texture */
bool bNeedsTextureCreate = false;
#if PLATFORM_ANDROID
/** Vulkan texture wrapper */
TSharedPtr<class MosisVulkanTexture> VulkanTexture;
#endif
TAtomic<bool> bNeedsTextureCreate{false};
};

View File

@@ -0,0 +1,67 @@
# Unreal Editor Python script to create the Mosis phone screen material
# Run this in the Editor: File > Execute Python Script
# Or from Python console: exec(open('Scripts/create_phone_material.py').read())
import unreal
def create_phone_screen_material():
"""Create an unlit emissive material for displaying the phone screen."""
asset_tools = unreal.AssetToolsHelpers.get_asset_tools()
material_factory = unreal.MaterialFactoryNew()
# Create the material asset
material_path = "/Game/Mosis/Materials"
material_name = "M_PhoneScreen"
# Check if material already exists
if unreal.EditorAssetLibrary.does_asset_exist(f"{material_path}/{material_name}"):
unreal.log_warning(f"Material {material_name} already exists, skipping creation")
return unreal.load_asset(f"{material_path}/{material_name}")
# Create the material
material = asset_tools.create_asset(
material_name,
material_path,
unreal.Material,
material_factory
)
if not material:
unreal.log_error("Failed to create material")
return None
# Configure material properties for phone screen display
material.set_editor_property("shading_model", unreal.MaterialShadingModel.MSM_UNLIT)
material.set_editor_property("blend_mode", unreal.BlendMode.BLEND_OPAQUE)
material.set_editor_property("two_sided", False)
# Get the material editor subsystem to add nodes
mel = unreal.MaterialEditingLibrary
# Create texture parameter node
texture_param = mel.create_material_expression(
material,
unreal.MaterialExpressionTextureSampleParameter2D,
-400, 0
)
texture_param.set_editor_property("parameter_name", "PhoneScreen")
# Connect texture RGB to emissive color
mel.connect_material_property(
texture_param, "RGB",
unreal.MaterialProperty.MP_EMISSIVE_COLOR
)
# Recompile the material
mel.recompile_material(material)
# Save the asset
unreal.EditorAssetLibrary.save_asset(f"{material_path}/{material_name}")
unreal.log(f"Created material: {material_path}/{material_name}")
return material
if __name__ == "__main__":
create_phone_screen_material()