Compare commits

..

6 Commits

37 changed files with 1134 additions and 597 deletions

View File

@@ -19,7 +19,7 @@ public static final XAPKFile[] xAPKS = {
new XAPKFile(
true, // true signifies a main file
"1", // the version of the APK that the file was uploaded against
97538444L // the length of the file in bytes
98985607L // the length of the file in bytes
)
};
};

File diff suppressed because one or more lines are too long

View File

@@ -85,7 +85,7 @@ bAllowClientSideNavigation=True
+ActiveGameNameRedirects=(OldGameName="/Script/TP_VirtualRealityBP",NewGameName="/Script/MosisUnreal")
[/Script/AndroidFileServerEditor.AndroidFileServerRuntimeSettings]
bEnablePlugin=True
bEnablePlugin=False
bAllowNetworkConnection=True
SecurityToken=40AD4D57409C43F8A8ADC6BE2AFF4735
bIncludeInShipping=False

View File

@@ -2,3 +2,6 @@
ProjectID=DD810CA045CA9288C9C53E8638A45978
bStartInVR=True
[/Script/UnrealEd.ProjectPackagingSettings]
+DirectoriesToAlwaysCook=(Path="/Game/Mosis")

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@@ -3,7 +3,7 @@
"Version": 1,
"VersionName": "1.0",
"FriendlyName": "MosisSDK",
"Description": "",
"Description": "Mosis Virtual Phone SDK for VR",
"Category": "Other",
"CreatedBy": "OmixLab LTD",
"CreatedByURL": "",
@@ -20,5 +20,11 @@
"Type": "Runtime",
"LoadingPhase": "Default"
}
],
"Plugins": [
{
"Name": "EnhancedInput",
"Enabled": true
}
]
}

View File

@@ -12,24 +12,33 @@ The MosisSDK plugin connects to the MosisService Android application via AIDL (A
## Architecture
```
┌─────────────────────────────────────────────────────────────┐
│ Unreal Engine Game │
┌─────────────────────┐ ┌─────────────────────────────┐
│ UMosisPhoneComponent│◄───│ MosisVulkanTexture │
│ │ (Touch Input) │ │ (HardwareBuffer Import) │ │
└──────────┬──────────┘ └──────────────▲──────────────┘
│ │
┌─────────────────────────────────────────┴───────────────┐
│ MosisClient ││
│ (AIDL IMosisListener)
└──────────────────────────┬──────────────────────────────
└─────────────────────────────┼───────────────────────────────┘
Binder IPC
─────────────────────────────▼───────────────────────────────┐
MosisService
(Renders phone UI via RmlUi)
─────────────────────────────────────────────────────────────
┌─────────────────────────────────────────────────────────────────────────
Unreal Engine Game
┌─────────────────┐ ┌──────────────────────────────────────────┐
│ │ AMosisPhoneActor│ │ UMosisPhoneTexture
│ (Plane Mesh) │ │ (extends UTexture2DDynamic) │
│ (Material) │ │
└────────┬────────┘ ▼ CPU lock/copy │
│ │ AHardwareBuffer_lock() │
▼ │ │
┌─────────────────────┐ │ ▼ RHIUpdateTexture2D
│ UMosisPhoneComponent│────► FTexture2DDynamicResource │
│ │ (Touch Input) │ │ │ │ │
(Callbacks) │ │ ▼
│ └────────────────────┘ │ UMaterialInstanceDynamic │ │
└──────────────────────────────────────────┘
│ ┌──────────────────────────────────────────────────────────────────┐ │
│ │ MosisClient │ │
│ │ (AIDL IMosisListener) │ │
│ └──────────────────────────────┬───────────────────────────────────┘ │
└─────────────────────────────────┼───────────────────────────────────────┘
│ Binder IPC
┌─────────────────────────────────▼───────────────────────────────────────┐
│ MosisService │
│ (Renders phone UI via RmlUi) │
└─────────────────────────────────────────────────────────────────────────┘
```
## Prerequisites
@@ -73,12 +82,12 @@ Plugins/MosisSDK/
│ │ └── BpMosisService.h
│ ├── Public/
│ │ ├── MosisSDK.h # Module interface
│ │ ├── MosisPhoneComponent.h # UE5 component
│ │ └── MosisPhoneActor.h # Blueprint actor
│ │ ├── MosisPhoneComponent.h # UE5 component (touch, callbacks)
│ │ └── MosisPhoneActor.h # Blueprint actor (mesh, material)
│ └── Private/
│ ├── MosisSDK.cpp # Module + JNI callbacks
│ ├── MosisClient.h/cpp # AIDL client implementation
│ ├── MosisVulkanTexture.h/cpp # Vulkan HardwareBuffer import
│ ├── MosisPhoneTexture.h/cpp # UTexture2DDynamic subclass for phone screen
│ ├── MosisPhoneComponent.cpp
│ ├── MosisPhoneActor.cpp
│ └── Android/
@@ -152,7 +161,8 @@ The separation ensures Windows builds don't try to compile Android-specific AIDL
| Slate, SlateCore | ✓ | ✓ | UI framework |
| RenderCore, RHI | ✓ | ✓ | Rendering abstraction |
| Launch, ApplicationCore | ✗ | ✓ | Android JNI access |
| Vulkan | ✗ | ✓ | Hardware buffer import |
| VulkanRHI | ✗ | ✓ | IVulkanDynamicRHI for HardwareBuffer import |
| Vulkan | ✗ | ✓ | Vulkan headers |
| binder_ndk, android, nativewindow | ✗ | ✓ | Android system libs |
### NDK Header Compatibility
@@ -353,6 +363,19 @@ HandTrackingVersion=V2
## Version History
- **v1.2** - UTexture2DDynamic integration
- Switched `UMosisPhoneTexture` to extend `UTexture2DDynamic` for proper material system integration
- Removed custom `FMosisPhoneTextureResource` - using UE5's built-in `FTexture2DDynamicResource`
- CPU-based texture upload via `AHardwareBuffer_lock()` and `RHIUpdateTexture2D()`
- Fixed stride conversion (AHardwareBuffer stride is in pixels, RHI expects bytes)
- **v1.1** - UE5 RHI texture integration
- New `UMosisPhoneTexture` using UE5's `IVulkanDynamicRHI::RHICreateTexture2DFromAndroidHardwareBuffer()`
- `FMosisPhoneTextureResource` for render thread HardwareBuffer import
- `AMosisPhoneActor` now includes default plane mesh and material setup
- Thread-safe callbacks using `TWeakObjectPtr` and `TAtomic`
- Automatic texture-to-material binding in tick
- **v1.0** - Initial implementation
- AIDL client for MosisService connection
- Vulkan HardwareBuffer import

View File

@@ -26,7 +26,9 @@ public class MosisSDK : ModuleRules
"Slate",
"SlateCore",
"RenderCore",
"RHI"
"RHI",
"EnhancedInput",
"InputCore"
}
);
@@ -35,7 +37,8 @@ public class MosisSDK : ModuleRules
// Android-specific module dependencies
PrivateDependencyModuleNames.AddRange(new string[] {
"Launch",
"ApplicationCore"
"ApplicationCore",
"VulkanRHI"
});
// Add Vulkan support (Vulkan headers for HardwareBuffer import)

View File

@@ -4,17 +4,41 @@
#include "Components/StaticMeshComponent.h"
#include "Materials/MaterialInstanceDynamic.h"
#include "Engine/StaticMesh.h"
#include "UObject/ConstructorHelpers.h"
DEFINE_LOG_CATEGORY_STATIC(LogMosisPhoneActor, Log, All);
AMosisPhoneActor::AMosisPhoneActor()
{
PrimaryActorTick.bCanEverTick = false;
PrimaryActorTick.bCanEverTick = true;
PrimaryActorTick.bStartWithTickEnabled = true;
// Create phone mesh component
PhoneMesh = CreateDefaultSubobject<UStaticMeshComponent>(TEXT("PhoneMesh"));
RootComponent = PhoneMesh;
// Configure collision for raycast detection
PhoneMesh->SetCollisionEnabled(ECollisionEnabled::QueryOnly);
PhoneMesh->SetCollisionResponseToAllChannels(ECR_Ignore);
PhoneMesh->SetCollisionResponseToChannel(ECC_Visibility, ECR_Block);
PhoneMesh->SetGenerateOverlapEvents(false);
// Load the default plane mesh
static ConstructorHelpers::FObjectFinder<UStaticMesh> PlaneMeshFinder(
TEXT("/Engine/BasicShapes/Plane.Plane"));
if (PlaneMeshFinder.Succeeded())
{
PhoneMesh->SetStaticMesh(PlaneMeshFinder.Object);
}
// Load the phone screen material
static ConstructorHelpers::FObjectFinder<UMaterialInterface> PhoneMaterialFinder(
TEXT("/Game/Mosis/Materials/M_PhoneScreen.M_PhoneScreen"));
if (PhoneMaterialFinder.Succeeded())
{
BaseMaterial = PhoneMaterialFinder.Object;
}
// Create phone component
PhoneComponent = CreateDefaultSubobject<UMosisPhoneComponent>(TEXT("PhoneComponent"));
}
@@ -25,13 +49,34 @@ void AMosisPhoneActor::BeginPlay()
UE_LOG(LogMosisPhoneActor, Log, TEXT("BeginPlay"));
// Note: We no longer override the scale set in the editor.
// The user can scale the actor as desired.
// ScreenBoundsLocal uses the mesh's local bounds (default plane is 100x100 centered at origin)
if (PhoneMesh)
{
// Default plane is 100x100 units, so bounds are -50 to 50 in X and Y
ScreenBoundsLocal = FVector4(-50.0f, -50.0f, 50.0f, 50.0f);
// Flip the mesh on Y axis to correct texture orientation
// (OpenGL renders with origin at bottom-left, Vulkan/UE5 expects top-left)
FVector CurrentScale = PhoneMesh->GetRelativeScale3D();
PhoneMesh->SetRelativeScale3D(FVector(CurrentScale.X, -CurrentScale.Y, CurrentScale.Z));
}
// Create dynamic material for the screen
if (PhoneMesh)
{
UMaterialInterface* BaseMaterial = PhoneMesh->GetMaterial(0);
if (BaseMaterial)
UMaterialInterface* MaterialToUse = BaseMaterial;
// If no base material set, use the mesh's existing material
if (!MaterialToUse)
{
ScreenMaterial = UMaterialInstanceDynamic::Create(BaseMaterial, this);
MaterialToUse = PhoneMesh->GetMaterial(0);
}
if (MaterialToUse)
{
ScreenMaterial = UMaterialInstanceDynamic::Create(MaterialToUse, this);
PhoneMesh->SetMaterial(0, ScreenMaterial);
// Set it on the phone component
@@ -42,6 +87,25 @@ void AMosisPhoneActor::BeginPlay()
UE_LOG(LogMosisPhoneActor, Log, TEXT("BeginPlay: Created dynamic material"));
}
else
{
UE_LOG(LogMosisPhoneActor, Warning, TEXT("BeginPlay: No base material available"));
}
}
}
void AMosisPhoneActor::Tick(float DeltaSeconds)
{
Super::Tick(DeltaSeconds);
// Update material texture if phone component has a new texture
if (ScreenMaterial && PhoneComponent)
{
UTexture* PhoneTexture = PhoneComponent->GetPhoneTexture();
if (PhoneTexture)
{
ScreenMaterial->SetTextureParameterValue(PhoneComponent->TextureParameterName, PhoneTexture);
}
}
}
@@ -84,5 +148,8 @@ bool AMosisPhoneActor::WorldToPhoneUV(FVector WorldLocation, FVector2D& OutUV) c
OutUV.X = (LocalLocation.X - MinX) / (MaxX - MinX);
OutUV.Y = (LocalLocation.Y - MinY) / (MaxY - MinY);
// Flip Y to match the flipped mesh scale (corrects for OpenGL vs Vulkan coordinate systems)
OutUV.Y = 1.0f - OutUV.Y;
return true;
}

View File

@@ -2,13 +2,12 @@
#include "MosisPhoneComponent.h"
#include "MosisSDK.h"
#include "Engine/Texture2D.h"
#include "MosisPhoneTexture.h"
#include "Materials/MaterialInstanceDynamic.h"
#include "RenderingThread.h"
#if PLATFORM_ANDROID
#include "MosisClient.h"
#include "MosisVulkanTexture.h"
#endif
DEFINE_LOG_CATEGORY_STATIC(LogMosisPhone, Log, All);
@@ -30,18 +29,32 @@ void UMosisPhoneComponent::BeginPlay()
auto Client = FMosisSDKModule::GetClient();
if (Client)
{
// Set up callbacks
Client->SetBufferCallback([this](AHardwareBuffer* buffer) {
// This runs on the binder thread - flag for main thread processing
bNeedsTextureCreate = true;
// Use weak pointer to prevent dangling references in callbacks
TWeakObjectPtr<UMosisPhoneComponent> WeakThis(this);
// Set up callbacks (these run on binder thread)
Client->SetBufferCallback([WeakThis](AHardwareBuffer* buffer) {
if (WeakThis.IsValid())
{
WeakThis->bNeedsTextureCreate.Store(true);
}
});
Client->SetFrameCallback([this]() {
// This runs on the binder thread - flag for tick processing
bPendingTextureUpdate = true;
Client->SetFrameCallback([WeakThis]() {
if (WeakThis.IsValid())
{
WeakThis->bPendingTextureUpdate.Store(true);
}
});
UE_LOG(LogMosisPhone, Log, TEXT("BeginPlay: Callbacks registered"));
// Check if buffer is already available (we may have missed the callback)
if (Client->GetHardwareBuffer() != nullptr)
{
UE_LOG(LogMosisPhone, Log, TEXT("BeginPlay: Buffer already available, triggering update"));
bNeedsTextureCreate.Store(true);
}
}
else
{
@@ -55,12 +68,6 @@ void UMosisPhoneComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
UE_LOG(LogMosisPhone, Log, TEXT("EndPlay"));
#if PLATFORM_ANDROID
// Clean up Vulkan texture
if (VulkanTexture)
{
VulkanTexture.Reset();
}
// Clear callbacks
auto Client = FMosisSDKModule::GetClient();
if (Client)
@@ -70,6 +77,9 @@ void UMosisPhoneComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
}
#endif
// Release texture
PhoneTexture = nullptr;
Super::EndPlay(EndPlayReason);
}
@@ -79,16 +89,14 @@ void UMosisPhoneComponent::TickComponent(float DeltaTime, ELevelTick TickType, F
#if PLATFORM_ANDROID
// Check if we need to create texture from new buffer
if (bNeedsTextureCreate)
if (bNeedsTextureCreate.Exchange(false))
{
bNeedsTextureCreate = false;
OnBufferAvailable();
}
// Check if we need to update texture for new frame
if (bPendingTextureUpdate && VulkanTexture && VulkanTexture->IsValid())
if (bPendingTextureUpdate.Exchange(false) && PhoneTexture)
{
bPendingTextureUpdate = false;
OnFrameAvailable();
}
#endif
@@ -155,25 +163,17 @@ void UMosisPhoneComponent::OnBufferAvailable()
UE_LOG(LogMosisPhone, Log, TEXT("OnBufferAvailable: %dx%d"), ScreenWidth, ScreenHeight);
// TODO: Initialize Vulkan texture on render thread
// This requires accessing UE5's Vulkan RHI which has platform-specific APIs.
// For now, we log the buffer availability and dimensions.
// Full implementation would:
// 1. Get VkDevice from GVulkanRHI or IVulkanDynamicRHI
// 2. Create MosisVulkanTexture and import the HardwareBuffer
// 3. Create UTexture2D wrapping the Vulkan image
// Create placeholder texture
// Create phone texture if needed
if (!PhoneTexture)
{
PhoneTexture = UTexture2D::CreateTransient(ScreenWidth, ScreenHeight, PF_R8G8B8A8);
if (PhoneTexture)
{
PhoneTexture->UpdateResource();
UE_LOG(LogMosisPhone, Log, TEXT("OnBufferAvailable: Created placeholder texture"));
}
PhoneTexture = NewObject<UMosisPhoneTexture>(this);
PhoneTexture->Initialize(ScreenWidth, ScreenHeight);
UE_LOG(LogMosisPhone, Log, TEXT("OnBufferAvailable: Created phone texture"));
}
// Import the hardware buffer for GPU-accelerated updates
PhoneTexture->ImportHardwareBuffer(Buffer);
// Update material if set
if (PhoneMaterial && PhoneTexture)
{
@@ -185,19 +185,29 @@ void UMosisPhoneComponent::OnBufferAvailable()
void UMosisPhoneComponent::OnFrameAvailable()
{
#if PLATFORM_ANDROID
if (!VulkanTexture || !VulkanTexture->IsValid())
if (!PhoneTexture || !PhoneTexture->HasImportedBuffer())
{
return;
}
// TODO: Copy texture on render thread
// This requires accessing UE5's Vulkan command buffer management.
// Full implementation would enqueue a render command to copy the
// imported image to the local image.
// Perform GPU-to-GPU copy from imported hardware buffer
PhoneTexture->CopyFromImportedBuffer();
// Log occasionally to avoid spam
static int32 FrameCount = 0;
if (++FrameCount % 60 == 0)
{
UE_LOG(LogMosisPhone, Log, TEXT("OnFrameAvailable: Frame %d (GPU copy)"), FrameCount);
}
#endif
}
void UMosisPhoneComponent::UpdateTextureOnRenderThread()
{
// Reserved for render thread texture updates
// No longer needed - handled by UMosisPhoneTexture
}
UTexture* UMosisPhoneComponent::GetPhoneTexture() const
{
return PhoneTexture;
}

View File

@@ -0,0 +1,160 @@
// Copyright OmixLab LTD. All Rights Reserved.
#include "MosisPhoneTexture.h"
#include "RenderingThread.h"
#include "RHICommandList.h"
#if PLATFORM_ANDROID
#include "IVulkanDynamicRHI.h"
#endif
DEFINE_LOG_CATEGORY_STATIC(LogMosisPhoneTexture, Log, All);
UMosisPhoneTexture::UMosisPhoneTexture()
{
SRGB = true;
Filter = TF_Bilinear;
}
UMosisPhoneTexture::~UMosisPhoneTexture()
{
#if PLATFORM_ANDROID
// Release imported texture RHI resource
if (ImportedTextureRHI.IsValid())
{
ImportedTextureRHI.SafeRelease();
}
CurrentBuffer = nullptr;
#endif
}
void UMosisPhoneTexture::Initialize(uint32 InWidth, uint32 InHeight)
{
UE_LOG(LogMosisPhoneTexture, Log, TEXT("Initialize: %dx%d"), InWidth, InHeight);
TextureWidth = InWidth;
TextureHeight = InHeight;
// Initialize the UTexture2DDynamic base - this creates our destination texture
Init(InWidth, InHeight, PF_R8G8B8A8, false);
bIsReady = true;
}
#if PLATFORM_ANDROID
void UMosisPhoneTexture::ImportHardwareBuffer(AHardwareBuffer* Buffer)
{
if (!Buffer)
{
UE_LOG(LogMosisPhoneTexture, Warning, TEXT("ImportHardwareBuffer: null buffer"));
return;
}
// Get buffer dimensions
AHardwareBuffer_Desc Desc{};
AHardwareBuffer_describe(Buffer, &Desc);
UE_LOG(LogMosisPhoneTexture, Log, TEXT("ImportHardwareBuffer: %dx%d, format=%d, usage=0x%x"),
Desc.width, Desc.height, Desc.format, Desc.usage);
// Verify buffer has GPU sampled image usage (required for Vulkan import)
if ((Desc.usage & AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE) == 0)
{
UE_LOG(LogMosisPhoneTexture, Error,
TEXT("ImportHardwareBuffer: Buffer missing AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE flag"));
return;
}
// Check if RHI is Vulkan
if (GDynamicRHI->GetInterfaceType() != ERHIInterfaceType::Vulkan)
{
UE_LOG(LogMosisPhoneTexture, Error,
TEXT("ImportHardwareBuffer: Vulkan RHI required for hardware buffer import"));
return;
}
// Release old imported texture if any
if (ImportedTextureRHI.IsValid())
{
ImportedTextureRHI.SafeRelease();
}
CurrentBuffer = Buffer;
// Import the hardware buffer using UE5's Vulkan RHI API
// This creates a VkImage backed by the AHardwareBuffer's shared memory (zero-copy)
IVulkanDynamicRHI* VulkanRHI = GetIVulkanDynamicRHI();
ImportedTextureRHI = VulkanRHI->RHICreateTexture2DFromAndroidHardwareBuffer(Buffer);
if (ImportedTextureRHI.IsValid())
{
UE_LOG(LogMosisPhoneTexture, Log,
TEXT("ImportHardwareBuffer: Successfully imported as Vulkan texture"));
// Initialize our destination texture if not done yet
if (!bIsReady)
{
Initialize(Desc.width, Desc.height);
}
}
else
{
UE_LOG(LogMosisPhoneTexture, Error,
TEXT("ImportHardwareBuffer: Failed to create Vulkan texture from hardware buffer"));
}
}
void UMosisPhoneTexture::CopyFromImportedBuffer()
{
if (!ImportedTextureRHI.IsValid() || !bIsReady)
{
return;
}
// Get our destination texture resource
FTexture2DDynamicResource* TextureResource = static_cast<FTexture2DDynamicResource*>(GetResource());
if (!TextureResource)
{
return;
}
FTextureRHIRef DestTextureRHI = TextureResource->GetTexture2DRHI();
if (!DestTextureRHI.IsValid())
{
return;
}
// Capture references for the render thread lambda
FTextureRHIRef SrcTexture = ImportedTextureRHI;
FTextureRHIRef DstTexture = DestTextureRHI;
uint32 Width = TextureWidth;
uint32 Height = TextureHeight;
// Enqueue GPU-to-GPU copy on render thread
ENQUEUE_RENDER_COMMAND(CopyMosisTexture)(
[SrcTexture, DstTexture, Width, Height](FRHICommandListImmediate& RHICmdList)
{
// Transition source to copy source state
RHICmdList.Transition(FRHITransitionInfo(SrcTexture, ERHIAccess::Unknown, ERHIAccess::CopySrc));
// Transition destination to copy dest state
RHICmdList.Transition(FRHITransitionInfo(DstTexture, ERHIAccess::Unknown, ERHIAccess::CopyDest));
// Perform GPU copy
FRHICopyTextureInfo CopyInfo;
CopyInfo.Size.X = Width;
CopyInfo.Size.Y = Height;
CopyInfo.Size.Z = 1;
CopyInfo.NumMips = 1;
CopyInfo.NumSlices = 1;
RHICmdList.CopyTexture(SrcTexture, DstTexture, CopyInfo);
// Transition destination back to shader resource for rendering
RHICmdList.Transition(FRHITransitionInfo(DstTexture, ERHIAccess::CopyDest, ERHIAccess::SRVMask));
}
);
}
#endif // PLATFORM_ANDROID

View File

@@ -0,0 +1,68 @@
// Copyright OmixLab LTD. All Rights Reserved.
#pragma once
#include "CoreMinimal.h"
#include "Engine/Texture2DDynamic.h"
#include "MosisPhoneTexture.generated.h"
#if PLATFORM_ANDROID
#include <android/hardware_buffer.h>
#endif
/**
* UMosisPhoneTexture - Dynamic texture that displays the Mosis phone screen.
*
* This texture is updated from an AHardwareBuffer received from MosisService.
* Uses GPU-to-GPU copy via Vulkan external memory import for optimal performance.
* Inherits from UTexture2DDynamic for proper material integration.
*/
UCLASS()
class UMosisPhoneTexture : public UTexture2DDynamic
{
GENERATED_BODY()
public:
UMosisPhoneTexture();
virtual ~UMosisPhoneTexture();
/** Initialize the texture with dimensions */
void Initialize(uint32 InWidth, uint32 InHeight);
#if PLATFORM_ANDROID
/**
* Import a hardware buffer for GPU-accelerated texture updates.
* Creates a Vulkan image from the AHardwareBuffer for zero-copy import.
* @param Buffer The AHardwareBuffer from MosisService
*/
void ImportHardwareBuffer(AHardwareBuffer* Buffer);
/**
* Perform GPU-to-GPU copy from imported buffer to this texture.
* Called each frame when new content is available.
*/
void CopyFromImportedBuffer();
/** Check if the imported buffer is valid */
bool HasImportedBuffer() const { return ImportedTextureRHI.IsValid(); }
#endif
/** Check if the texture is ready for rendering */
bool IsReady() const { return bIsReady; }
private:
#if PLATFORM_ANDROID
/** RHI texture imported from AHardwareBuffer (zero-copy Vulkan import) */
FTextureRHIRef ImportedTextureRHI;
/** Current hardware buffer reference */
AHardwareBuffer* CurrentBuffer = nullptr;
#endif
/** True when texture has been initialized */
bool bIsReady = false;
/** Texture dimensions */
uint32 TextureWidth = 0;
uint32 TextureHeight = 0;
};

View File

@@ -0,0 +1,236 @@
// Copyright Epic Games, Inc. All Rights Reserved.
#include "MosisPointerComponent.h"
#include "MosisPhoneActor.h"
#include "EnhancedInputComponent.h"
#include "EnhancedInputSubsystems.h"
#include "InputAction.h"
#include "GameFramework/PlayerController.h"
#include "DrawDebugHelpers.h"
DEFINE_LOG_CATEGORY_STATIC(LogMosisPointer, Log, All);
UMosisPointerComponent::UMosisPointerComponent()
{
PrimaryComponentTick.bCanEverTick = true;
PrimaryComponentTick.bStartWithTickEnabled = true;
}
void UMosisPointerComponent::BeginPlay()
{
Super::BeginPlay();
// Try to setup input bindings after a short delay to ensure player controller is ready
if (TriggerAction)
{
SetupInputBindings();
}
UE_LOG(LogMosisPointer, Log, TEXT("MosisPointerComponent initialized. RayLength: %.1f"), RayLength);
}
void UMosisPointerComponent::SetupInputBindings()
{
if (bInputBound)
{
return;
}
AActor* Owner = GetOwner();
if (!Owner)
{
return;
}
// Find the owning pawn and its controller
APawn* OwningPawn = Cast<APawn>(Owner);
if (!OwningPawn)
{
OwningPawn = Owner->GetInstigator<APawn>();
}
if (!OwningPawn)
{
UE_LOG(LogMosisPointer, Warning, TEXT("Could not find owning pawn for input binding"));
return;
}
APlayerController* PC = Cast<APlayerController>(OwningPawn->GetController());
if (!PC)
{
UE_LOG(LogMosisPointer, Warning, TEXT("Could not find player controller for input binding"));
return;
}
// Get the Enhanced Input component from the pawn
UEnhancedInputComponent* EnhancedInput = Cast<UEnhancedInputComponent>(OwningPawn->InputComponent);
if (!EnhancedInput)
{
UE_LOG(LogMosisPointer, Warning, TEXT("Could not find Enhanced Input component on pawn"));
return;
}
// Bind the trigger action
EnhancedInput->BindAction(TriggerAction, ETriggerEvent::Triggered, this, &UMosisPointerComponent::OnTriggerAction);
EnhancedInput->BindAction(TriggerAction, ETriggerEvent::Completed, this, &UMosisPointerComponent::OnTriggerAction);
bInputBound = true;
UE_LOG(LogMosisPointer, Log, TEXT("Successfully bound trigger input action"));
}
void UMosisPointerComponent::OnTriggerAction(const FInputActionInstance& Instance)
{
// Triggered = pressed, Completed = released
bIsTriggerPressed = (Instance.GetTriggerEvent() == ETriggerEvent::Triggered);
}
void UMosisPointerComponent::SetTriggerPressed(bool bPressed)
{
bIsTriggerPressed = bPressed;
}
void UMosisPointerComponent::TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction)
{
Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
// Try to bind input if not yet bound and we have an action
if (TriggerAction && !bInputBound)
{
SetupInputBindings();
}
// Perform raycast to find phone
PerformRaycast();
// Update touch state and send events
UpdateTouchState();
// Draw debug visualization
if (bShowDebugRay)
{
DrawDebugRay();
}
}
void UMosisPointerComponent::PerformRaycast()
{
FVector Start = GetComponentLocation();
FVector Direction = GetForwardVector();
FVector End = Start + Direction * RayLength;
FHitResult Hit;
FCollisionQueryParams Params;
Params.AddIgnoredActor(GetOwner());
// Also ignore any parent actors
AActor* Parent = GetOwner() ? GetOwner()->GetAttachParentActor() : nullptr;
while (Parent)
{
Params.AddIgnoredActor(Parent);
Parent = Parent->GetAttachParentActor();
}
bool bHit = GetWorld()->LineTraceSingleByChannel(
Hit, Start, End, TraceChannel, Params);
if (bHit)
{
AMosisPhoneActor* HitPhone = Cast<AMosisPhoneActor>(Hit.GetActor());
if (HitPhone)
{
CurrentPhone = HitPhone;
CurrentHitLocation = Hit.ImpactPoint;
bIsOverPhone = true;
}
else
{
CurrentPhone = nullptr;
bIsOverPhone = false;
}
}
else
{
CurrentPhone = nullptr;
bIsOverPhone = false;
}
}
void UMosisPointerComponent::UpdateTouchState()
{
// Detect state transitions
bool bJustPressed = bIsTriggerPressed && !bWasTriggerPressed;
bool bJustReleased = !bIsTriggerPressed && bWasTriggerPressed;
if (bIsOverPhone && CurrentPhone.IsValid())
{
if (bJustPressed)
{
// Touch down
CurrentPhone->SendTouchAtWorldLocation(CurrentHitLocation, EMosisTouchType::Down);
LastTouchedPhone = CurrentPhone;
LastHitLocation = CurrentHitLocation;
UE_LOG(LogMosisPointer, Verbose, TEXT("Touch Down at (%.1f, %.1f, %.1f)"),
CurrentHitLocation.X, CurrentHitLocation.Y, CurrentHitLocation.Z);
}
else if (bIsTriggerPressed)
{
// Touch move (only if position changed significantly)
if (FVector::DistSquared(CurrentHitLocation, LastHitLocation) > 0.01f)
{
CurrentPhone->SendTouchAtWorldLocation(CurrentHitLocation, EMosisTouchType::Move);
LastHitLocation = CurrentHitLocation;
}
}
else if (bJustReleased)
{
// Touch up
CurrentPhone->SendTouchAtWorldLocation(CurrentHitLocation, EMosisTouchType::Up);
LastTouchedPhone = nullptr;
UE_LOG(LogMosisPointer, Verbose, TEXT("Touch Up at (%.1f, %.1f, %.1f)"),
CurrentHitLocation.X, CurrentHitLocation.Y, CurrentHitLocation.Z);
}
}
else if (bJustReleased && LastTouchedPhone.IsValid())
{
// Ray moved off phone while pressed, but trigger was just released
// Send up event to the last touched phone
LastTouchedPhone->SendTouchAtWorldLocation(LastHitLocation, EMosisTouchType::Up);
LastTouchedPhone = nullptr;
UE_LOG(LogMosisPointer, Verbose, TEXT("Touch Up (off-phone) at (%.1f, %.1f, %.1f)"),
LastHitLocation.X, LastHitLocation.Y, LastHitLocation.Z);
}
bWasTriggerPressed = bIsTriggerPressed;
}
void UMosisPointerComponent::DrawDebugRay() const
{
FVector Start = GetComponentLocation();
FVector Direction = GetForwardVector();
FVector End = bIsOverPhone ? CurrentHitLocation : (Start + Direction * RayLength);
FColor Color = bIsOverPhone ? DebugRayHitColor.ToFColor(true) : DebugRayColor.ToFColor(true);
DrawDebugLine(GetWorld(), Start, End, Color, false, -1.0f, 0, 1.0f);
if (bIsOverPhone)
{
// Draw hit point
DrawDebugSphere(GetWorld(), CurrentHitLocation, 2.0f, 8, Color, false, -1.0f, 0, 0.5f);
}
}
AMosisPhoneActor* UMosisPointerComponent::GetTargetPhone() const
{
return CurrentPhone.Get();
}
FVector UMosisPointerComponent::GetRayOrigin() const
{
return GetComponentLocation();
}
FVector UMosisPointerComponent::GetRayDirection() const
{
return GetForwardVector();
}

View File

@@ -1,421 +0,0 @@
#include "MosisVulkanTexture.h"
#if PLATFORM_ANDROID
#include "Logging/LogMacros.h"
DEFINE_LOG_CATEGORY_STATIC(LogMosisVulkan, Log, All);
MosisVulkanTexture::~MosisVulkanTexture()
{
Destroy();
}
bool MosisVulkanTexture::Initialize(VkInstance instance, VkPhysicalDevice physDevice, VkDevice device, uint32_t queueFamilyIndex)
{
if (m_Initialized)
{
return true;
}
m_Instance = instance;
m_PhysicalDevice = physDevice;
m_Device = device;
m_QueueFamilyIndex = queueFamilyIndex;
// Load extension function
vkGetAndroidHardwareBufferPropertiesANDROID =
reinterpret_cast<PFN_vkGetAndroidHardwareBufferPropertiesANDROID>(
vkGetInstanceProcAddr(m_Instance, "vkGetAndroidHardwareBufferPropertiesANDROID")
);
if (!vkGetAndroidHardwareBufferPropertiesANDROID)
{
UE_LOG(LogMosisVulkan, Error, TEXT("Initialize: vkGetAndroidHardwareBufferPropertiesANDROID not available"));
return false;
}
m_Initialized = true;
UE_LOG(LogMosisVulkan, Log, TEXT("Initialize: Success"));
return true;
}
bool MosisVulkanTexture::Create(AHardwareBuffer* buffer)
{
if (!m_Initialized)
{
UE_LOG(LogMosisVulkan, Error, TEXT("Create: Not initialized"));
return false;
}
if (m_Created)
{
Destroy();
}
// Get buffer dimensions
AHardwareBuffer_Desc desc{};
AHardwareBuffer_describe(buffer, &desc);
m_Width = desc.width;
m_Height = desc.height;
UE_LOG(LogMosisVulkan, Log, TEXT("Create: %dx%d texture"), m_Width, m_Height);
if (!ImportHardwareBuffer(buffer))
{
UE_LOG(LogMosisVulkan, Error, TEXT("Create: Failed to import hardware buffer"));
return false;
}
if (!CreateLocalImage())
{
UE_LOG(LogMosisVulkan, Error, TEXT("Create: Failed to create local image"));
DestroyImportedResources();
return false;
}
m_Created = true;
UE_LOG(LogMosisVulkan, Log, TEXT("Create: Success"));
return true;
}
bool MosisVulkanTexture::ImportHardwareBuffer(AHardwareBuffer* buffer)
{
// Query hardware buffer properties
VkAndroidHardwareBufferFormatPropertiesANDROID formatProps{};
formatProps.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_FORMAT_PROPERTIES_ANDROID;
VkAndroidHardwareBufferPropertiesANDROID props{};
props.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID;
props.pNext = &formatProps;
if (vkGetAndroidHardwareBufferPropertiesANDROID(m_Device, buffer, &props) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("ImportHardwareBuffer: Failed to get properties"));
return false;
}
m_Format = formatProps.format;
UE_LOG(LogMosisVulkan, Log, TEXT("ImportHardwareBuffer: Format=%d"), static_cast<int>(m_Format));
// Create VkImage with external memory
VkExternalMemoryImageCreateInfo extMemImageInfo{};
extMemImageInfo.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO;
extMemImageInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID;
VkImageCreateInfo imageInfo{};
imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
imageInfo.pNext = &extMemImageInfo;
imageInfo.imageType = VK_IMAGE_TYPE_2D;
imageInfo.format = m_Format;
imageInfo.extent = {m_Width, m_Height, 1};
imageInfo.mipLevels = 1;
imageInfo.arrayLayers = 1;
imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
imageInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
if (vkCreateImage(m_Device, &imageInfo, nullptr, &m_ImportedImage) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("ImportHardwareBuffer: Failed to create image"));
return false;
}
// Import memory from hardware buffer
VkImportAndroidHardwareBufferInfoANDROID importInfo{};
importInfo.sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID;
importInfo.buffer = buffer;
VkMemoryDedicatedAllocateInfo dedicatedInfo{};
dedicatedInfo.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO;
dedicatedInfo.pNext = &importInfo;
dedicatedInfo.image = m_ImportedImage;
VkMemoryAllocateInfo allocInfo{};
allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
allocInfo.pNext = &dedicatedInfo;
allocInfo.allocationSize = props.allocationSize;
allocInfo.memoryTypeIndex = FindMemoryType(props.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
if (vkAllocateMemory(m_Device, &allocInfo, nullptr, &m_ImportedMemory) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("ImportHardwareBuffer: Failed to allocate memory"));
vkDestroyImage(m_Device, m_ImportedImage, nullptr);
m_ImportedImage = VK_NULL_HANDLE;
return false;
}
if (vkBindImageMemory(m_Device, m_ImportedImage, m_ImportedMemory, 0) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("ImportHardwareBuffer: Failed to bind memory"));
DestroyImportedResources();
return false;
}
UE_LOG(LogMosisVulkan, Log, TEXT("ImportHardwareBuffer: Success"));
return true;
}
bool MosisVulkanTexture::CreateLocalImage()
{
// Create local VkImage
VkImageCreateInfo imageInfo{};
imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
imageInfo.imageType = VK_IMAGE_TYPE_2D;
imageInfo.format = m_Format;
imageInfo.extent = {m_Width, m_Height, 1};
imageInfo.mipLevels = 1;
imageInfo.arrayLayers = 1;
imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
imageInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
if (vkCreateImage(m_Device, &imageInfo, nullptr, &m_LocalImage) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to create image"));
return false;
}
// Allocate memory
VkMemoryRequirements memReqs;
vkGetImageMemoryRequirements(m_Device, m_LocalImage, &memReqs);
VkMemoryAllocateInfo allocInfo{};
allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
allocInfo.allocationSize = memReqs.size;
allocInfo.memoryTypeIndex = FindMemoryType(memReqs.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
if (vkAllocateMemory(m_Device, &allocInfo, nullptr, &m_LocalMemory) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to allocate memory"));
vkDestroyImage(m_Device, m_LocalImage, nullptr);
m_LocalImage = VK_NULL_HANDLE;
return false;
}
if (vkBindImageMemory(m_Device, m_LocalImage, m_LocalMemory, 0) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to bind memory"));
DestroyLocalResources();
return false;
}
// Create image view
VkImageViewCreateInfo viewInfo{};
viewInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
viewInfo.image = m_LocalImage;
viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D;
viewInfo.format = m_Format;
viewInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
viewInfo.subresourceRange.baseMipLevel = 0;
viewInfo.subresourceRange.levelCount = 1;
viewInfo.subresourceRange.baseArrayLayer = 0;
viewInfo.subresourceRange.layerCount = 1;
if (vkCreateImageView(m_Device, &viewInfo, nullptr, &m_LocalImageView) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to create image view"));
DestroyLocalResources();
return false;
}
// Create sampler
VkSamplerCreateInfo samplerInfo{};
samplerInfo.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO;
samplerInfo.magFilter = VK_FILTER_LINEAR;
samplerInfo.minFilter = VK_FILTER_LINEAR;
samplerInfo.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
samplerInfo.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
samplerInfo.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
samplerInfo.anisotropyEnable = VK_FALSE;
samplerInfo.maxAnisotropy = 1.0f;
samplerInfo.borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK;
samplerInfo.unnormalizedCoordinates = VK_FALSE;
samplerInfo.compareEnable = VK_FALSE;
samplerInfo.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR;
if (vkCreateSampler(m_Device, &samplerInfo, nullptr, &m_Sampler) != VK_SUCCESS)
{
UE_LOG(LogMosisVulkan, Error, TEXT("CreateLocalImage: Failed to create sampler"));
DestroyLocalResources();
return false;
}
UE_LOG(LogMosisVulkan, Log, TEXT("CreateLocalImage: Success"));
return true;
}
void MosisVulkanTexture::CopyToLocal(VkCommandBuffer cmd)
{
if (!m_Created)
{
return;
}
// Transition imported image to transfer src
VkImageMemoryBarrier srcBarrier{};
srcBarrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
srcBarrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
srcBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
srcBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
srcBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
srcBarrier.image = m_ImportedImage;
srcBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
srcBarrier.subresourceRange.baseMipLevel = 0;
srcBarrier.subresourceRange.levelCount = 1;
srcBarrier.subresourceRange.baseArrayLayer = 0;
srcBarrier.subresourceRange.layerCount = 1;
srcBarrier.srcAccessMask = 0;
srcBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
vkCmdPipelineBarrier(
cmd,
VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT,
0, 0, nullptr, 0, nullptr, 1, &srcBarrier
);
// Transition local image to transfer dst
VkImageMemoryBarrier dstBarrier{};
dstBarrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
dstBarrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
dstBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
dstBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
dstBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
dstBarrier.image = m_LocalImage;
dstBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
dstBarrier.subresourceRange.baseMipLevel = 0;
dstBarrier.subresourceRange.levelCount = 1;
dstBarrier.subresourceRange.baseArrayLayer = 0;
dstBarrier.subresourceRange.layerCount = 1;
dstBarrier.srcAccessMask = 0;
dstBarrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
vkCmdPipelineBarrier(
cmd,
VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT,
0, 0, nullptr, 0, nullptr, 1, &dstBarrier
);
// Copy image
VkImageCopy copyRegion{};
copyRegion.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
copyRegion.srcSubresource.layerCount = 1;
copyRegion.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
copyRegion.dstSubresource.layerCount = 1;
copyRegion.extent = {m_Width, m_Height, 1};
vkCmdCopyImage(
cmd,
m_ImportedImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
m_LocalImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
1, &copyRegion
);
// Transition local image to shader read
VkImageMemoryBarrier shaderBarrier{};
shaderBarrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
shaderBarrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
shaderBarrier.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
shaderBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
shaderBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
shaderBarrier.image = m_LocalImage;
shaderBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
shaderBarrier.subresourceRange.baseMipLevel = 0;
shaderBarrier.subresourceRange.levelCount = 1;
shaderBarrier.subresourceRange.baseArrayLayer = 0;
shaderBarrier.subresourceRange.layerCount = 1;
shaderBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
shaderBarrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
vkCmdPipelineBarrier(
cmd,
VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
0, 0, nullptr, 0, nullptr, 1, &shaderBarrier
);
}
uint32_t MosisVulkanTexture::FindMemoryType(uint32_t typeFilter, VkMemoryPropertyFlags properties)
{
VkPhysicalDeviceMemoryProperties memProps;
vkGetPhysicalDeviceMemoryProperties(m_PhysicalDevice, &memProps);
for (uint32_t i = 0; i < memProps.memoryTypeCount; i++)
{
if ((typeFilter & (1 << i)) &&
(memProps.memoryTypes[i].propertyFlags & properties) == properties)
{
return i;
}
}
UE_LOG(LogMosisVulkan, Error, TEXT("FindMemoryType: Failed to find suitable memory type"));
return 0;
}
void MosisVulkanTexture::Destroy()
{
if (m_Device && m_Created)
{
vkDeviceWaitIdle(m_Device);
}
DestroyLocalResources();
DestroyImportedResources();
m_Width = 0;
m_Height = 0;
m_Created = false;
}
void MosisVulkanTexture::DestroyImportedResources()
{
if (m_Device)
{
if (m_ImportedMemory != VK_NULL_HANDLE)
{
vkFreeMemory(m_Device, m_ImportedMemory, nullptr);
m_ImportedMemory = VK_NULL_HANDLE;
}
if (m_ImportedImage != VK_NULL_HANDLE)
{
vkDestroyImage(m_Device, m_ImportedImage, nullptr);
m_ImportedImage = VK_NULL_HANDLE;
}
}
}
void MosisVulkanTexture::DestroyLocalResources()
{
if (m_Device)
{
if (m_Sampler != VK_NULL_HANDLE)
{
vkDestroySampler(m_Device, m_Sampler, nullptr);
m_Sampler = VK_NULL_HANDLE;
}
if (m_LocalImageView != VK_NULL_HANDLE)
{
vkDestroyImageView(m_Device, m_LocalImageView, nullptr);
m_LocalImageView = VK_NULL_HANDLE;
}
if (m_LocalMemory != VK_NULL_HANDLE)
{
vkFreeMemory(m_Device, m_LocalMemory, nullptr);
m_LocalMemory = VK_NULL_HANDLE;
}
if (m_LocalImage != VK_NULL_HANDLE)
{
vkDestroyImage(m_Device, m_LocalImage, nullptr);
m_LocalImage = VK_NULL_HANDLE;
}
}
}
#endif // PLATFORM_ANDROID

View File

@@ -1,87 +0,0 @@
#pragma once
#if PLATFORM_ANDROID
#include "CoreMinimal.h"
#include <vulkan/vulkan.h>
#include <vulkan/vulkan_android.h>
#include <android/hardware_buffer.h>
/**
* MosisVulkanTexture - Imports AHardwareBuffer as Vulkan texture for Unreal.
* Creates both imported and local copy images for safe rendering.
*/
class MosisVulkanTexture
{
public:
MosisVulkanTexture() = default;
~MosisVulkanTexture();
/**
* Initialize with Vulkan device objects.
* Must be called before Create().
*/
bool Initialize(VkInstance instance, VkPhysicalDevice physDevice, VkDevice device, uint32_t queueFamilyIndex);
/**
* Create texture from a hardware buffer.
* @param buffer The AHardwareBuffer from the Mosis service
* @return true if creation succeeded
*/
bool Create(AHardwareBuffer* buffer);
/**
* Copy from imported image to local image.
* Call this each frame when a new frame is available.
*/
void CopyToLocal(VkCommandBuffer cmd);
/**
* Destroy all resources.
*/
void Destroy();
// Accessors
VkImage GetLocalImage() const { return m_LocalImage; }
VkImageView GetLocalImageView() const { return m_LocalImageView; }
VkFormat GetFormat() const { return m_Format; }
uint32_t GetWidth() const { return m_Width; }
uint32_t GetHeight() const { return m_Height; }
bool IsValid() const { return m_Created; }
private:
bool ImportHardwareBuffer(AHardwareBuffer* buffer);
bool CreateLocalImage();
void DestroyImportedResources();
void DestroyLocalResources();
uint32_t FindMemoryType(uint32_t typeFilter, VkMemoryPropertyFlags properties);
// Vulkan objects
VkInstance m_Instance = VK_NULL_HANDLE;
VkPhysicalDevice m_PhysicalDevice = VK_NULL_HANDLE;
VkDevice m_Device = VK_NULL_HANDLE;
uint32_t m_QueueFamilyIndex = 0;
// Imported from HardwareBuffer
VkImage m_ImportedImage = VK_NULL_HANDLE;
VkDeviceMemory m_ImportedMemory = VK_NULL_HANDLE;
// Local copy for safe rendering
VkImage m_LocalImage = VK_NULL_HANDLE;
VkDeviceMemory m_LocalMemory = VK_NULL_HANDLE;
VkImageView m_LocalImageView = VK_NULL_HANDLE;
VkSampler m_Sampler = VK_NULL_HANDLE;
// Format info
VkFormat m_Format = VK_FORMAT_R8G8B8A8_UNORM;
uint32_t m_Width = 0;
uint32_t m_Height = 0;
bool m_Initialized = false;
bool m_Created = false;
// Extension function pointer
PFN_vkGetAndroidHardwareBufferPropertiesANDROID vkGetAndroidHardwareBufferPropertiesANDROID = nullptr;
};
#endif // PLATFORM_ANDROID

View File

@@ -13,6 +13,9 @@ class UMaterialInstanceDynamic;
/**
* AMosisPhoneActor - Pre-configured actor for displaying the Mosis phone.
* Contains mesh, material, and phone component ready for use.
*
* By default, uses Engine's basic plane mesh. For custom phone models,
* set the PhoneMesh property to your mesh asset.
*/
UCLASS(BlueprintType, Blueprintable)
class MOSISSDK_API AMosisPhoneActor : public AActor
@@ -23,6 +26,7 @@ public:
AMosisPhoneActor();
virtual void BeginPlay() override;
virtual void Tick(float DeltaSeconds) override;
/**
* Send a touch event to the phone at world coordinates.
@@ -66,6 +70,21 @@ protected:
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis")
FVector4 ScreenBoundsLocal = FVector4(-50.0f, -100.0f, 50.0f, 100.0f);
/**
* Base material to use for the phone screen.
* Should have a texture parameter matching TextureParameterName on the PhoneComponent.
* If not set, a simple unlit emissive material will be created.
*/
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis")
TObjectPtr<UMaterialInterface> BaseMaterial;
/**
* Phone screen dimensions in world units.
* Default is 10x21.7 cm (typical smartphone aspect ratio 9:19.5)
*/
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis")
FVector2D ScreenSizeWorld = FVector2D(10.0f, 21.7f);
private:
/** Convert world hit to UV coordinates */
bool WorldToPhoneUV(FVector WorldLocation, FVector2D& OutUV) const;

View File

@@ -7,7 +7,7 @@
#include "MosisPhoneComponent.generated.h"
class UMaterialInstanceDynamic;
class UTexture2D;
class UMosisPhoneTexture;
/**
* Touch event type for phone interactions.
@@ -50,7 +50,7 @@ public:
* @return The texture showing the phone screen, or nullptr if not ready
*/
UFUNCTION(BlueprintCallable, Category = "Mosis")
UTexture2D* GetPhoneTexture() const { return PhoneTexture; }
UTexture* GetPhoneTexture() const;
/**
* Check if the phone is connected and ready.
@@ -90,20 +90,15 @@ protected:
private:
/** Phone screen texture */
UPROPERTY()
UTexture2D* PhoneTexture;
TObjectPtr<UMosisPhoneTexture> PhoneTexture;
/** Screen dimensions from service */
int32 ScreenWidth = 0;
int32 ScreenHeight = 0;
/** Flag to track pending texture updates */
bool bPendingTextureUpdate = false;
TAtomic<bool> bPendingTextureUpdate{false};
/** Flag to track if we need to recreate the texture */
bool bNeedsTextureCreate = false;
#if PLATFORM_ANDROID
/** Vulkan texture wrapper */
TSharedPtr<class MosisVulkanTexture> VulkanTexture;
#endif
TAtomic<bool> bNeedsTextureCreate{false};
};

View File

@@ -0,0 +1,133 @@
// Copyright Epic Games, Inc. All Rights Reserved.
#pragma once
#include "CoreMinimal.h"
#include "Components/SceneComponent.h"
#include "MosisPhoneComponent.h"
#include "MosisPointerComponent.generated.h"
class AMosisPhoneActor;
class UInputAction;
struct FInputActionInstance;
/**
* UMosisPointerComponent - VR ray interaction component for Mosis phone.
*
* Attach this component as a child of a motion controller to enable
* VR pointer-based touch interaction with MosisPhoneActor.
*
* Usage:
* 1. Add as child of MotionControllerComponent
* 2. Set TriggerAction to your trigger input action (optional)
* 3. Touch events are sent automatically when pointing at a phone and triggering
*
* Manual control (without Enhanced Input):
* - Call SetTriggerPressed(true/false) from your input handling code
*/
UCLASS(ClassGroup=(Mosis), meta=(BlueprintSpawnableComponent))
class MOSISSDK_API UMosisPointerComponent : public USceneComponent
{
GENERATED_BODY()
public:
UMosisPointerComponent();
// Configuration
/** Maximum ray length for detecting phone actors (in cm) */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis|Pointer")
float RayLength = 500.0f;
/** Draw debug ray visualization */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis|Pointer")
bool bShowDebugRay = false;
/** Color of the debug ray when not hitting anything */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis|Pointer", meta=(EditCondition="bShowDebugRay"))
FLinearColor DebugRayColor = FLinearColor::Red;
/** Color of the debug ray when hitting a phone */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis|Pointer", meta=(EditCondition="bShowDebugRay"))
FLinearColor DebugRayHitColor = FLinearColor::Green;
/** Collision channel used for raycast */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis|Pointer")
TEnumAsByte<ECollisionChannel> TraceChannel = ECC_Visibility;
// Input
/**
* Enhanced Input Action for trigger press.
* If set, the component will automatically bind to this action.
* Leave null to use manual SetTriggerPressed() calls.
*/
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mosis|Input")
TObjectPtr<UInputAction> TriggerAction;
// State queries
/** Check if the pointer is currently pointing at a phone */
UFUNCTION(BlueprintCallable, Category = "Mosis|Pointer")
bool IsPointingAtPhone() const { return bIsOverPhone; }
/** Get the phone actor currently being pointed at (nullptr if none) */
UFUNCTION(BlueprintCallable, Category = "Mosis|Pointer")
AMosisPhoneActor* GetTargetPhone() const;
/** Get the world location where the ray hits the phone */
UFUNCTION(BlueprintCallable, Category = "Mosis|Pointer")
FVector GetHitLocation() const { return CurrentHitLocation; }
/** Get the current ray origin in world space */
UFUNCTION(BlueprintCallable, Category = "Mosis|Pointer")
FVector GetRayOrigin() const;
/** Get the current ray direction in world space */
UFUNCTION(BlueprintCallable, Category = "Mosis|Pointer")
FVector GetRayDirection() const;
/** Check if the trigger is currently pressed */
UFUNCTION(BlueprintCallable, Category = "Mosis|Pointer")
bool IsTriggerPressed() const { return bIsTriggerPressed; }
// Manual control
/**
* Manually set the trigger pressed state.
* Use this when not using Enhanced Input, or for custom input handling.
*/
UFUNCTION(BlueprintCallable, Category = "Mosis|Pointer")
void SetTriggerPressed(bool bPressed);
protected:
// UActorComponent interface
virtual void BeginPlay() override;
virtual void TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override;
private:
/** Perform raycast and update CurrentPhone/CurrentHitLocation */
void PerformRaycast();
/** Update touch state machine and send touch events */
void UpdateTouchState();
/** Draw debug visualization */
void DrawDebugRay() const;
/** Callback for Enhanced Input trigger action */
void OnTriggerAction(const FInputActionInstance& Instance);
/** Setup Enhanced Input bindings */
void SetupInputBindings();
// State
TWeakObjectPtr<AMosisPhoneActor> CurrentPhone;
TWeakObjectPtr<AMosisPhoneActor> LastTouchedPhone;
FVector CurrentHitLocation = FVector::ZeroVector;
FVector LastHitLocation = FVector::ZeroVector;
bool bIsTriggerPressed = false;
bool bWasTriggerPressed = false;
bool bIsOverPhone = false;
bool bInputBound = false;
};

View File

@@ -0,0 +1,67 @@
# Unreal Editor Python script to create the Mosis phone screen material
# Run this in the Editor: File > Execute Python Script
# Or from Python console: exec(open('Scripts/create_phone_material.py').read())
import unreal
def create_phone_screen_material():
"""Create an unlit emissive material for displaying the phone screen."""
asset_tools = unreal.AssetToolsHelpers.get_asset_tools()
material_factory = unreal.MaterialFactoryNew()
# Create the material asset
material_path = "/Game/Mosis/Materials"
material_name = "M_PhoneScreen"
# Check if material already exists
if unreal.EditorAssetLibrary.does_asset_exist(f"{material_path}/{material_name}"):
unreal.log_warning(f"Material {material_name} already exists, skipping creation")
return unreal.load_asset(f"{material_path}/{material_name}")
# Create the material
material = asset_tools.create_asset(
material_name,
material_path,
unreal.Material,
material_factory
)
if not material:
unreal.log_error("Failed to create material")
return None
# Configure material properties for phone screen display
material.set_editor_property("shading_model", unreal.MaterialShadingModel.MSM_UNLIT)
material.set_editor_property("blend_mode", unreal.BlendMode.BLEND_OPAQUE)
material.set_editor_property("two_sided", False)
# Get the material editor subsystem to add nodes
mel = unreal.MaterialEditingLibrary
# Create texture parameter node
texture_param = mel.create_material_expression(
material,
unreal.MaterialExpressionTextureSampleParameter2D,
-400, 0
)
texture_param.set_editor_property("parameter_name", "PhoneScreen")
# Connect texture RGB to emissive color
mel.connect_material_property(
texture_param, "RGB",
unreal.MaterialProperty.MP_EMISSIVE_COLOR
)
# Recompile the material
mel.recompile_material(material)
# Save the asset
unreal.EditorAssetLibrary.save_asset(f"{material_path}/{material_name}")
unreal.log(f"Created material: {material_path}/{material_name}")
return material
if __name__ == "__main__":
create_phone_screen_material()

207
build-unreal.bat Normal file
View File

@@ -0,0 +1,207 @@
@echo off
setlocal enabledelayedexpansion
:: MosisUnreal Build and Deploy Script
:: Usage: build-unreal.bat [build|install|deploy|launch|clean]
:: build - Build Android APK only
:: install - Install APK to device only
:: deploy - Build and install (default)
:: launch - Launch app (starts MosisService first)
:: clean - Clean build artifacts
set ENGINE_PATH=D:\Epic\UE_5.5
set PROJECT_PATH=%~dp0MosisUnreal.uproject
set UAT_PATH=%ENGINE_PATH%\Engine\Build\BatchFiles\RunUAT.bat
set CONFIG=Development
:: Parse command line argument
set ACTION=%1
if "%ACTION%"=="" set ACTION=deploy
:: Validate engine path
if not exist "%UAT_PATH%" (
echo ERROR: Unreal Engine not found at %ENGINE_PATH%
echo Please update ENGINE_PATH in this script.
exit /b 1
)
:: Execute action
if "%ACTION%"=="build" goto :build
if "%ACTION%"=="install" goto :install
if "%ACTION%"=="deploy" goto :deploy
if "%ACTION%"=="launch" goto :launch
if "%ACTION%"=="clean" goto :clean
echo Unknown action: %ACTION%
echo Usage: build-unreal.bat [build^|install^|deploy^|launch^|clean]
exit /b 1
:build
echo.
echo ============================================
echo Building MosisUnreal for Android...
echo ============================================
echo.
call "%UAT_PATH%" BuildCookRun ^
-project="%PROJECT_PATH%" ^
-platform=Android ^
-clientconfig=%CONFIG% ^
-build -cook -stage -pak -package ^
-noP4 ^
-utf8output
if errorlevel 1 (
echo.
echo ERROR: Build failed!
exit /b 1
)
echo.
echo Build completed successfully!
echo APK: %~dp0Binaries\Android\MosisUnreal-arm64.apk
goto :eof
:install
echo.
echo ============================================
echo Installing MosisUnreal to device...
echo ============================================
echo.
:: Check for connected device
adb devices | findstr /r /c:"device$" >nul
if errorlevel 1 (
echo ERROR: No Android device connected!
echo Connect a device and enable USB debugging.
exit /b 1
)
:: Get device ID
for /f "tokens=1" %%d in ('adb devices ^| findstr /r /c:"device$"') do (
set DEVICE=%%d
goto :found_device
)
:found_device
echo Device: %DEVICE%
set APK_PATH=%~dp0Binaries\Android\MosisUnreal-arm64.apk
set OBB_PATH=%~dp0Binaries\Android\main.1.com.omixlab.MosisUnreal.obb
:: Check APK exists
if not exist "%APK_PATH%" (
echo ERROR: APK not found at %APK_PATH%
echo Run 'build-unreal.bat build' first.
exit /b 1
)
:: Install APK
echo Installing APK...
adb -s %DEVICE% install -r "%APK_PATH%"
if errorlevel 1 (
echo ERROR: APK installation failed!
exit /b 1
)
:: Install OBB if exists
if exist "%OBB_PATH%" (
echo.
echo Installing OBB...
:: Create temp directory
adb -s %DEVICE% shell "mkdir -p /data/local/tmp/obb/com.omixlab.MosisUnreal"
:: Push OBB (use forward slashes for adb)
set OBB_UNIX=%OBB_PATH:\=/%
adb -s %DEVICE% push "!OBB_UNIX!" /data/local/tmp/obb/com.omixlab.MosisUnreal/
:: Move to final location
adb -s %DEVICE% shell "rm -rf /sdcard/Android/obb/com.omixlab.MosisUnreal"
adb -s %DEVICE% shell "mv /data/local/tmp/obb/com.omixlab.MosisUnreal /sdcard/Android/obb/"
echo OBB installed.
) else (
echo No OBB file found, skipping.
)
echo.
echo Installation completed!
echo.
echo To launch: adb -s %DEVICE% shell am start -n com.omixlab.MosisUnreal/com.epicgames.unreal.GameActivity
goto :eof
:deploy
call :build
if errorlevel 1 exit /b 1
call :install
if errorlevel 1 exit /b 1
call :launch
goto :eof
:launch
echo.
echo ============================================
echo Launching MosisUnreal...
echo ============================================
echo.
:: Check for connected device
adb devices | findstr /r /c:"device$" >nul
if errorlevel 1 (
echo ERROR: No Android device connected!
exit /b 1
)
:: Get device ID
for /f "tokens=1" %%d in ('adb devices ^| findstr /r /c:"device$"') do (
set DEVICE=%%d
goto :found_device_launch
)
:found_device_launch
echo Device: %DEVICE%
:: Stop any running instances
echo Stopping existing instances...
adb -s %DEVICE% shell am force-stop com.omixlab.MosisUnreal >nul 2>&1
adb -s %DEVICE% shell am force-stop com.omixlab.mosis >nul 2>&1
timeout /t 1 /nobreak >nul
:: Start MosisService first
echo Starting MosisService...
adb -s %DEVICE% shell am start -n com.omixlab.mosis/.MainActivity
timeout /t 2 /nobreak >nul
:: Start MosisUnreal
echo Starting MosisUnreal...
adb -s %DEVICE% shell am start -n com.omixlab.MosisUnreal/com.epicgames.unreal.GameActivity
echo.
echo Launched! Monitoring logs (Ctrl+C to stop)...
echo.
adb -s %DEVICE% logcat -s MosisSDK MosisOS MosisTest UE
goto :eof
:clean
echo.
echo ============================================
echo Cleaning build artifacts...
echo ============================================
echo.
if exist "%~dp0Binaries" (
echo Removing Binaries...
rmdir /s /q "%~dp0Binaries"
)
if exist "%~dp0Intermediate\Build" (
echo Removing Intermediate\Build...
rmdir /s /q "%~dp0Intermediate\Build"
)
if exist "%~dp0Saved\StagedBuilds" (
echo Removing Saved\StagedBuilds...
rmdir /s /q "%~dp0Saved\StagedBuilds"
)
echo Clean completed!
goto :eof