497 lines
13 KiB
Markdown
497 lines
13 KiB
Markdown
# Milestone 7: Game Integration
|
|
|
|
**Status**: Not Started
|
|
**Goal**: Production-ready Unity and Unreal plugins for seamless VR phone integration.
|
|
|
|
---
|
|
|
|
## Overview
|
|
|
|
Game engine plugins enable:
|
|
- Rendering the phone in VR scenes
|
|
- Touch/raycast interaction
|
|
- Virtual hardware provision (camera, mic, speaker)
|
|
- Event callbacks to game code
|
|
|
|
---
|
|
|
|
## Current State
|
|
|
|
| Platform | Location | Status |
|
|
|----------|----------|--------|
|
|
| Unity | `D:\Dev\Mosis\Mosis Unity` | Basic Binder client |
|
|
| Unreal | `D:\Dev\Mosis\Mosis Unreal` | WIP |
|
|
|
|
---
|
|
|
|
## Unity Plugin
|
|
|
|
### Package Structure
|
|
|
|
```
|
|
com.mosis.phone/
|
|
├── package.json
|
|
├── Runtime/
|
|
│ ├── MosisPhone.cs # Main component
|
|
│ ├── MosisService.cs # Binder client
|
|
│ ├── MosisInputHandler.cs # Touch/raycast
|
|
│ ├── MosisHardwareProvider.cs # Virtual hardware
|
|
│ └── Native/
|
|
│ ├── MosisNative.cs # P/Invoke declarations
|
|
│ └── Plugins/
|
|
│ └── Android/
|
|
│ └── libmosis-client.so
|
|
├── Prefabs/
|
|
│ ├── MosisPhone.prefab # Ready-to-use phone
|
|
│ └── MosisPhoneVR.prefab # VR-optimized version
|
|
├── Samples~/
|
|
│ └── BasicIntegration/
|
|
│ └── PhoneDemo.unity
|
|
└── Documentation~/
|
|
└── integration-guide.md
|
|
```
|
|
|
|
### Main Component
|
|
|
|
**File**: `MosisPhone.cs`
|
|
|
|
```csharp
|
|
using UnityEngine;
|
|
using UnityEngine.Events;
|
|
|
|
namespace Mosis {
|
|
public class MosisPhone : MonoBehaviour {
|
|
[Header("Display")]
|
|
[SerializeField] private MeshRenderer phoneScreen;
|
|
[SerializeField] private Vector2Int resolution = new Vector2Int(540, 960);
|
|
|
|
[Header("Input")]
|
|
[SerializeField] private bool enableRaycast = true;
|
|
[SerializeField] private LayerMask raycastLayers;
|
|
|
|
[Header("Virtual Hardware")]
|
|
[SerializeField] private Camera virtualCamera;
|
|
[SerializeField] private AudioSource virtualMicrophone;
|
|
[SerializeField] private AudioSource virtualSpeaker;
|
|
|
|
[Header("Events")]
|
|
public UnityEvent onPhoneReady;
|
|
public UnityEvent<string> onNavigate;
|
|
public UnityEvent<string, string> onMessageReceived;
|
|
public UnityEvent<string> onCallStarted;
|
|
|
|
private MosisService service;
|
|
private RenderTexture screenTexture;
|
|
private Material screenMaterial;
|
|
|
|
void Awake() {
|
|
// Create render texture for phone screen
|
|
screenTexture = new RenderTexture(resolution.x, resolution.y, 0);
|
|
screenMaterial = new Material(Shader.Find("Unlit/Texture"));
|
|
screenMaterial.mainTexture = screenTexture;
|
|
phoneScreen.material = screenMaterial;
|
|
}
|
|
|
|
void Start() {
|
|
service = new MosisService();
|
|
service.OnFrameAvailable += OnFrameAvailable;
|
|
service.OnServiceInitialized += () => onPhoneReady?.Invoke();
|
|
service.Connect();
|
|
}
|
|
|
|
void Update() {
|
|
if (enableRaycast) {
|
|
HandleRaycastInput();
|
|
}
|
|
}
|
|
|
|
private void HandleRaycastInput() {
|
|
// VR controller raycast
|
|
if (OVRInput.GetDown(OVRInput.Button.PrimaryIndexTrigger)) {
|
|
var ray = new Ray(
|
|
OVRInput.GetLocalControllerPosition(OVRInput.Controller.RTouch),
|
|
OVRInput.GetLocalControllerRotation(OVRInput.Controller.RTouch) * Vector3.forward
|
|
);
|
|
|
|
if (Physics.Raycast(ray, out var hit, 10f, raycastLayers)) {
|
|
if (hit.collider.gameObject == phoneScreen.gameObject) {
|
|
var uv = hit.textureCoord;
|
|
var x = uv.x * resolution.x;
|
|
var y = (1 - uv.y) * resolution.y;
|
|
service.SendTouchDown(x, y);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private void OnFrameAvailable(byte[] pixels) {
|
|
// Update screen texture
|
|
screenTexture.LoadRawTextureData(pixels);
|
|
screenTexture.Apply();
|
|
}
|
|
|
|
// Public API
|
|
public void NavigateTo(string screen) {
|
|
service.Navigate(screen);
|
|
}
|
|
|
|
public void SendMessage(string to, string text) {
|
|
service.SendMessage(to, text);
|
|
}
|
|
|
|
public void MakeCall(string number) {
|
|
service.MakeCall(number);
|
|
}
|
|
|
|
void OnDestroy() {
|
|
service?.Disconnect();
|
|
Destroy(screenTexture);
|
|
Destroy(screenMaterial);
|
|
}
|
|
}
|
|
}
|
|
```
|
|
|
|
### Hardware Provider
|
|
|
|
**File**: `MosisHardwareProvider.cs`
|
|
|
|
```csharp
|
|
namespace Mosis {
|
|
public class MosisHardwareProvider : MonoBehaviour {
|
|
[SerializeField] private MosisPhone phone;
|
|
[SerializeField] private Camera gameCamera;
|
|
[SerializeField] private AudioListener audioListener;
|
|
|
|
private RenderTexture cameraTexture;
|
|
|
|
void Start() {
|
|
// Setup virtual camera
|
|
cameraTexture = new RenderTexture(640, 480, 0);
|
|
gameCamera.targetTexture = cameraTexture;
|
|
|
|
// Register with phone service
|
|
phone.Service.SetCameraProvider(() => {
|
|
var pixels = new byte[cameraTexture.width * cameraTexture.height * 4];
|
|
// Read pixels from RenderTexture
|
|
return pixels;
|
|
});
|
|
|
|
phone.Service.SetMicrophoneProvider(() => {
|
|
// Get audio from AudioListener
|
|
return audioSamples;
|
|
});
|
|
}
|
|
|
|
public void PlayAudio(float[] samples) {
|
|
// Play on virtual speaker
|
|
var audioSource = GetComponent<AudioSource>();
|
|
audioSource.clip = AudioClip.Create("phone", samples.Length, 1, 44100, false);
|
|
audioSource.clip.SetData(samples, 0);
|
|
audioSource.Play();
|
|
}
|
|
}
|
|
}
|
|
```
|
|
|
|
### VR Interaction
|
|
|
|
**File**: `MosisInputHandler.cs`
|
|
|
|
```csharp
|
|
namespace Mosis {
|
|
public class MosisInputHandler : MonoBehaviour {
|
|
[SerializeField] private MosisPhone phone;
|
|
[SerializeField] private Transform pointerOrigin;
|
|
[SerializeField] private LineRenderer laserPointer;
|
|
|
|
private bool isTouching;
|
|
private Vector2 lastTouchPos;
|
|
|
|
void Update() {
|
|
// Update laser pointer
|
|
var ray = new Ray(pointerOrigin.position, pointerOrigin.forward);
|
|
|
|
if (Physics.Raycast(ray, out var hit, 10f)) {
|
|
laserPointer.SetPosition(1, hit.point);
|
|
|
|
if (hit.collider.GetComponent<MosisPhone>() != null) {
|
|
var uv = hit.textureCoord;
|
|
var touchPos = new Vector2(
|
|
uv.x * phone.Resolution.x,
|
|
(1 - uv.y) * phone.Resolution.y
|
|
);
|
|
|
|
HandleTouch(touchPos);
|
|
}
|
|
}
|
|
}
|
|
|
|
private void HandleTouch(Vector2 pos) {
|
|
bool triggerDown = OVRInput.Get(OVRInput.Button.PrimaryIndexTrigger);
|
|
|
|
if (triggerDown && !isTouching) {
|
|
phone.Service.SendTouchDown(pos.x, pos.y);
|
|
isTouching = true;
|
|
}
|
|
else if (triggerDown && isTouching) {
|
|
if (Vector2.Distance(pos, lastTouchPos) > 2f) {
|
|
phone.Service.SendTouchMove(pos.x, pos.y);
|
|
}
|
|
}
|
|
else if (!triggerDown && isTouching) {
|
|
phone.Service.SendTouchUp(pos.x, pos.y);
|
|
isTouching = false;
|
|
}
|
|
|
|
lastTouchPos = pos;
|
|
}
|
|
}
|
|
}
|
|
```
|
|
|
|
---
|
|
|
|
## Unreal Plugin
|
|
|
|
### Module Structure
|
|
|
|
```
|
|
MosisPhone/
|
|
├── MosisPhone.uplugin
|
|
├── Source/
|
|
│ ├── MosisPhone/
|
|
│ │ ├── MosisPhone.Build.cs
|
|
│ │ ├── Public/
|
|
│ │ │ ├── MosisPhoneActor.h
|
|
│ │ │ ├── MosisService.h
|
|
│ │ │ └── MosisHardwareProvider.h
|
|
│ │ └── Private/
|
|
│ │ ├── MosisPhoneActor.cpp
|
|
│ │ ├── MosisService.cpp
|
|
│ │ └── MosisHardwareProvider.cpp
|
|
│ └── ThirdParty/
|
|
│ └── MosisClient/
|
|
│ └── libmosis-client.so
|
|
├── Content/
|
|
│ ├── BP_MosisPhone.uasset
|
|
│ └── M_PhoneScreen.uasset
|
|
└── Documentation/
|
|
└── integration-guide.md
|
|
```
|
|
|
|
### Main Actor
|
|
|
|
**File**: `MosisPhoneActor.h`
|
|
|
|
```cpp
|
|
#pragma once
|
|
|
|
#include "CoreMinimal.h"
|
|
#include "GameFramework/Actor.h"
|
|
#include "MosisPhoneActor.generated.h"
|
|
|
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnPhoneReady);
|
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnNavigate, FString, Screen);
|
|
|
|
UCLASS()
|
|
class MOSISPHONE_API AMosisPhoneActor : public AActor {
|
|
GENERATED_BODY()
|
|
|
|
public:
|
|
AMosisPhoneActor();
|
|
|
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Display")
|
|
FIntPoint Resolution = FIntPoint(540, 960);
|
|
|
|
UPROPERTY(BlueprintAssignable, Category = "Events")
|
|
FOnPhoneReady OnPhoneReady;
|
|
|
|
UPROPERTY(BlueprintAssignable, Category = "Events")
|
|
FOnNavigate OnNavigate;
|
|
|
|
UFUNCTION(BlueprintCallable, Category = "Mosis")
|
|
void NavigateTo(const FString& Screen);
|
|
|
|
UFUNCTION(BlueprintCallable, Category = "Mosis")
|
|
void SendMessage(const FString& To, const FString& Text);
|
|
|
|
UFUNCTION(BlueprintCallable, Category = "Mosis")
|
|
void MakeCall(const FString& Number);
|
|
|
|
protected:
|
|
virtual void BeginPlay() override;
|
|
virtual void Tick(float DeltaTime) override;
|
|
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
|
|
|
|
private:
|
|
UPROPERTY()
|
|
UTextureRenderTarget2D* ScreenTexture;
|
|
|
|
UPROPERTY()
|
|
UMaterialInstanceDynamic* ScreenMaterial;
|
|
|
|
TSharedPtr<FMosisService> Service;
|
|
|
|
void HandleRaycastInput();
|
|
void OnFrameAvailable(const TArray<uint8>& Pixels);
|
|
};
|
|
```
|
|
|
|
### Blueprint Integration
|
|
|
|
```cpp
|
|
// Blueprint callable functions for easy integration
|
|
|
|
UFUNCTION(BlueprintCallable, Category = "Mosis", meta = (WorldContext = "WorldContextObject"))
|
|
static AMosisPhoneActor* SpawnMosisPhone(
|
|
UObject* WorldContextObject,
|
|
FTransform SpawnTransform,
|
|
FIntPoint Resolution = FIntPoint(540, 960)
|
|
);
|
|
```
|
|
|
|
---
|
|
|
|
## Hardware Button Simulation
|
|
|
|
### Physical Buttons
|
|
|
|
| Button | Function |
|
|
|--------|----------|
|
|
| Power | Lock/wake screen |
|
|
| Volume Up | Increase volume |
|
|
| Volume Down | Decrease volume |
|
|
| Home (soft) | Go to home screen |
|
|
| Back (soft) | Navigation back |
|
|
|
|
### Unity Implementation
|
|
|
|
```csharp
|
|
public class MosisPhoneButtons : MonoBehaviour {
|
|
[SerializeField] private MosisPhone phone;
|
|
[SerializeField] private Collider powerButton;
|
|
[SerializeField] private Collider volumeUp;
|
|
[SerializeField] private Collider volumeDown;
|
|
|
|
void Update() {
|
|
// Check for button presses via physics overlap
|
|
if (IsButtonPressed(powerButton)) {
|
|
phone.Service.SendButton("power");
|
|
}
|
|
// ...
|
|
}
|
|
}
|
|
```
|
|
|
|
### Unreal Implementation
|
|
|
|
```cpp
|
|
// In Blueprint or C++
|
|
UFUNCTION(BlueprintCallable)
|
|
void PressButton(EMosisButton Button) {
|
|
switch (Button) {
|
|
case EMosisButton::Power:
|
|
Service->SendButton("power");
|
|
break;
|
|
case EMosisButton::VolumeUp:
|
|
Service->SendButton("volume_up");
|
|
break;
|
|
case EMosisButton::VolumeDown:
|
|
Service->SendButton("volume_down");
|
|
break;
|
|
}
|
|
}
|
|
```
|
|
|
|
---
|
|
|
|
## Implementation Plan
|
|
|
|
### Phase 1: Unity Core
|
|
- [ ] Package structure
|
|
- [ ] MosisPhone component
|
|
- [ ] Basic touch input
|
|
- [ ] Frame rendering
|
|
|
|
### Phase 2: Unity VR
|
|
- [ ] VR raycast interaction
|
|
- [ ] Laser pointer visualization
|
|
- [ ] Controller haptics
|
|
- [ ] Two-handed support
|
|
|
|
### Phase 3: Unity Hardware
|
|
- [ ] Camera provider (RenderTexture)
|
|
- [ ] Microphone provider
|
|
- [ ] Speaker output
|
|
- [ ] Vibration
|
|
|
|
### Phase 4: Unreal Core
|
|
- [ ] Plugin structure
|
|
- [ ] MosisPhoneActor
|
|
- [ ] Blueprint integration
|
|
- [ ] Touch input
|
|
|
|
### Phase 5: Unreal VR
|
|
- [ ] Motion controller input
|
|
- [ ] Widget interaction
|
|
- [ ] Hardware providers
|
|
|
|
### Phase 6: Documentation
|
|
- [ ] Quick start guide
|
|
- [ ] API reference
|
|
- [ ] Sample scenes
|
|
- [ ] Troubleshooting
|
|
|
|
---
|
|
|
|
## Testing
|
|
|
|
### Unity Test Scene
|
|
|
|
1. Phone mounted in VR scene
|
|
2. Touch interaction works
|
|
3. Virtual camera shows game view
|
|
4. Audio plays through virtual speaker
|
|
|
|
### Unreal Test Level
|
|
|
|
1. Blueprint phone actor
|
|
2. Motion controller interaction
|
|
3. Event callbacks work
|
|
4. Performance acceptable
|
|
|
|
---
|
|
|
|
## Performance Targets
|
|
|
|
| Metric | Target |
|
|
|--------|--------|
|
|
| Frame latency | < 16ms |
|
|
| Touch latency | < 50ms |
|
|
| Memory usage | < 100MB |
|
|
| Draw calls | < 5 per phone |
|
|
|
|
---
|
|
|
|
## Acceptance Criteria
|
|
|
|
### Unity
|
|
- [ ] One-click prefab placement
|
|
- [ ] VR raycast touch works
|
|
- [ ] Events fire correctly
|
|
- [ ] Camera feed from game
|
|
- [ ] Audio bidirectional
|
|
|
|
### Unreal
|
|
- [ ] Blueprint spawnable
|
|
- [ ] Motion controller support
|
|
- [ ] Blueprint events
|
|
- [ ] Hardware providers
|
|
|
|
### Documentation
|
|
- [ ] Integration guide
|
|
- [ ] API docs
|
|
- [ ] Example projects
|
|
- [ ] Video tutorial
|