move docs to docs/ folder, merge architecture files, update references
This commit is contained in:
496
docs/MILESTONE-7.md
Normal file
496
docs/MILESTONE-7.md
Normal file
@@ -0,0 +1,496 @@
|
||||
# Milestone 7: Game Integration
|
||||
|
||||
**Status**: Not Started
|
||||
**Goal**: Production-ready Unity and Unreal plugins for seamless VR phone integration.
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Game engine plugins enable:
|
||||
- Rendering the phone in VR scenes
|
||||
- Touch/raycast interaction
|
||||
- Virtual hardware provision (camera, mic, speaker)
|
||||
- Event callbacks to game code
|
||||
|
||||
---
|
||||
|
||||
## Current State
|
||||
|
||||
| Platform | Location | Status |
|
||||
|----------|----------|--------|
|
||||
| Unity | `D:\Dev\Mosis\Mosis Unity` | Basic Binder client |
|
||||
| Unreal | `D:\Dev\Mosis\Mosis Unreal` | WIP |
|
||||
|
||||
---
|
||||
|
||||
## Unity Plugin
|
||||
|
||||
### Package Structure
|
||||
|
||||
```
|
||||
com.mosis.phone/
|
||||
├── package.json
|
||||
├── Runtime/
|
||||
│ ├── MosisPhone.cs # Main component
|
||||
│ ├── MosisService.cs # Binder client
|
||||
│ ├── MosisInputHandler.cs # Touch/raycast
|
||||
│ ├── MosisHardwareProvider.cs # Virtual hardware
|
||||
│ └── Native/
|
||||
│ ├── MosisNative.cs # P/Invoke declarations
|
||||
│ └── Plugins/
|
||||
│ └── Android/
|
||||
│ └── libmosis-client.so
|
||||
├── Prefabs/
|
||||
│ ├── MosisPhone.prefab # Ready-to-use phone
|
||||
│ └── MosisPhoneVR.prefab # VR-optimized version
|
||||
├── Samples~/
|
||||
│ └── BasicIntegration/
|
||||
│ └── PhoneDemo.unity
|
||||
└── Documentation~/
|
||||
└── integration-guide.md
|
||||
```
|
||||
|
||||
### Main Component
|
||||
|
||||
**File**: `MosisPhone.cs`
|
||||
|
||||
```csharp
|
||||
using UnityEngine;
|
||||
using UnityEngine.Events;
|
||||
|
||||
namespace Mosis {
|
||||
public class MosisPhone : MonoBehaviour {
|
||||
[Header("Display")]
|
||||
[SerializeField] private MeshRenderer phoneScreen;
|
||||
[SerializeField] private Vector2Int resolution = new Vector2Int(540, 960);
|
||||
|
||||
[Header("Input")]
|
||||
[SerializeField] private bool enableRaycast = true;
|
||||
[SerializeField] private LayerMask raycastLayers;
|
||||
|
||||
[Header("Virtual Hardware")]
|
||||
[SerializeField] private Camera virtualCamera;
|
||||
[SerializeField] private AudioSource virtualMicrophone;
|
||||
[SerializeField] private AudioSource virtualSpeaker;
|
||||
|
||||
[Header("Events")]
|
||||
public UnityEvent onPhoneReady;
|
||||
public UnityEvent<string> onNavigate;
|
||||
public UnityEvent<string, string> onMessageReceived;
|
||||
public UnityEvent<string> onCallStarted;
|
||||
|
||||
private MosisService service;
|
||||
private RenderTexture screenTexture;
|
||||
private Material screenMaterial;
|
||||
|
||||
void Awake() {
|
||||
// Create render texture for phone screen
|
||||
screenTexture = new RenderTexture(resolution.x, resolution.y, 0);
|
||||
screenMaterial = new Material(Shader.Find("Unlit/Texture"));
|
||||
screenMaterial.mainTexture = screenTexture;
|
||||
phoneScreen.material = screenMaterial;
|
||||
}
|
||||
|
||||
void Start() {
|
||||
service = new MosisService();
|
||||
service.OnFrameAvailable += OnFrameAvailable;
|
||||
service.OnServiceInitialized += () => onPhoneReady?.Invoke();
|
||||
service.Connect();
|
||||
}
|
||||
|
||||
void Update() {
|
||||
if (enableRaycast) {
|
||||
HandleRaycastInput();
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleRaycastInput() {
|
||||
// VR controller raycast
|
||||
if (OVRInput.GetDown(OVRInput.Button.PrimaryIndexTrigger)) {
|
||||
var ray = new Ray(
|
||||
OVRInput.GetLocalControllerPosition(OVRInput.Controller.RTouch),
|
||||
OVRInput.GetLocalControllerRotation(OVRInput.Controller.RTouch) * Vector3.forward
|
||||
);
|
||||
|
||||
if (Physics.Raycast(ray, out var hit, 10f, raycastLayers)) {
|
||||
if (hit.collider.gameObject == phoneScreen.gameObject) {
|
||||
var uv = hit.textureCoord;
|
||||
var x = uv.x * resolution.x;
|
||||
var y = (1 - uv.y) * resolution.y;
|
||||
service.SendTouchDown(x, y);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void OnFrameAvailable(byte[] pixels) {
|
||||
// Update screen texture
|
||||
screenTexture.LoadRawTextureData(pixels);
|
||||
screenTexture.Apply();
|
||||
}
|
||||
|
||||
// Public API
|
||||
public void NavigateTo(string screen) {
|
||||
service.Navigate(screen);
|
||||
}
|
||||
|
||||
public void SendMessage(string to, string text) {
|
||||
service.SendMessage(to, text);
|
||||
}
|
||||
|
||||
public void MakeCall(string number) {
|
||||
service.MakeCall(number);
|
||||
}
|
||||
|
||||
void OnDestroy() {
|
||||
service?.Disconnect();
|
||||
Destroy(screenTexture);
|
||||
Destroy(screenMaterial);
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Hardware Provider
|
||||
|
||||
**File**: `MosisHardwareProvider.cs`
|
||||
|
||||
```csharp
|
||||
namespace Mosis {
|
||||
public class MosisHardwareProvider : MonoBehaviour {
|
||||
[SerializeField] private MosisPhone phone;
|
||||
[SerializeField] private Camera gameCamera;
|
||||
[SerializeField] private AudioListener audioListener;
|
||||
|
||||
private RenderTexture cameraTexture;
|
||||
|
||||
void Start() {
|
||||
// Setup virtual camera
|
||||
cameraTexture = new RenderTexture(640, 480, 0);
|
||||
gameCamera.targetTexture = cameraTexture;
|
||||
|
||||
// Register with phone service
|
||||
phone.Service.SetCameraProvider(() => {
|
||||
var pixels = new byte[cameraTexture.width * cameraTexture.height * 4];
|
||||
// Read pixels from RenderTexture
|
||||
return pixels;
|
||||
});
|
||||
|
||||
phone.Service.SetMicrophoneProvider(() => {
|
||||
// Get audio from AudioListener
|
||||
return audioSamples;
|
||||
});
|
||||
}
|
||||
|
||||
public void PlayAudio(float[] samples) {
|
||||
// Play on virtual speaker
|
||||
var audioSource = GetComponent<AudioSource>();
|
||||
audioSource.clip = AudioClip.Create("phone", samples.Length, 1, 44100, false);
|
||||
audioSource.clip.SetData(samples, 0);
|
||||
audioSource.Play();
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### VR Interaction
|
||||
|
||||
**File**: `MosisInputHandler.cs`
|
||||
|
||||
```csharp
|
||||
namespace Mosis {
|
||||
public class MosisInputHandler : MonoBehaviour {
|
||||
[SerializeField] private MosisPhone phone;
|
||||
[SerializeField] private Transform pointerOrigin;
|
||||
[SerializeField] private LineRenderer laserPointer;
|
||||
|
||||
private bool isTouching;
|
||||
private Vector2 lastTouchPos;
|
||||
|
||||
void Update() {
|
||||
// Update laser pointer
|
||||
var ray = new Ray(pointerOrigin.position, pointerOrigin.forward);
|
||||
|
||||
if (Physics.Raycast(ray, out var hit, 10f)) {
|
||||
laserPointer.SetPosition(1, hit.point);
|
||||
|
||||
if (hit.collider.GetComponent<MosisPhone>() != null) {
|
||||
var uv = hit.textureCoord;
|
||||
var touchPos = new Vector2(
|
||||
uv.x * phone.Resolution.x,
|
||||
(1 - uv.y) * phone.Resolution.y
|
||||
);
|
||||
|
||||
HandleTouch(touchPos);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleTouch(Vector2 pos) {
|
||||
bool triggerDown = OVRInput.Get(OVRInput.Button.PrimaryIndexTrigger);
|
||||
|
||||
if (triggerDown && !isTouching) {
|
||||
phone.Service.SendTouchDown(pos.x, pos.y);
|
||||
isTouching = true;
|
||||
}
|
||||
else if (triggerDown && isTouching) {
|
||||
if (Vector2.Distance(pos, lastTouchPos) > 2f) {
|
||||
phone.Service.SendTouchMove(pos.x, pos.y);
|
||||
}
|
||||
}
|
||||
else if (!triggerDown && isTouching) {
|
||||
phone.Service.SendTouchUp(pos.x, pos.y);
|
||||
isTouching = false;
|
||||
}
|
||||
|
||||
lastTouchPos = pos;
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Unreal Plugin
|
||||
|
||||
### Module Structure
|
||||
|
||||
```
|
||||
MosisPhone/
|
||||
├── MosisPhone.uplugin
|
||||
├── Source/
|
||||
│ ├── MosisPhone/
|
||||
│ │ ├── MosisPhone.Build.cs
|
||||
│ │ ├── Public/
|
||||
│ │ │ ├── MosisPhoneActor.h
|
||||
│ │ │ ├── MosisService.h
|
||||
│ │ │ └── MosisHardwareProvider.h
|
||||
│ │ └── Private/
|
||||
│ │ ├── MosisPhoneActor.cpp
|
||||
│ │ ├── MosisService.cpp
|
||||
│ │ └── MosisHardwareProvider.cpp
|
||||
│ └── ThirdParty/
|
||||
│ └── MosisClient/
|
||||
│ └── libmosis-client.so
|
||||
├── Content/
|
||||
│ ├── BP_MosisPhone.uasset
|
||||
│ └── M_PhoneScreen.uasset
|
||||
└── Documentation/
|
||||
└── integration-guide.md
|
||||
```
|
||||
|
||||
### Main Actor
|
||||
|
||||
**File**: `MosisPhoneActor.h`
|
||||
|
||||
```cpp
|
||||
#pragma once
|
||||
|
||||
#include "CoreMinimal.h"
|
||||
#include "GameFramework/Actor.h"
|
||||
#include "MosisPhoneActor.generated.h"
|
||||
|
||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnPhoneReady);
|
||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnNavigate, FString, Screen);
|
||||
|
||||
UCLASS()
|
||||
class MOSISPHONE_API AMosisPhoneActor : public AActor {
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
AMosisPhoneActor();
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Display")
|
||||
FIntPoint Resolution = FIntPoint(540, 960);
|
||||
|
||||
UPROPERTY(BlueprintAssignable, Category = "Events")
|
||||
FOnPhoneReady OnPhoneReady;
|
||||
|
||||
UPROPERTY(BlueprintAssignable, Category = "Events")
|
||||
FOnNavigate OnNavigate;
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "Mosis")
|
||||
void NavigateTo(const FString& Screen);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "Mosis")
|
||||
void SendMessage(const FString& To, const FString& Text);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "Mosis")
|
||||
void MakeCall(const FString& Number);
|
||||
|
||||
protected:
|
||||
virtual void BeginPlay() override;
|
||||
virtual void Tick(float DeltaTime) override;
|
||||
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
|
||||
|
||||
private:
|
||||
UPROPERTY()
|
||||
UTextureRenderTarget2D* ScreenTexture;
|
||||
|
||||
UPROPERTY()
|
||||
UMaterialInstanceDynamic* ScreenMaterial;
|
||||
|
||||
TSharedPtr<FMosisService> Service;
|
||||
|
||||
void HandleRaycastInput();
|
||||
void OnFrameAvailable(const TArray<uint8>& Pixels);
|
||||
};
|
||||
```
|
||||
|
||||
### Blueprint Integration
|
||||
|
||||
```cpp
|
||||
// Blueprint callable functions for easy integration
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "Mosis", meta = (WorldContext = "WorldContextObject"))
|
||||
static AMosisPhoneActor* SpawnMosisPhone(
|
||||
UObject* WorldContextObject,
|
||||
FTransform SpawnTransform,
|
||||
FIntPoint Resolution = FIntPoint(540, 960)
|
||||
);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Hardware Button Simulation
|
||||
|
||||
### Physical Buttons
|
||||
|
||||
| Button | Function |
|
||||
|--------|----------|
|
||||
| Power | Lock/wake screen |
|
||||
| Volume Up | Increase volume |
|
||||
| Volume Down | Decrease volume |
|
||||
| Home (soft) | Go to home screen |
|
||||
| Back (soft) | Navigation back |
|
||||
|
||||
### Unity Implementation
|
||||
|
||||
```csharp
|
||||
public class MosisPhoneButtons : MonoBehaviour {
|
||||
[SerializeField] private MosisPhone phone;
|
||||
[SerializeField] private Collider powerButton;
|
||||
[SerializeField] private Collider volumeUp;
|
||||
[SerializeField] private Collider volumeDown;
|
||||
|
||||
void Update() {
|
||||
// Check for button presses via physics overlap
|
||||
if (IsButtonPressed(powerButton)) {
|
||||
phone.Service.SendButton("power");
|
||||
}
|
||||
// ...
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Unreal Implementation
|
||||
|
||||
```cpp
|
||||
// In Blueprint or C++
|
||||
UFUNCTION(BlueprintCallable)
|
||||
void PressButton(EMosisButton Button) {
|
||||
switch (Button) {
|
||||
case EMosisButton::Power:
|
||||
Service->SendButton("power");
|
||||
break;
|
||||
case EMosisButton::VolumeUp:
|
||||
Service->SendButton("volume_up");
|
||||
break;
|
||||
case EMosisButton::VolumeDown:
|
||||
Service->SendButton("volume_down");
|
||||
break;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Phase 1: Unity Core
|
||||
- [ ] Package structure
|
||||
- [ ] MosisPhone component
|
||||
- [ ] Basic touch input
|
||||
- [ ] Frame rendering
|
||||
|
||||
### Phase 2: Unity VR
|
||||
- [ ] VR raycast interaction
|
||||
- [ ] Laser pointer visualization
|
||||
- [ ] Controller haptics
|
||||
- [ ] Two-handed support
|
||||
|
||||
### Phase 3: Unity Hardware
|
||||
- [ ] Camera provider (RenderTexture)
|
||||
- [ ] Microphone provider
|
||||
- [ ] Speaker output
|
||||
- [ ] Vibration
|
||||
|
||||
### Phase 4: Unreal Core
|
||||
- [ ] Plugin structure
|
||||
- [ ] MosisPhoneActor
|
||||
- [ ] Blueprint integration
|
||||
- [ ] Touch input
|
||||
|
||||
### Phase 5: Unreal VR
|
||||
- [ ] Motion controller input
|
||||
- [ ] Widget interaction
|
||||
- [ ] Hardware providers
|
||||
|
||||
### Phase 6: Documentation
|
||||
- [ ] Quick start guide
|
||||
- [ ] API reference
|
||||
- [ ] Sample scenes
|
||||
- [ ] Troubleshooting
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
### Unity Test Scene
|
||||
|
||||
1. Phone mounted in VR scene
|
||||
2. Touch interaction works
|
||||
3. Virtual camera shows game view
|
||||
4. Audio plays through virtual speaker
|
||||
|
||||
### Unreal Test Level
|
||||
|
||||
1. Blueprint phone actor
|
||||
2. Motion controller interaction
|
||||
3. Event callbacks work
|
||||
4. Performance acceptable
|
||||
|
||||
---
|
||||
|
||||
## Performance Targets
|
||||
|
||||
| Metric | Target |
|
||||
|--------|--------|
|
||||
| Frame latency | < 16ms |
|
||||
| Touch latency | < 50ms |
|
||||
| Memory usage | < 100MB |
|
||||
| Draw calls | < 5 per phone |
|
||||
|
||||
---
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
### Unity
|
||||
- [ ] One-click prefab placement
|
||||
- [ ] VR raycast touch works
|
||||
- [ ] Events fire correctly
|
||||
- [ ] Camera feed from game
|
||||
- [ ] Audio bidirectional
|
||||
|
||||
### Unreal
|
||||
- [ ] Blueprint spawnable
|
||||
- [ ] Motion controller support
|
||||
- [ ] Blueprint events
|
||||
- [ ] Hardware providers
|
||||
|
||||
### Documentation
|
||||
- [ ] Integration guide
|
||||
- [ ] API docs
|
||||
- [ ] Example projects
|
||||
- [ ] Video tutorial
|
||||
Reference in New Issue
Block a user