MosisService/MILESTONE-7.md

Milestone 7: Game Integration

Status: Not Started Goal: Production-ready Unity and Unreal plugins for seamless VR phone integration.

---

Overview

Game engine plugins enable:

• Rendering the phone in VR scenes
• Touch/raycast interaction
• Virtual hardware provision (camera, mic, speaker)
• Event callbacks to game code

---

Current State

Platform	Location	Status
Unity	D:\Dev\Mosis\Mosis Unity	Basic Binder client
Unreal	D:\Dev\Mosis\Mosis Unreal	WIP

---

Unity Plugin

Package Structure

com.mosis.phone/
├── package.json
├── Runtime/
│   ├── MosisPhone.cs           # Main component
│   ├── MosisService.cs         # Binder client
│   ├── MosisInputHandler.cs    # Touch/raycast
│   ├── MosisHardwareProvider.cs # Virtual hardware
│   └── Native/
│       ├── MosisNative.cs      # P/Invoke declarations
│       └── Plugins/
│           └── Android/
│               └── libmosis-client.so
├── Prefabs/
│   ├── MosisPhone.prefab       # Ready-to-use phone
│   └── MosisPhoneVR.prefab     # VR-optimized version
├── Samples~/
│   └── BasicIntegration/
│       └── PhoneDemo.unity
└── Documentation~/
    └── integration-guide.md

Main Component

File: MosisPhone.cs

using UnityEngine;
using UnityEngine.Events;

namespace Mosis {
    public class MosisPhone : MonoBehaviour {
        [Header("Display")]
        [SerializeField] private MeshRenderer phoneScreen;
        [SerializeField] private Vector2Int resolution = new Vector2Int(540, 960);

        [Header("Input")]
        [SerializeField] private bool enableRaycast = true;
        [SerializeField] private LayerMask raycastLayers;

        [Header("Virtual Hardware")]
        [SerializeField] private Camera virtualCamera;
        [SerializeField] private AudioSource virtualMicrophone;
        [SerializeField] private AudioSource virtualSpeaker;

        [Header("Events")]
        public UnityEvent onPhoneReady;
        public UnityEvent<string> onNavigate;
        public UnityEvent<string, string> onMessageReceived;
        public UnityEvent<string> onCallStarted;

        private MosisService service;
        private RenderTexture screenTexture;
        private Material screenMaterial;

        void Awake() {
            // Create render texture for phone screen
            screenTexture = new RenderTexture(resolution.x, resolution.y, 0);
            screenMaterial = new Material(Shader.Find("Unlit/Texture"));
            screenMaterial.mainTexture = screenTexture;
            phoneScreen.material = screenMaterial;
        }

        void Start() {
            service = new MosisService();
            service.OnFrameAvailable += OnFrameAvailable;
            service.OnServiceInitialized += () => onPhoneReady?.Invoke();
            service.Connect();
        }

        void Update() {
            if (enableRaycast) {
                HandleRaycastInput();
            }
        }

        private void HandleRaycastInput() {
            // VR controller raycast
            if (OVRInput.GetDown(OVRInput.Button.PrimaryIndexTrigger)) {
                var ray = new Ray(
                    OVRInput.GetLocalControllerPosition(OVRInput.Controller.RTouch),
                    OVRInput.GetLocalControllerRotation(OVRInput.Controller.RTouch) * Vector3.forward
                );

                if (Physics.Raycast(ray, out var hit, 10f, raycastLayers)) {
                    if (hit.collider.gameObject == phoneScreen.gameObject) {
                        var uv = hit.textureCoord;
                        var x = uv.x * resolution.x;
                        var y = (1 - uv.y) * resolution.y;
                        service.SendTouchDown(x, y);
                    }
                }
            }
        }

        private void OnFrameAvailable(byte[] pixels) {
            // Update screen texture
            screenTexture.LoadRawTextureData(pixels);
            screenTexture.Apply();
        }

        // Public API
        public void NavigateTo(string screen) {
            service.Navigate(screen);
        }

        public void SendMessage(string to, string text) {
            service.SendMessage(to, text);
        }

        public void MakeCall(string number) {
            service.MakeCall(number);
        }

        void OnDestroy() {
            service?.Disconnect();
            Destroy(screenTexture);
            Destroy(screenMaterial);
        }
    }
}

Hardware Provider

File: MosisHardwareProvider.cs

namespace Mosis {
    public class MosisHardwareProvider : MonoBehaviour {
        [SerializeField] private MosisPhone phone;
        [SerializeField] private Camera gameCamera;
        [SerializeField] private AudioListener audioListener;

        private RenderTexture cameraTexture;

        void Start() {
            // Setup virtual camera
            cameraTexture = new RenderTexture(640, 480, 0);
            gameCamera.targetTexture = cameraTexture;

            // Register with phone service
            phone.Service.SetCameraProvider(() => {
                var pixels = new byte[cameraTexture.width * cameraTexture.height * 4];
                // Read pixels from RenderTexture
                return pixels;
            });

            phone.Service.SetMicrophoneProvider(() => {
                // Get audio from AudioListener
                return audioSamples;
            });
        }

        public void PlayAudio(float[] samples) {
            // Play on virtual speaker
            var audioSource = GetComponent<AudioSource>();
            audioSource.clip = AudioClip.Create("phone", samples.Length, 1, 44100, false);
            audioSource.clip.SetData(samples, 0);
            audioSource.Play();
        }
    }
}

VR Interaction

File: MosisInputHandler.cs

namespace Mosis {
    public class MosisInputHandler : MonoBehaviour {
        [SerializeField] private MosisPhone phone;
        [SerializeField] private Transform pointerOrigin;
        [SerializeField] private LineRenderer laserPointer;

        private bool isTouching;
        private Vector2 lastTouchPos;

        void Update() {
            // Update laser pointer
            var ray = new Ray(pointerOrigin.position, pointerOrigin.forward);

            if (Physics.Raycast(ray, out var hit, 10f)) {
                laserPointer.SetPosition(1, hit.point);

                if (hit.collider.GetComponent<MosisPhone>() != null) {
                    var uv = hit.textureCoord;
                    var touchPos = new Vector2(
                        uv.x * phone.Resolution.x,
                        (1 - uv.y) * phone.Resolution.y
                    );

                    HandleTouch(touchPos);
                }
            }
        }

        private void HandleTouch(Vector2 pos) {
            bool triggerDown = OVRInput.Get(OVRInput.Button.PrimaryIndexTrigger);

            if (triggerDown && !isTouching) {
                phone.Service.SendTouchDown(pos.x, pos.y);
                isTouching = true;
            }
            else if (triggerDown && isTouching) {
                if (Vector2.Distance(pos, lastTouchPos) > 2f) {
                    phone.Service.SendTouchMove(pos.x, pos.y);
                }
            }
            else if (!triggerDown && isTouching) {
                phone.Service.SendTouchUp(pos.x, pos.y);
                isTouching = false;
            }

            lastTouchPos = pos;
        }
    }
}

---

Unreal Plugin

Module Structure

MosisPhone/
├── MosisPhone.uplugin
├── Source/
│   ├── MosisPhone/
│   │   ├── MosisPhone.Build.cs
│   │   ├── Public/
│   │   │   ├── MosisPhoneActor.h
│   │   │   ├── MosisService.h
│   │   │   └── MosisHardwareProvider.h
│   │   └── Private/
│   │       ├── MosisPhoneActor.cpp
│   │       ├── MosisService.cpp
│   │       └── MosisHardwareProvider.cpp
│   └── ThirdParty/
│       └── MosisClient/
│           └── libmosis-client.so
├── Content/
│   ├── BP_MosisPhone.uasset
│   └── M_PhoneScreen.uasset
└── Documentation/
    └── integration-guide.md

Main Actor

File: MosisPhoneActor.h

#pragma once

#include "CoreMinimal.h"
#include "GameFramework/Actor.h"
#include "MosisPhoneActor.generated.h"

DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnPhoneReady);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnNavigate, FString, Screen);

UCLASS()
class MOSISPHONE_API AMosisPhoneActor : public AActor {
    GENERATED_BODY()

public:
    AMosisPhoneActor();

    UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Display")
    FIntPoint Resolution = FIntPoint(540, 960);

    UPROPERTY(BlueprintAssignable, Category = "Events")
    FOnPhoneReady OnPhoneReady;

    UPROPERTY(BlueprintAssignable, Category = "Events")
    FOnNavigate OnNavigate;

    UFUNCTION(BlueprintCallable, Category = "Mosis")
    void NavigateTo(const FString& Screen);

    UFUNCTION(BlueprintCallable, Category = "Mosis")
    void SendMessage(const FString& To, const FString& Text);

    UFUNCTION(BlueprintCallable, Category = "Mosis")
    void MakeCall(const FString& Number);

protected:
    virtual void BeginPlay() override;
    virtual void Tick(float DeltaTime) override;
    virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;

private:
    UPROPERTY()
    UTextureRenderTarget2D* ScreenTexture;

    UPROPERTY()
    UMaterialInstanceDynamic* ScreenMaterial;

    TSharedPtr<FMosisService> Service;

    void HandleRaycastInput();
    void OnFrameAvailable(const TArray<uint8>& Pixels);
};

Blueprint Integration

// Blueprint callable functions for easy integration

UFUNCTION(BlueprintCallable, Category = "Mosis", meta = (WorldContext = "WorldContextObject"))
static AMosisPhoneActor* SpawnMosisPhone(
    UObject* WorldContextObject,
    FTransform SpawnTransform,
    FIntPoint Resolution = FIntPoint(540, 960)
);

---

Hardware Button Simulation

Physical Buttons

Button	Function
Power	Lock/wake screen
Volume Up	Increase volume
Volume Down	Decrease volume
Home (soft)	Go to home screen
Back (soft)	Navigation back

Unity Implementation

public class MosisPhoneButtons : MonoBehaviour {
    [SerializeField] private MosisPhone phone;
    [SerializeField] private Collider powerButton;
    [SerializeField] private Collider volumeUp;
    [SerializeField] private Collider volumeDown;

    void Update() {
        // Check for button presses via physics overlap
        if (IsButtonPressed(powerButton)) {
            phone.Service.SendButton("power");
        }
        // ...
    }
}

Unreal Implementation

// In Blueprint or C++
UFUNCTION(BlueprintCallable)
void PressButton(EMosisButton Button) {
    switch (Button) {
        case EMosisButton::Power:
            Service->SendButton("power");
            break;
        case EMosisButton::VolumeUp:
            Service->SendButton("volume_up");
            break;
        case EMosisButton::VolumeDown:
            Service->SendButton("volume_down");
            break;
    }
}

---

Implementation Plan

Phase 1: Unity Core

• Package structure
• MosisPhone component
• Basic touch input
• Frame rendering

Phase 2: Unity VR

• VR raycast interaction
• Laser pointer visualization
• Controller haptics
• Two-handed support

Phase 3: Unity Hardware

• Camera provider (RenderTexture)
• Microphone provider
• Speaker output
• Vibration

Phase 4: Unreal Core

• Plugin structure
• MosisPhoneActor
• Blueprint integration
• Touch input

Phase 5: Unreal VR

• Motion controller input
• Widget interaction
• Hardware providers

Phase 6: Documentation

• Quick start guide
• API reference
• Sample scenes
• Troubleshooting

---

Testing

Unity Test Scene

1. Phone mounted in VR scene
2. Touch interaction works
3. Virtual camera shows game view
4. Audio plays through virtual speaker

Unreal Test Level

1. Blueprint phone actor
2. Motion controller interaction
3. Event callbacks work
4. Performance acceptable

---

Performance Targets

Metric	Target
Frame latency	< 16ms
Touch latency	< 50ms
Memory usage	< 100MB
Draw calls	< 5 per phone

---

Acceptance Criteria

Unity

• One-click prefab placement
• VR raycast touch works
• Events fire correctly
• Camera feed from game
• Audio bidirectional

Unreal

• Blueprint spawnable
• Motion controller support
• Blueprint events
• Hardware providers

Documentation

• Integration guide
• API docs
• Example projects
• Video tutorial