add testing and plan milestones
This commit is contained in:
407
MILESTONE-5.md
Normal file
407
MILESTONE-5.md
Normal file
@@ -0,0 +1,407 @@
|
||||
# Milestone 5: WebRTC Bridge
|
||||
|
||||
**Status**: Not Started
|
||||
**Goal**: Cross-device communication via WebRTC for calls, messaging, and file sharing.
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
WebRTC enables:
|
||||
- Voice/video calls between virtual phones
|
||||
- Text messaging across different games
|
||||
- File sharing between devices
|
||||
- Screen sharing
|
||||
- Connection to real smartphones (companion app)
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────┐
|
||||
│ Virtual Phone A │
|
||||
│ (VR Game on User's PC) │
|
||||
│ ┌───────────────────────────────────────────────────────────────┐ │
|
||||
│ │ WebRTCBridge │ │
|
||||
│ │ ├── DataChannel (messages, files) │ │
|
||||
│ │ ├── AudioTrack (voice) │ │
|
||||
│ │ └── VideoTrack (camera/screen share) │ │
|
||||
│ └───────────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
│ WebRTC (UDP/DTLS-SRTP)
|
||||
│
|
||||
┌─────────┴─────────┐
|
||||
│ Signaling Server │
|
||||
│ (WebSocket) │
|
||||
└─────────┬─────────┘
|
||||
│
|
||||
┌───────────────┼───────────────┐
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Virtual Phone B │ │ Virtual Phone C │ │ Real Smartphone │
|
||||
│ (Different Game)│ │ (Same Game) │ │ (Companion App) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Dependencies
|
||||
|
||||
### libdatachannel
|
||||
|
||||
C++ WebRTC library for data channels, audio, and video.
|
||||
|
||||
**vcpkg installation**:
|
||||
```bash
|
||||
vcpkg install libdatachannel
|
||||
```
|
||||
|
||||
**CMakeLists.txt**:
|
||||
```cmake
|
||||
find_package(LibDataChannel CONFIG REQUIRED)
|
||||
target_link_libraries(mosis-kernel PRIVATE LibDataChannel::LibDataChannel)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Core Components
|
||||
|
||||
### WebRTC Bridge
|
||||
|
||||
**File**: `src/main/kernel/include/webrtc_bridge.h`
|
||||
|
||||
```cpp
|
||||
namespace mosis {
|
||||
|
||||
struct PeerInfo {
|
||||
std::string peer_id;
|
||||
std::string display_name;
|
||||
bool is_online;
|
||||
};
|
||||
|
||||
struct CallState {
|
||||
std::string peer_id;
|
||||
bool is_active;
|
||||
bool is_video;
|
||||
bool is_muted;
|
||||
int64_t start_time;
|
||||
};
|
||||
|
||||
class IWebRTCBridge {
|
||||
public:
|
||||
virtual ~IWebRTCBridge() = default;
|
||||
|
||||
// Connection
|
||||
virtual void Connect(const std::string& signaling_server_url) = 0;
|
||||
virtual void Disconnect() = 0;
|
||||
virtual bool IsConnected() const = 0;
|
||||
|
||||
// Identity
|
||||
virtual void SetIdentity(const std::string& phone_id,
|
||||
const std::string& display_name) = 0;
|
||||
virtual std::string GetPhoneId() const = 0;
|
||||
|
||||
// Peer discovery
|
||||
virtual std::vector<PeerInfo> GetOnlinePeers() = 0;
|
||||
|
||||
// Messaging
|
||||
using MessageCallback = std::function<void(const std::string& from,
|
||||
const std::string& message)>;
|
||||
virtual void SetMessageCallback(MessageCallback callback) = 0;
|
||||
virtual void SendMessage(const std::string& to, const std::string& message) = 0;
|
||||
|
||||
// Voice calls
|
||||
using CallCallback = std::function<void(const CallState& state)>;
|
||||
virtual void SetCallCallback(CallCallback callback) = 0;
|
||||
virtual void StartCall(const std::string& peer_id, bool with_video) = 0;
|
||||
virtual void AnswerCall(const std::string& peer_id) = 0;
|
||||
virtual void EndCall(const std::string& peer_id) = 0;
|
||||
virtual void MuteCall(bool muted) = 0;
|
||||
|
||||
// File transfer
|
||||
using FileTransferCallback = std::function<void(const std::string& from,
|
||||
const std::string& filename,
|
||||
const std::vector<uint8_t>& data)>;
|
||||
virtual void SetFileTransferCallback(FileTransferCallback callback) = 0;
|
||||
virtual void SendFile(const std::string& to,
|
||||
const std::string& filename,
|
||||
const std::vector<uint8_t>& data) = 0;
|
||||
};
|
||||
|
||||
} // namespace mosis
|
||||
```
|
||||
|
||||
### Implementation
|
||||
|
||||
**File**: `src/main/kernel/src/webrtc_bridge.cpp`
|
||||
|
||||
```cpp
|
||||
#include <rtc/rtc.hpp>
|
||||
#include "webrtc_bridge.h"
|
||||
|
||||
namespace mosis {
|
||||
|
||||
class WebRTCBridgeImpl : public IWebRTCBridge {
|
||||
public:
|
||||
WebRTCBridgeImpl() {
|
||||
rtc::InitLogger(rtc::LogLevel::Warning);
|
||||
}
|
||||
|
||||
void Connect(const std::string& signaling_url) override {
|
||||
m_ws = std::make_shared<rtc::WebSocket>();
|
||||
|
||||
m_ws->onOpen([this]() {
|
||||
SendSignaling({{"type", "register"}, {"id", m_phone_id}});
|
||||
});
|
||||
|
||||
m_ws->onMessage([this](std::variant<rtc::binary, rtc::string> msg) {
|
||||
if (auto* str = std::get_if<rtc::string>(&msg)) {
|
||||
HandleSignaling(nlohmann::json::parse(*str));
|
||||
}
|
||||
});
|
||||
|
||||
m_ws->open(signaling_url);
|
||||
}
|
||||
|
||||
void SendMessage(const std::string& to, const std::string& message) override {
|
||||
if (auto it = m_peers.find(to); it != m_peers.end()) {
|
||||
auto& peer = it->second;
|
||||
if (peer.data_channel && peer.data_channel->isOpen()) {
|
||||
nlohmann::json msg = {
|
||||
{"type", "message"},
|
||||
{"text", message}
|
||||
};
|
||||
peer.data_channel->send(msg.dump());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
struct PeerConnection {
|
||||
std::shared_ptr<rtc::PeerConnection> pc;
|
||||
std::shared_ptr<rtc::DataChannel> data_channel;
|
||||
std::shared_ptr<rtc::Track> audio_track;
|
||||
std::shared_ptr<rtc::Track> video_track;
|
||||
};
|
||||
|
||||
void CreatePeerConnection(const std::string& peer_id) {
|
||||
rtc::Configuration config;
|
||||
config.iceServers.emplace_back("stun:stun.l.google.com:19302");
|
||||
|
||||
auto pc = std::make_shared<rtc::PeerConnection>(config);
|
||||
|
||||
pc->onStateChange([this, peer_id](rtc::PeerConnection::State state) {
|
||||
// Handle connection state changes
|
||||
});
|
||||
|
||||
pc->onLocalDescription([this, peer_id](rtc::Description desc) {
|
||||
SendSignaling({
|
||||
{"type", desc.typeString()},
|
||||
{"to", peer_id},
|
||||
{"sdp", std::string(desc)}
|
||||
});
|
||||
});
|
||||
|
||||
pc->onLocalCandidate([this, peer_id](rtc::Candidate cand) {
|
||||
SendSignaling({
|
||||
{"type", "candidate"},
|
||||
{"to", peer_id},
|
||||
{"candidate", std::string(cand)}
|
||||
});
|
||||
});
|
||||
|
||||
m_peers[peer_id] = {pc, nullptr, nullptr, nullptr};
|
||||
}
|
||||
|
||||
void HandleSignaling(const nlohmann::json& msg);
|
||||
void SendSignaling(const nlohmann::json& msg);
|
||||
|
||||
std::string m_phone_id;
|
||||
std::shared_ptr<rtc::WebSocket> m_ws;
|
||||
std::map<std::string, PeerConnection> m_peers;
|
||||
MessageCallback m_message_cb;
|
||||
CallCallback m_call_cb;
|
||||
};
|
||||
|
||||
} // namespace mosis
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Signaling Protocol
|
||||
|
||||
### Message Types
|
||||
|
||||
```json
|
||||
// Register with server
|
||||
{"type": "register", "id": "phone_123", "name": "John's Phone"}
|
||||
|
||||
// Peer discovery
|
||||
{"type": "get_peers"}
|
||||
{"type": "peers", "list": [{"id": "phone_456", "name": "Jane's Phone", "online": true}]}
|
||||
|
||||
// WebRTC signaling
|
||||
{"type": "offer", "to": "phone_456", "sdp": "v=0\r\n..."}
|
||||
{"type": "answer", "to": "phone_123", "sdp": "v=0\r\n..."}
|
||||
{"type": "candidate", "to": "phone_456", "candidate": "candidate:..."}
|
||||
|
||||
// Call signaling
|
||||
{"type": "call_request", "to": "phone_456", "video": false}
|
||||
{"type": "call_accept", "to": "phone_123"}
|
||||
{"type": "call_reject", "to": "phone_123", "reason": "busy"}
|
||||
{"type": "call_end", "to": "phone_456"}
|
||||
```
|
||||
|
||||
### Signaling Server
|
||||
|
||||
**Simple Node.js reference implementation**:
|
||||
|
||||
```javascript
|
||||
const WebSocket = require('ws');
|
||||
const wss = new WebSocket.Server({ port: 8080 });
|
||||
|
||||
const peers = new Map();
|
||||
|
||||
wss.on('connection', (ws) => {
|
||||
ws.on('message', (data) => {
|
||||
const msg = JSON.parse(data);
|
||||
|
||||
if (msg.type === 'register') {
|
||||
peers.set(msg.id, { ws, name: msg.name });
|
||||
broadcast({ type: 'peer_joined', id: msg.id, name: msg.name });
|
||||
}
|
||||
else if (msg.to) {
|
||||
// Forward to recipient
|
||||
const peer = peers.get(msg.to);
|
||||
if (peer) {
|
||||
msg.from = [...peers.entries()].find(([id, p]) => p.ws === ws)?.[0];
|
||||
peer.ws.send(JSON.stringify(msg));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
ws.on('close', () => {
|
||||
const id = [...peers.entries()].find(([_, p]) => p.ws === ws)?.[0];
|
||||
if (id) {
|
||||
peers.delete(id);
|
||||
broadcast({ type: 'peer_left', id });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
function broadcast(msg) {
|
||||
peers.forEach((peer) => peer.ws.send(JSON.stringify(msg)));
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Lua API
|
||||
|
||||
```lua
|
||||
-- Connect to signaling server
|
||||
mosis.webrtc.connect("wss://signal.mosis.dev")
|
||||
|
||||
-- Set identity
|
||||
mosis.webrtc.setIdentity("user_12345", "John's Phone")
|
||||
|
||||
-- Get online peers
|
||||
local peers = mosis.webrtc.getPeers()
|
||||
for _, peer in ipairs(peers) do
|
||||
print(peer.id, peer.name, peer.online)
|
||||
end
|
||||
|
||||
-- Send message
|
||||
mosis.webrtc.sendMessage("peer_id", "Hello!")
|
||||
|
||||
-- Receive messages
|
||||
mosis.webrtc.onMessage(function(from, message)
|
||||
print("Message from " .. from .. ": " .. message)
|
||||
end)
|
||||
|
||||
-- Make a call
|
||||
mosis.webrtc.call("peer_id", { video = false })
|
||||
|
||||
-- Handle incoming call
|
||||
mosis.webrtc.onIncomingCall(function(from, hasVideo)
|
||||
-- Show call UI
|
||||
-- Accept: mosis.webrtc.answerCall(from)
|
||||
-- Reject: mosis.webrtc.rejectCall(from)
|
||||
end)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Real Smartphone Bridge
|
||||
|
||||
### Companion App
|
||||
|
||||
A mobile app (Android/iOS) that:
|
||||
1. Connects to same signaling server
|
||||
2. Bridges to real phone's contacts, messages
|
||||
3. Allows receiving calls from virtual phone
|
||||
4. Sends notifications to virtual phone
|
||||
|
||||
**Use Cases**:
|
||||
- Receive real SMS in virtual phone
|
||||
- Make real calls from VR
|
||||
- Sync contacts between real and virtual phone
|
||||
- Get push notifications in VR
|
||||
|
||||
---
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Phase 1: Core WebRTC
|
||||
- [ ] Add libdatachannel to vcpkg.json
|
||||
- [ ] Implement WebRTCBridgeImpl
|
||||
- [ ] Basic signaling protocol
|
||||
|
||||
### Phase 2: Messaging
|
||||
- [ ] Data channel messaging
|
||||
- [ ] Message history storage
|
||||
- [ ] Messages app integration
|
||||
|
||||
### Phase 3: Voice Calls
|
||||
- [ ] Audio track setup
|
||||
- [ ] Microphone/speaker integration
|
||||
- [ ] Dialer app integration
|
||||
|
||||
### Phase 4: Signaling Server
|
||||
- [ ] Production signaling server
|
||||
- [ ] User authentication
|
||||
- [ ] Peer discovery service
|
||||
|
||||
### Phase 5: Companion App
|
||||
- [ ] Android companion app
|
||||
- [ ] iOS companion app (future)
|
||||
- [ ] Contact/message sync
|
||||
|
||||
---
|
||||
|
||||
## Network Requirements
|
||||
|
||||
- UDP ports for WebRTC media
|
||||
- WebSocket for signaling
|
||||
- TURN server for NAT traversal (optional)
|
||||
|
||||
**TURN/STUN Configuration**:
|
||||
```cpp
|
||||
rtc::Configuration config;
|
||||
config.iceServers.emplace_back("stun:stun.l.google.com:19302");
|
||||
config.iceServers.emplace_back("turn:turn.mosis.dev:3478", "user", "pass");
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [ ] Two virtual phones can exchange messages
|
||||
- [ ] Voice calls work between virtual phones
|
||||
- [ ] Files can be transferred between devices
|
||||
- [ ] Connection survives game restarts (rejoin)
|
||||
- [ ] Messages app shows real-time chat
|
||||
- [ ] Dialer app can place/receive calls
|
||||
Reference in New Issue
Block a user