Files
MosisService/designer-test/src/window_controller.cpp
omigamedev 984e8715d7 Fix desktop designer click handling and add goHome API
Designer click handling:
- Fix DPI scaling in MouseButtonCallback and CursorPosCallback
- Scale coordinates from window space to framebuffer/RmlUi context
- Remove window resizing in ResizeToPhone (caused DPI mismatches)

Test framework:
- Fix SendMouseDown to use MOUSEEVENTF_MOVE before button down
- Remove double-scaling in ScaleToPhysical (WindowController handles it)
- All 5 UI navigation tests now pass

Kernel API:
- Add goHome() Lua function to return to home screen
- Stops any running third-party apps before navigating

Test app:
- Update sandbox-test to use goHome() instead of goBack()

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-19 14:52:15 +01:00

311 lines
9.8 KiB
C++

// Window controller implementation
#include "window_controller.h"
#include <iostream>
#include <chrono>
#include <thread>
namespace mosis::test {
// Callback for EnumWindows to find window by partial title match
struct FindWindowData {
std::string searchTitle;
HWND foundHwnd = nullptr;
};
static BOOL CALLBACK EnumWindowsCallback(HWND hwnd, LPARAM lParam) {
auto* data = reinterpret_cast<FindWindowData*>(lParam);
char title[256] = {0};
GetWindowTextA(hwnd, title, sizeof(title));
// Check if the window title contains our search string
if (std::string(title).find(data->searchTitle) != std::string::npos) {
data->foundHwnd = hwnd;
return FALSE; // Stop enumeration
}
return TRUE; // Continue enumeration
}
bool WindowController::FindWindow(const std::string& title) {
// Use EnumWindows to find window by partial title match
// This allows finding "Mosis Designer - home.rml" when searching for "Mosis Designer"
FindWindowData data;
data.searchTitle = title;
EnumWindows(EnumWindowsCallback, reinterpret_cast<LPARAM>(&data));
m_hwnd = data.foundHwnd;
if (!m_hwnd) {
return false;
}
m_info.hwnd = m_hwnd;
// Get window rect
::GetWindowRect(m_hwnd, &m_info.windowRect);
// Get client rect (relative to window)
RECT clientRect;
::GetClientRect(m_hwnd, &clientRect);
m_info.clientRect = clientRect;
// Convert client area origin to screen coordinates
POINT clientOrigin = {0, 0};
::ClientToScreen(m_hwnd, &clientOrigin);
m_info.clientX = clientOrigin.x;
m_info.clientY = clientOrigin.y;
// Calculate client size and DPI scale
int clientWidth = clientRect.right - clientRect.left;
int clientHeight = clientRect.bottom - clientRect.top;
m_info.clientWidth = clientWidth;
m_info.clientHeight = clientHeight;
m_info.scaleX = static_cast<float>(clientWidth) / PHONE_WIDTH;
m_info.scaleY = static_cast<float>(clientHeight) / PHONE_HEIGHT;
std::cout << "Found window: " << title << std::endl;
std::cout << " Window rect: " << m_info.windowRect.left << "," << m_info.windowRect.top
<< " - " << m_info.windowRect.right << "," << m_info.windowRect.bottom << std::endl;
std::cout << " Client origin: " << m_info.clientX << "," << m_info.clientY << std::endl;
std::cout << " Client size: " << clientWidth << "x" << clientHeight << std::endl;
std::cout << " DPI scale: " << m_info.scaleX << " x " << m_info.scaleY << std::endl;
return true;
}
bool WindowController::WaitForWindow(const std::string& title, int timeoutMs) {
auto startTime = std::chrono::steady_clock::now();
while (true) {
if (FindWindow(title)) {
return true;
}
auto elapsed = std::chrono::steady_clock::now() - startTime;
if (std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count() >= timeoutMs) {
std::cerr << "Timeout waiting for window: " << title << std::endl;
return false;
}
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
}
LPARAM WindowController::PhoneToClientLParam(int phoneX, int phoneY) {
// Scale phone coordinates to client coordinates
int clientX = static_cast<int>(phoneX * m_info.scaleX);
int clientY = static_cast<int>(phoneY * m_info.scaleY);
// Clamp to client area bounds
clientX = std::max(0, std::min(clientX, static_cast<int>(PHONE_WIDTH * m_info.scaleX) - 1));
clientY = std::max(0, std::min(clientY, static_cast<int>(PHONE_HEIGHT * m_info.scaleY) - 1));
return MAKELPARAM(clientX, clientY);
}
bool WindowController::SendMouseDown(int phoneX, int phoneY) {
if (!m_hwnd) return false;
// Convert phone coordinates to client coordinates (using window scale)
int clientX = static_cast<int>(phoneX * m_info.scaleX);
int clientY = static_cast<int>(phoneY * m_info.scaleY);
// Calculate screen coordinates
int screenX = m_info.clientX + clientX;
int screenY = m_info.clientY + clientY;
// Get DPI info for debugging
UINT dpi = GetDpiForWindow(m_hwnd);
// Ensure window is foreground before clicking
SetForegroundWindow(m_hwnd);
Sleep(10);
// Get screen dimensions for absolute coordinate conversion
int screenWidth = GetSystemMetrics(SM_CXSCREEN);
int screenHeight = GetSystemMetrics(SM_CYSCREEN);
// Convert to normalized absolute coordinates (0-65535)
LONG absX = static_cast<LONG>((screenX * 65535) / screenWidth);
LONG absY = static_cast<LONG>((screenY * 65535) / screenHeight);
// Send mouse move then button down
INPUT moveInput = {};
moveInput.type = INPUT_MOUSE;
moveInput.mi.dx = absX;
moveInput.mi.dy = absY;
moveInput.mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE;
SendInput(1, &moveInput, sizeof(INPUT));
Sleep(20);
INPUT clickInput = {};
clickInput.type = INPUT_MOUSE;
clickInput.mi.dwFlags = MOUSEEVENTF_LEFTDOWN;
SendInput(1, &clickInput, sizeof(INPUT));
std::cout << "MouseDown at phone(" << phoneX << "," << phoneY << ") -> client("
<< clientX << "," << clientY << ") screen(" << screenX << "," << screenY
<< ") dpi=" << dpi << std::endl;
return true;
}
bool WindowController::SendMouseUp(int phoneX, int phoneY) {
if (!m_hwnd) return false;
Sleep(10); // Small delay before release
INPUT input = {};
input.type = INPUT_MOUSE;
input.mi.dwFlags = MOUSEEVENTF_LEFTUP;
SendInput(1, &input, sizeof(INPUT));
Sleep(10); // Small delay after release
std::cout << "MouseUp at phone(" << phoneX << "," << phoneY << ")" << std::endl;
return true;
}
bool WindowController::SendMouseMove(int phoneX, int phoneY) {
if (!m_hwnd) return false;
int clientX = static_cast<int>(phoneX * m_info.scaleX);
int clientY = static_cast<int>(phoneY * m_info.scaleY);
int screenX = m_info.clientX + clientX;
int screenY = m_info.clientY + clientY;
SetCursorPos(screenX, screenY);
return true;
}
bool WindowController::SendClick(int phoneX, int phoneY) {
if (!m_hwnd) return false;
// Save current cursor position
POINT oldPos;
GetCursorPos(&oldPos);
// Ensure window is active
SetForegroundWindow(m_hwnd);
std::this_thread::sleep_for(std::chrono::milliseconds(50));
// Perform click
SendMouseDown(phoneX, phoneY);
std::this_thread::sleep_for(std::chrono::milliseconds(50));
SendMouseUp(phoneX, phoneY);
// Restore cursor position (optional - comment out if cursor restoration causes issues)
// std::this_thread::sleep_for(std::chrono::milliseconds(50));
// SetCursorPos(oldPos.x, oldPos.y);
return true;
}
bool WindowController::SendKey(UINT vkCode) {
if (!m_hwnd) return false;
// Send key down and up
LPARAM lParam = 1; // Repeat count = 1
::PostMessage(m_hwnd, WM_KEYDOWN, vkCode, lParam);
std::this_thread::sleep_for(std::chrono::milliseconds(50));
::PostMessage(m_hwnd, WM_KEYUP, vkCode, lParam | (1 << 30) | (1 << 31));
return true;
}
bool WindowController::SendChar(char c) {
if (!m_hwnd) return false;
::PostMessage(m_hwnd, WM_CHAR, static_cast<WPARAM>(c), 0);
return true;
}
bool WindowController::Activate() {
if (!m_hwnd) return false;
::SetForegroundWindow(m_hwnd);
return true;
}
bool WindowController::Close() {
if (!m_hwnd) return false;
::PostMessage(m_hwnd, WM_CLOSE, 0, 0);
return true;
}
bool WindowController::ResizeToPhone() {
if (!m_hwnd) return false;
// Skip resizing - the designer creates the window at the correct size
// and resizing causes DPI scaling mismatches between window and framebuffer.
// Just move window to top-left for consistent test positioning.
RECT currentRect;
::GetWindowRect(m_hwnd, &currentRect);
int currentWidth = currentRect.right - currentRect.left;
int currentHeight = currentRect.bottom - currentRect.top;
std::cout << "ResizeToPhone: keeping current size " << currentWidth << "x" << currentHeight
<< ", moving to (0,0)" << std::endl;
// Move window to top-left for consistent positioning
::MoveWindow(m_hwnd, 0, 0, currentWidth, currentHeight, TRUE);
// Re-acquire window info
std::this_thread::sleep_for(std::chrono::milliseconds(100));
return FindWindow("Mosis Designer");
}
bool WindowController::SendClickFromBottom(int clientX, int offsetFromBottom) {
if (!m_hwnd) return false;
// Calculate Y position from bottom of client area
int clientY = m_info.clientHeight - offsetFromBottom;
// Convert to screen coordinates
int screenX = m_info.clientX + clientX;
int screenY = m_info.clientY + clientY;
// Ensure window is active
SetForegroundWindow(m_hwnd);
std::this_thread::sleep_for(std::chrono::milliseconds(50));
// Get screen dimensions for absolute coordinate conversion
int screenWidth = GetSystemMetrics(SM_CXSCREEN);
int screenHeight = GetSystemMetrics(SM_CYSCREEN);
// Convert to normalized absolute coordinates (0-65535)
LONG absX = static_cast<LONG>((screenX * 65535) / screenWidth);
LONG absY = static_cast<LONG>((screenY * 65535) / screenHeight);
// Use SendInput with absolute move + click
INPUT inputs[3] = {};
// Move cursor
inputs[0].type = INPUT_MOUSE;
inputs[0].mi.dx = absX;
inputs[0].mi.dy = absY;
inputs[0].mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE;
// Mouse down
inputs[1].type = INPUT_MOUSE;
inputs[1].mi.dwFlags = MOUSEEVENTF_LEFTDOWN;
// Mouse up
inputs[2].type = INPUT_MOUSE;
inputs[2].mi.dwFlags = MOUSEEVENTF_LEFTUP;
SendInput(3, inputs, sizeof(INPUT));
std::cout << "ClickFromBottom at client(" << clientX << "," << clientY
<< ") -> screen(" << screenX << "," << screenY << ")" << std::endl;
return true;
}
} // namespace mosis::test