Designer click handling: - Fix DPI scaling in MouseButtonCallback and CursorPosCallback - Scale coordinates from window space to framebuffer/RmlUi context - Remove window resizing in ResizeToPhone (caused DPI mismatches) Test framework: - Fix SendMouseDown to use MOUSEEVENTF_MOVE before button down - Remove double-scaling in ScaleToPhysical (WindowController handles it) - All 5 UI navigation tests now pass Kernel API: - Add goHome() Lua function to return to home screen - Stops any running third-party apps before navigating Test app: - Update sandbox-test to use goHome() instead of goBack() Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
311 lines
9.8 KiB
C++
311 lines
9.8 KiB
C++
// Window controller implementation
|
|
#include "window_controller.h"
|
|
#include <iostream>
|
|
#include <chrono>
|
|
#include <thread>
|
|
|
|
namespace mosis::test {
|
|
|
|
// Callback for EnumWindows to find window by partial title match
|
|
struct FindWindowData {
|
|
std::string searchTitle;
|
|
HWND foundHwnd = nullptr;
|
|
};
|
|
|
|
static BOOL CALLBACK EnumWindowsCallback(HWND hwnd, LPARAM lParam) {
|
|
auto* data = reinterpret_cast<FindWindowData*>(lParam);
|
|
|
|
char title[256] = {0};
|
|
GetWindowTextA(hwnd, title, sizeof(title));
|
|
|
|
// Check if the window title contains our search string
|
|
if (std::string(title).find(data->searchTitle) != std::string::npos) {
|
|
data->foundHwnd = hwnd;
|
|
return FALSE; // Stop enumeration
|
|
}
|
|
return TRUE; // Continue enumeration
|
|
}
|
|
|
|
bool WindowController::FindWindow(const std::string& title) {
|
|
// Use EnumWindows to find window by partial title match
|
|
// This allows finding "Mosis Designer - home.rml" when searching for "Mosis Designer"
|
|
FindWindowData data;
|
|
data.searchTitle = title;
|
|
|
|
EnumWindows(EnumWindowsCallback, reinterpret_cast<LPARAM>(&data));
|
|
|
|
m_hwnd = data.foundHwnd;
|
|
if (!m_hwnd) {
|
|
return false;
|
|
}
|
|
|
|
m_info.hwnd = m_hwnd;
|
|
|
|
// Get window rect
|
|
::GetWindowRect(m_hwnd, &m_info.windowRect);
|
|
|
|
// Get client rect (relative to window)
|
|
RECT clientRect;
|
|
::GetClientRect(m_hwnd, &clientRect);
|
|
m_info.clientRect = clientRect;
|
|
|
|
// Convert client area origin to screen coordinates
|
|
POINT clientOrigin = {0, 0};
|
|
::ClientToScreen(m_hwnd, &clientOrigin);
|
|
m_info.clientX = clientOrigin.x;
|
|
m_info.clientY = clientOrigin.y;
|
|
|
|
// Calculate client size and DPI scale
|
|
int clientWidth = clientRect.right - clientRect.left;
|
|
int clientHeight = clientRect.bottom - clientRect.top;
|
|
m_info.clientWidth = clientWidth;
|
|
m_info.clientHeight = clientHeight;
|
|
|
|
m_info.scaleX = static_cast<float>(clientWidth) / PHONE_WIDTH;
|
|
m_info.scaleY = static_cast<float>(clientHeight) / PHONE_HEIGHT;
|
|
|
|
std::cout << "Found window: " << title << std::endl;
|
|
std::cout << " Window rect: " << m_info.windowRect.left << "," << m_info.windowRect.top
|
|
<< " - " << m_info.windowRect.right << "," << m_info.windowRect.bottom << std::endl;
|
|
std::cout << " Client origin: " << m_info.clientX << "," << m_info.clientY << std::endl;
|
|
std::cout << " Client size: " << clientWidth << "x" << clientHeight << std::endl;
|
|
std::cout << " DPI scale: " << m_info.scaleX << " x " << m_info.scaleY << std::endl;
|
|
|
|
return true;
|
|
}
|
|
|
|
bool WindowController::WaitForWindow(const std::string& title, int timeoutMs) {
|
|
auto startTime = std::chrono::steady_clock::now();
|
|
|
|
while (true) {
|
|
if (FindWindow(title)) {
|
|
return true;
|
|
}
|
|
|
|
auto elapsed = std::chrono::steady_clock::now() - startTime;
|
|
if (std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count() >= timeoutMs) {
|
|
std::cerr << "Timeout waiting for window: " << title << std::endl;
|
|
return false;
|
|
}
|
|
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
|
}
|
|
}
|
|
|
|
LPARAM WindowController::PhoneToClientLParam(int phoneX, int phoneY) {
|
|
// Scale phone coordinates to client coordinates
|
|
int clientX = static_cast<int>(phoneX * m_info.scaleX);
|
|
int clientY = static_cast<int>(phoneY * m_info.scaleY);
|
|
|
|
// Clamp to client area bounds
|
|
clientX = std::max(0, std::min(clientX, static_cast<int>(PHONE_WIDTH * m_info.scaleX) - 1));
|
|
clientY = std::max(0, std::min(clientY, static_cast<int>(PHONE_HEIGHT * m_info.scaleY) - 1));
|
|
|
|
return MAKELPARAM(clientX, clientY);
|
|
}
|
|
|
|
bool WindowController::SendMouseDown(int phoneX, int phoneY) {
|
|
if (!m_hwnd) return false;
|
|
|
|
// Convert phone coordinates to client coordinates (using window scale)
|
|
int clientX = static_cast<int>(phoneX * m_info.scaleX);
|
|
int clientY = static_cast<int>(phoneY * m_info.scaleY);
|
|
|
|
// Calculate screen coordinates
|
|
int screenX = m_info.clientX + clientX;
|
|
int screenY = m_info.clientY + clientY;
|
|
|
|
// Get DPI info for debugging
|
|
UINT dpi = GetDpiForWindow(m_hwnd);
|
|
|
|
// Ensure window is foreground before clicking
|
|
SetForegroundWindow(m_hwnd);
|
|
Sleep(10);
|
|
|
|
// Get screen dimensions for absolute coordinate conversion
|
|
int screenWidth = GetSystemMetrics(SM_CXSCREEN);
|
|
int screenHeight = GetSystemMetrics(SM_CYSCREEN);
|
|
|
|
// Convert to normalized absolute coordinates (0-65535)
|
|
LONG absX = static_cast<LONG>((screenX * 65535) / screenWidth);
|
|
LONG absY = static_cast<LONG>((screenY * 65535) / screenHeight);
|
|
|
|
// Send mouse move then button down
|
|
INPUT moveInput = {};
|
|
moveInput.type = INPUT_MOUSE;
|
|
moveInput.mi.dx = absX;
|
|
moveInput.mi.dy = absY;
|
|
moveInput.mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE;
|
|
SendInput(1, &moveInput, sizeof(INPUT));
|
|
|
|
Sleep(20);
|
|
|
|
INPUT clickInput = {};
|
|
clickInput.type = INPUT_MOUSE;
|
|
clickInput.mi.dwFlags = MOUSEEVENTF_LEFTDOWN;
|
|
SendInput(1, &clickInput, sizeof(INPUT));
|
|
|
|
std::cout << "MouseDown at phone(" << phoneX << "," << phoneY << ") -> client("
|
|
<< clientX << "," << clientY << ") screen(" << screenX << "," << screenY
|
|
<< ") dpi=" << dpi << std::endl;
|
|
|
|
return true;
|
|
}
|
|
|
|
bool WindowController::SendMouseUp(int phoneX, int phoneY) {
|
|
if (!m_hwnd) return false;
|
|
|
|
Sleep(10); // Small delay before release
|
|
|
|
INPUT input = {};
|
|
input.type = INPUT_MOUSE;
|
|
input.mi.dwFlags = MOUSEEVENTF_LEFTUP;
|
|
SendInput(1, &input, sizeof(INPUT));
|
|
|
|
Sleep(10); // Small delay after release
|
|
|
|
std::cout << "MouseUp at phone(" << phoneX << "," << phoneY << ")" << std::endl;
|
|
|
|
return true;
|
|
}
|
|
|
|
bool WindowController::SendMouseMove(int phoneX, int phoneY) {
|
|
if (!m_hwnd) return false;
|
|
|
|
int clientX = static_cast<int>(phoneX * m_info.scaleX);
|
|
int clientY = static_cast<int>(phoneY * m_info.scaleY);
|
|
int screenX = m_info.clientX + clientX;
|
|
int screenY = m_info.clientY + clientY;
|
|
|
|
SetCursorPos(screenX, screenY);
|
|
return true;
|
|
}
|
|
|
|
bool WindowController::SendClick(int phoneX, int phoneY) {
|
|
if (!m_hwnd) return false;
|
|
|
|
// Save current cursor position
|
|
POINT oldPos;
|
|
GetCursorPos(&oldPos);
|
|
|
|
// Ensure window is active
|
|
SetForegroundWindow(m_hwnd);
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(50));
|
|
|
|
// Perform click
|
|
SendMouseDown(phoneX, phoneY);
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(50));
|
|
SendMouseUp(phoneX, phoneY);
|
|
|
|
// Restore cursor position (optional - comment out if cursor restoration causes issues)
|
|
// std::this_thread::sleep_for(std::chrono::milliseconds(50));
|
|
// SetCursorPos(oldPos.x, oldPos.y);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool WindowController::SendKey(UINT vkCode) {
|
|
if (!m_hwnd) return false;
|
|
|
|
// Send key down and up
|
|
LPARAM lParam = 1; // Repeat count = 1
|
|
::PostMessage(m_hwnd, WM_KEYDOWN, vkCode, lParam);
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(50));
|
|
::PostMessage(m_hwnd, WM_KEYUP, vkCode, lParam | (1 << 30) | (1 << 31));
|
|
|
|
return true;
|
|
}
|
|
|
|
bool WindowController::SendChar(char c) {
|
|
if (!m_hwnd) return false;
|
|
|
|
::PostMessage(m_hwnd, WM_CHAR, static_cast<WPARAM>(c), 0);
|
|
return true;
|
|
}
|
|
|
|
bool WindowController::Activate() {
|
|
if (!m_hwnd) return false;
|
|
|
|
::SetForegroundWindow(m_hwnd);
|
|
return true;
|
|
}
|
|
|
|
bool WindowController::Close() {
|
|
if (!m_hwnd) return false;
|
|
|
|
::PostMessage(m_hwnd, WM_CLOSE, 0, 0);
|
|
return true;
|
|
}
|
|
|
|
bool WindowController::ResizeToPhone() {
|
|
if (!m_hwnd) return false;
|
|
|
|
// Skip resizing - the designer creates the window at the correct size
|
|
// and resizing causes DPI scaling mismatches between window and framebuffer.
|
|
// Just move window to top-left for consistent test positioning.
|
|
|
|
RECT currentRect;
|
|
::GetWindowRect(m_hwnd, ¤tRect);
|
|
int currentWidth = currentRect.right - currentRect.left;
|
|
int currentHeight = currentRect.bottom - currentRect.top;
|
|
|
|
std::cout << "ResizeToPhone: keeping current size " << currentWidth << "x" << currentHeight
|
|
<< ", moving to (0,0)" << std::endl;
|
|
|
|
// Move window to top-left for consistent positioning
|
|
::MoveWindow(m_hwnd, 0, 0, currentWidth, currentHeight, TRUE);
|
|
|
|
// Re-acquire window info
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
|
return FindWindow("Mosis Designer");
|
|
}
|
|
|
|
bool WindowController::SendClickFromBottom(int clientX, int offsetFromBottom) {
|
|
if (!m_hwnd) return false;
|
|
|
|
// Calculate Y position from bottom of client area
|
|
int clientY = m_info.clientHeight - offsetFromBottom;
|
|
|
|
// Convert to screen coordinates
|
|
int screenX = m_info.clientX + clientX;
|
|
int screenY = m_info.clientY + clientY;
|
|
|
|
// Ensure window is active
|
|
SetForegroundWindow(m_hwnd);
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(50));
|
|
|
|
// Get screen dimensions for absolute coordinate conversion
|
|
int screenWidth = GetSystemMetrics(SM_CXSCREEN);
|
|
int screenHeight = GetSystemMetrics(SM_CYSCREEN);
|
|
|
|
// Convert to normalized absolute coordinates (0-65535)
|
|
LONG absX = static_cast<LONG>((screenX * 65535) / screenWidth);
|
|
LONG absY = static_cast<LONG>((screenY * 65535) / screenHeight);
|
|
|
|
// Use SendInput with absolute move + click
|
|
INPUT inputs[3] = {};
|
|
|
|
// Move cursor
|
|
inputs[0].type = INPUT_MOUSE;
|
|
inputs[0].mi.dx = absX;
|
|
inputs[0].mi.dy = absY;
|
|
inputs[0].mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE;
|
|
|
|
// Mouse down
|
|
inputs[1].type = INPUT_MOUSE;
|
|
inputs[1].mi.dwFlags = MOUSEEVENTF_LEFTDOWN;
|
|
|
|
// Mouse up
|
|
inputs[2].type = INPUT_MOUSE;
|
|
inputs[2].mi.dwFlags = MOUSEEVENTF_LEFTUP;
|
|
|
|
SendInput(3, inputs, sizeof(INPUT));
|
|
|
|
std::cout << "ClickFromBottom at client(" << clientX << "," << clientY
|
|
<< ") -> screen(" << screenX << "," << screenY << ")" << std::endl;
|
|
|
|
return true;
|
|
}
|
|
|
|
} // namespace mosis::test
|