From 984e8715d7581336a9ee6b4c2cb40e7d3d0a3f04 Mon Sep 17 00:00:00 2001 From: omigamedev Date: Mon, 19 Jan 2026 14:52:15 +0100 Subject: [PATCH] Fix desktop designer click handling and add goHome API Designer click handling: - Fix DPI scaling in MouseButtonCallback and CursorPosCallback - Scale coordinates from window space to framebuffer/RmlUi context - Remove window resizing in ResizeToPhone (caused DPI mismatches) Test framework: - Fix SendMouseDown to use MOUSEEVENTF_MOVE before button down - Remove double-scaling in ScaleToPhysical (WindowController handles it) - All 5 UI navigation tests now pass Kernel API: - Add goHome() Lua function to return to home screen - Stops any running third-party apps before navigating Test app: - Update sandbox-test to use goHome() instead of goBack() Co-Authored-By: Claude Opus 4.5 --- designer-test/src/main.cpp | 17 +-- designer-test/src/window_controller.cpp | 120 ++++++++++++---------- designer/main.cpp | 41 ++++++-- src/main/cpp/kernel.cpp | 54 +++++++++- test-apps/com.mosis.sandbox-test/main.rml | 4 +- 5 files changed, 160 insertions(+), 76 deletions(-) diff --git a/designer-test/src/main.cpp b/designer-test/src/main.cpp index ea62d4a..7ff52ad 100644 --- a/designer-test/src/main.cpp +++ b/designer-test/src/main.cpp @@ -10,19 +10,20 @@ using namespace mosis::test; -// Helper: Scale coordinates from hierarchy (logical) to window (physical) space +// Helper: Scale coordinates from hierarchy to phone space +// The WindowController expects coordinates in phone space (540x960) and handles DPI scaling internally void ScaleToPhysical(TestContext& ctx, int& x, int& y) { - // The hierarchy reports coordinates in RmlUi's logical space - // which may be DPI-scaled. We need to convert to physical window coordinates. + // The hierarchy reports coordinates in RmlUi's context space (540x960) + // which matches the phone resolution. No scaling needed since + // WindowController::SendClick handles DPI scaling internally. + // Just validate the hierarchy dimensions match expected phone size. int hierarchyWidth = ctx.hierarchy.GetWidth(); int hierarchyHeight = ctx.hierarchy.GetHeight(); - int windowWidth = ctx.window.GetInfo().clientWidth; - int windowHeight = ctx.window.GetInfo().clientHeight; - if (hierarchyWidth > 0 && hierarchyHeight > 0) { - x = static_cast(x * static_cast(windowWidth) / hierarchyWidth); - y = static_cast(y * static_cast(windowHeight) / hierarchyHeight); + if (hierarchyWidth != 540 || hierarchyHeight != 960) { + std::cerr << " Warning: Unexpected hierarchy size " << hierarchyWidth << "x" << hierarchyHeight << std::endl; } + // Coordinates stay in phone space (540x960) - no scaling needed } // Helper: Click on an element found by ID in the hierarchy diff --git a/designer-test/src/window_controller.cpp b/designer-test/src/window_controller.cpp index e1977b5..d2c7b34 100644 --- a/designer-test/src/window_controller.cpp +++ b/designer-test/src/window_controller.cpp @@ -107,33 +107,47 @@ LPARAM WindowController::PhoneToClientLParam(int phoneX, int phoneY) { bool WindowController::SendMouseDown(int phoneX, int phoneY) { if (!m_hwnd) return false; - // Convert phone coordinates to client coordinates + // Convert phone coordinates to client coordinates (using window scale) int clientX = static_cast(phoneX * m_info.scaleX); int clientY = static_cast(phoneY * m_info.scaleY); + // Calculate screen coordinates + int screenX = m_info.clientX + clientX; + int screenY = m_info.clientY + clientY; + // Get DPI info for debugging UINT dpi = GetDpiForWindow(m_hwnd); - // Calculate screen coordinates from client position - // On DPI-aware systems, Windows APIs return consistent coordinate spaces - int screenX = m_info.clientX + clientX; - int screenY = m_info.clientY + clientY; - // Ensure window is foreground before clicking SetForegroundWindow(m_hwnd); - Sleep(10); // Small delay + Sleep(10); - // Use SendInput for GLFW compatibility - SetCursorPos(screenX, screenY); - Sleep(10); // Small delay for cursor move + // Get screen dimensions for absolute coordinate conversion + int screenWidth = GetSystemMetrics(SM_CXSCREEN); + int screenHeight = GetSystemMetrics(SM_CYSCREEN); - INPUT input = {}; - input.type = INPUT_MOUSE; - input.mi.dwFlags = MOUSEEVENTF_LEFTDOWN; - SendInput(1, &input, sizeof(INPUT)); + // Convert to normalized absolute coordinates (0-65535) + LONG absX = static_cast((screenX * 65535) / screenWidth); + LONG absY = static_cast((screenY * 65535) / screenHeight); - std::cout << "MouseDown at phone(" << phoneX << "," << phoneY << ") -> screen(" - << screenX << "," << screenY << ") dpi=" << dpi << std::endl; + // Send mouse move then button down + INPUT moveInput = {}; + moveInput.type = INPUT_MOUSE; + moveInput.mi.dx = absX; + moveInput.mi.dy = absY; + moveInput.mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE; + SendInput(1, &moveInput, sizeof(INPUT)); + + Sleep(20); + + INPUT clickInput = {}; + clickInput.type = INPUT_MOUSE; + clickInput.mi.dwFlags = MOUSEEVENTF_LEFTDOWN; + SendInput(1, &clickInput, sizeof(INPUT)); + + std::cout << "MouseDown at phone(" << phoneX << "," << phoneY << ") -> client(" + << clientX << "," << clientY << ") screen(" << screenX << "," << screenY + << ") dpi=" << dpi << std::endl; return true; } @@ -226,38 +240,20 @@ bool WindowController::Close() { bool WindowController::ResizeToPhone() { if (!m_hwnd) return false; - // Get current window rect to preserve position + // Skip resizing - the designer creates the window at the correct size + // and resizing causes DPI scaling mismatches between window and framebuffer. + // Just move window to top-left for consistent test positioning. + RECT currentRect; ::GetWindowRect(m_hwnd, ¤tRect); + int currentWidth = currentRect.right - currentRect.left; + int currentHeight = currentRect.bottom - currentRect.top; - // Calculate window size needed for phone-sized client area - RECT desiredRect = {0, 0, PHONE_WIDTH, PHONE_HEIGHT}; - DWORD style = ::GetWindowLong(m_hwnd, GWL_STYLE); - DWORD exStyle = ::GetWindowLong(m_hwnd, GWL_EXSTYLE); - ::AdjustWindowRectEx(&desiredRect, style, FALSE, exStyle); + std::cout << "ResizeToPhone: keeping current size " << currentWidth << "x" << currentHeight + << ", moving to (0,0)" << std::endl; - int newWidth = desiredRect.right - desiredRect.left; - int newHeight = desiredRect.bottom - desiredRect.top; - - // Check screen dimensions - int screenWidth = ::GetSystemMetrics(SM_CXSCREEN); - int screenHeight = ::GetSystemMetrics(SM_CYSCREEN); - - std::cout << "ResizeToPhone: screen=" << screenWidth << "x" << screenHeight - << ", needed=" << newWidth << "x" << newHeight << std::endl; - - // If screen is too small, we can't resize to full phone size - if (newHeight > screenHeight) { - std::cout << " Warning: Screen too small for full phone resolution, keeping current size" << std::endl; - return true; // Not an error, just can't resize - } - - // Move window to top-left to ensure it fits on screen - int newX = 0; - int newY = 0; - - std::cout << " Moving to (" << newX << "," << newY << ") size " << newWidth << "x" << newHeight << std::endl; - ::MoveWindow(m_hwnd, newX, newY, newWidth, newHeight, TRUE); + // Move window to top-left for consistent positioning + ::MoveWindow(m_hwnd, 0, 0, currentWidth, currentHeight, TRUE); // Re-acquire window info std::this_thread::sleep_for(std::chrono::milliseconds(100)); @@ -274,26 +270,36 @@ bool WindowController::SendClickFromBottom(int clientX, int offsetFromBottom) { int screenX = m_info.clientX + clientX; int screenY = m_info.clientY + clientY; - // Save current cursor position - POINT oldPos; - GetCursorPos(&oldPos); - // Ensure window is active SetForegroundWindow(m_hwnd); std::this_thread::sleep_for(std::chrono::milliseconds(50)); - // Move cursor and click - SetCursorPos(screenX, screenY); + // Get screen dimensions for absolute coordinate conversion + int screenWidth = GetSystemMetrics(SM_CXSCREEN); + int screenHeight = GetSystemMetrics(SM_CYSCREEN); - INPUT input = {}; - input.type = INPUT_MOUSE; - input.mi.dwFlags = MOUSEEVENTF_LEFTDOWN; - SendInput(1, &input, sizeof(INPUT)); + // Convert to normalized absolute coordinates (0-65535) + LONG absX = static_cast((screenX * 65535) / screenWidth); + LONG absY = static_cast((screenY * 65535) / screenHeight); - std::this_thread::sleep_for(std::chrono::milliseconds(50)); + // Use SendInput with absolute move + click + INPUT inputs[3] = {}; - input.mi.dwFlags = MOUSEEVENTF_LEFTUP; - SendInput(1, &input, sizeof(INPUT)); + // Move cursor + inputs[0].type = INPUT_MOUSE; + inputs[0].mi.dx = absX; + inputs[0].mi.dy = absY; + inputs[0].mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE; + + // Mouse down + inputs[1].type = INPUT_MOUSE; + inputs[1].mi.dwFlags = MOUSEEVENTF_LEFTDOWN; + + // Mouse up + inputs[2].type = INPUT_MOUSE; + inputs[2].mi.dwFlags = MOUSEEVENTF_LEFTUP; + + SendInput(3, inputs, sizeof(INPUT)); std::cout << "ClickFromBottom at client(" << clientX << "," << clientY << ") -> screen(" << screenX << "," << screenY << ")" << std::endl; diff --git a/designer/main.cpp b/designer/main.cpp index 103615c..ebb0e77 100644 --- a/designer/main.cpp +++ b/designer/main.cpp @@ -156,10 +156,30 @@ static void MouseButtonCallback(GLFWwindow* window, int button, int action, int double xpos, ypos; glfwGetCursorPos(window, &xpos, &ypos); - // GLFW cursor callbacks report coordinates in screen/window coordinates (logical pixels) - // which match the RmlUi context dimensions, so no scaling needed - int mouseX = static_cast(xpos); - int mouseY = static_cast(ypos); + // glfwGetCursorPos returns position in screen coordinates (same as window size) + // which may differ from framebuffer size on high-DPI displays. + // We need to scale to match the RmlUi context (which matches framebuffer). + int winWidth, winHeight; + glfwGetWindowSize(window, &winWidth, &winHeight); + + int fbWidth, fbHeight; + glfwGetFramebufferSize(window, &fbWidth, &fbHeight); + + // Scale cursor position: screen coords -> framebuffer coords -> RmlUi context + // On high DPI: winWidth=432, fbWidth=540, g_width=540 + // Cursor in screen space needs to scale to framebuffer/context space + int mouseX = static_cast(xpos * fbWidth / winWidth); + int mouseY = static_cast(ypos * fbHeight / winHeight); + + // Debug logging for click events + std::cout << "MouseButton: " << (action == GLFW_PRESS ? "DOWN" : "UP") + << " at raw(" << xpos << "," << ypos << ") -> scaled(" << mouseX << "," << mouseY << ")" + << " win=" << winWidth << "x" << winHeight << " fb=" << fbWidth << "x" << fbHeight << std::endl; + if (g_log_file.is_open()) { + g_log_file << "[DEBUG] MouseButton: " << (action == GLFW_PRESS ? "DOWN" : "UP") + << " at raw(" << xpos << "," << ypos << ") -> scaled(" << mouseX << "," << mouseY << ")" << std::endl; + g_log_file.flush(); + } int key_modifier = 0; if (mods & GLFW_MOD_CONTROL) key_modifier |= Rml::Input::KM_CTRL; @@ -187,10 +207,15 @@ static void MouseButtonCallback(GLFWwindow* window, int button, int action, int static void CursorPosCallback(GLFWwindow* window, double xpos, double ypos) { if (!g_context) return; - // GLFW cursor callbacks report coordinates in screen/window coordinates (logical pixels) - // which match the RmlUi context dimensions, so no scaling needed - int mouseX = static_cast(xpos); - int mouseY = static_cast(ypos); + // Scale from screen coordinates to framebuffer/RmlUi context coordinates + int winWidth, winHeight; + glfwGetWindowSize(window, &winWidth, &winHeight); + + int fbWidth, fbHeight; + glfwGetFramebufferSize(window, &fbWidth, &fbHeight); + + int mouseX = static_cast(xpos * fbWidth / winWidth); + int mouseY = static_cast(ypos * fbHeight / winHeight); g_context->ProcessMouseMove(mouseX, mouseY, 0); } diff --git a/src/main/cpp/kernel.cpp b/src/main/cpp/kernel.cpp index 62ac050..cfb5d87 100644 --- a/src/main/cpp/kernel.cpp +++ b/src/main/cpp/kernel.cpp @@ -257,13 +257,65 @@ static int LuaLoadScreen(lua_State* L) return 1; } +// Lua function to go back to home screen +static int LuaGoHome(lua_State* L) +{ + if (!g_context) + { + lua_pushboolean(L, false); + return 1; + } + + Logger::Log("goHome called - returning to home screen"); + + // Stop any running third-party app + if (g_sandbox_manager) { + auto running_apps = g_sandbox_manager->GetRunningApps(); + for (const auto& app_id : running_apps) { + // Don't stop system apps + if (app_id.find("com.mosis.") == 0 && app_id != "com.mosis.home") { + Logger::Log(std::format("Stopping app: {}", app_id)); + g_sandbox_manager->StopApp(app_id); + } + } + } + + // Load home screen + const char* home_path = "apps/home/home.rml"; + + // Unload current document + if (g_document) + { + g_context->UnloadDocument(g_document); + g_document = nullptr; + } + + // Load home document + g_document = g_context->LoadDocument(home_path); + if (g_document) + { + g_document->Show(); + Logger::Log("Returned to home screen"); + lua_pushboolean(L, true); + } + else + { + Logger::Log("Failed to load home screen"); + lua_pushboolean(L, false); + } + + return 1; +} + // Register Lua functions for navigation static void RegisterLuaFunctions(const std::string& current_app_id, bool is_system_app) { lua_State* L = Rml::Lua::Interpreter::GetLuaState(); lua_pushcfunction(L, LuaLoadScreen); lua_setglobal(L, "loadScreen"); - Logger::Log("Registered Lua loadScreen function"); + lua_pushcfunction(L, LuaGoHome); + lua_setglobal(L, "goHome"); + Logger::Log("Registered Lua loadScreen and goHome functions"); // Register app management APIs if (g_app_manager && g_update_service) { diff --git a/test-apps/com.mosis.sandbox-test/main.rml b/test-apps/com.mosis.sandbox-test/main.rml index e14c55d..96b4878 100644 --- a/test-apps/com.mosis.sandbox-test/main.rml +++ b/test-apps/com.mosis.sandbox-test/main.rml @@ -6,8 +6,8 @@
-
- ← +
+ <
Sandbox Test