omar/MosisService
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix desktop designer click handling and add goHome API

Browse Source 
Designer click handling:
- Fix DPI scaling in MouseButtonCallback and CursorPosCallback
- Scale coordinates from window space to framebuffer/RmlUi context
- Remove window resizing in ResizeToPhone (caused DPI mismatches)

Test framework:
- Fix SendMouseDown to use MOUSEEVENTF_MOVE before button down
- Remove double-scaling in ScaleToPhysical (WindowController handles it)
- All 5 UI navigation tests now pass

Kernel API:
- Add goHome() Lua function to return to home screen
- Stops any running third-party apps before navigating

Test app:
- Update sandbox-test to use goHome() instead of goBack()

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
omigamedev
2026-01-19 14:52:15 +01:00
parent d6b7504408
commit 984e8715d7
5 changed files with 160 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
designer-test/src/main.cpp
View File

@@ -10,19 +10,20 @@
using namespace mosis::test;
// Helper: Scale coordinates from hierarchy (logical) to window (physical) space
// Helper: Scale coordinates from hierarchy to phone space
// The WindowController expects coordinates in phone space (540x960) and handles DPI scaling internally
void ScaleToPhysical(TestContext& ctx, int& x, int& y) {
// The hierarchy reports coordinates in RmlUi's logical space
// which may be DPI-scaled. We need to convert to physical window coordinates.
// The hierarchy reports coordinates in RmlUi's context space (540x960)
// which matches the phone resolution. No scaling needed since
// WindowController::SendClick handles DPI scaling internally.
// Just validate the hierarchy dimensions match expected phone size.
int hierarchyWidth = ctx.hierarchy.GetWidth();
int hierarchyHeight = ctx.hierarchy.GetHeight();
int windowWidth = ctx.window.GetInfo().clientWidth;
int windowHeight = ctx.window.GetInfo().clientHeight;
if (hierarchyWidth > 0 && hierarchyHeight > 0) {
x = static_cast<int>(x * static_cast<float>(windowWidth) / hierarchyWidth);
y = static_cast<int>(y * static_cast<float>(windowHeight) / hierarchyHeight);
if (hierarchyWidth != 540 || hierarchyHeight != 960) {
std::cerr << " Warning: Unexpected hierarchy size " << hierarchyWidth << "x" << hierarchyHeight << std::endl;
}
// Coordinates stay in phone space (540x960) - no scaling needed
}
// Helper: Click on an element found by ID in the hierarchy

120
designer-test/src/window_controller.cpp
View File

@@ -107,33 +107,47 @@ LPARAM WindowController::PhoneToClientLParam(int phoneX, int phoneY) {
bool WindowController::SendMouseDown(int phoneX, int phoneY) {
if (!m_hwnd) return false;
// Convert phone coordinates to client coordinates
// Convert phone coordinates to client coordinates (using window scale)
int clientX = static_cast<int>(phoneX * m_info.scaleX);
int clientY = static_cast<int>(phoneY * m_info.scaleY);
// Calculate screen coordinates
int screenX = m_info.clientX + clientX;
int screenY = m_info.clientY + clientY;
// Get DPI info for debugging
UINT dpi = GetDpiForWindow(m_hwnd);
// Calculate screen coordinates from client position
// On DPI-aware systems, Windows APIs return consistent coordinate spaces
int screenX = m_info.clientX + clientX;
int screenY = m_info.clientY + clientY;
// Ensure window is foreground before clicking
SetForegroundWindow(m_hwnd);
Sleep(10); // Small delay
Sleep(10);
// Use SendInput for GLFW compatibility
SetCursorPos(screenX, screenY);
Sleep(10); // Small delay for cursor move
// Get screen dimensions for absolute coordinate conversion
int screenWidth = GetSystemMetrics(SM_CXSCREEN);
int screenHeight = GetSystemMetrics(SM_CYSCREEN);
INPUT input = {};
input.type = INPUT_MOUSE;
input.mi.dwFlags = MOUSEEVENTF_LEFTDOWN;
SendInput(1, &input, sizeof(INPUT));
// Convert to normalized absolute coordinates (0-65535)
LONG absX = static_cast<LONG>((screenX * 65535) / screenWidth);
LONG absY = static_cast<LONG>((screenY * 65535) / screenHeight);
std::cout << "MouseDown at phone(" << phoneX << "," << phoneY << ") -> screen("
<< screenX << "," << screenY << ") dpi=" << dpi << std::endl;
// Send mouse move then button down
INPUT moveInput = {};
moveInput.type = INPUT_MOUSE;
moveInput.mi.dx = absX;
moveInput.mi.dy = absY;
moveInput.mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE;
SendInput(1, &moveInput, sizeof(INPUT));
Sleep(20);
INPUT clickInput = {};
clickInput.type = INPUT_MOUSE;
clickInput.mi.dwFlags = MOUSEEVENTF_LEFTDOWN;
SendInput(1, &clickInput, sizeof(INPUT));
std::cout << "MouseDown at phone(" << phoneX << "," << phoneY << ") -> client("
<< clientX << "," << clientY << ") screen(" << screenX << "," << screenY
<< ") dpi=" << dpi << std::endl;
return true;
}
@@ -226,38 +240,20 @@ bool WindowController::Close() {
bool WindowController::ResizeToPhone() {
if (!m_hwnd) return false;
// Get current window rect to preserve position
// Skip resizing - the designer creates the window at the correct size
// and resizing causes DPI scaling mismatches between window and framebuffer.
// Just move window to top-left for consistent test positioning.
RECT currentRect;
::GetWindowRect(m_hwnd, &currentRect);
int currentWidth = currentRect.right - currentRect.left;
int currentHeight = currentRect.bottom - currentRect.top;
// Calculate window size needed for phone-sized client area
RECT desiredRect = {0, 0, PHONE_WIDTH, PHONE_HEIGHT};
DWORD style = ::GetWindowLong(m_hwnd, GWL_STYLE);
DWORD exStyle = ::GetWindowLong(m_hwnd, GWL_EXSTYLE);
::AdjustWindowRectEx(&desiredRect, style, FALSE, exStyle);
std::cout << "ResizeToPhone: keeping current size " << currentWidth << "x" << currentHeight
<< ", moving to (0,0)" << std::endl;
int newWidth = desiredRect.right - desiredRect.left;
int newHeight = desiredRect.bottom - desiredRect.top;
// Check screen dimensions
int screenWidth = ::GetSystemMetrics(SM_CXSCREEN);
int screenHeight = ::GetSystemMetrics(SM_CYSCREEN);
std::cout << "ResizeToPhone: screen=" << screenWidth << "x" << screenHeight
<< ", needed=" << newWidth << "x" << newHeight << std::endl;
// If screen is too small, we can't resize to full phone size
if (newHeight > screenHeight) {
std::cout << " Warning: Screen too small for full phone resolution, keeping current size" << std::endl;
return true; // Not an error, just can't resize
}
// Move window to top-left to ensure it fits on screen
int newX = 0;
int newY = 0;
std::cout << " Moving to (" << newX << "," << newY << ") size " << newWidth << "x" << newHeight << std::endl;
::MoveWindow(m_hwnd, newX, newY, newWidth, newHeight, TRUE);
// Move window to top-left for consistent positioning
::MoveWindow(m_hwnd, 0, 0, currentWidth, currentHeight, TRUE);
// Re-acquire window info
std::this_thread::sleep_for(std::chrono::milliseconds(100));
@@ -274,26 +270,36 @@ bool WindowController::SendClickFromBottom(int clientX, int offsetFromBottom) {
int screenX = m_info.clientX + clientX;
int screenY = m_info.clientY + clientY;
// Save current cursor position
POINT oldPos;
GetCursorPos(&oldPos);
// Ensure window is active
SetForegroundWindow(m_hwnd);
std::this_thread::sleep_for(std::chrono::milliseconds(50));
// Move cursor and click
SetCursorPos(screenX, screenY);
// Get screen dimensions for absolute coordinate conversion
int screenWidth = GetSystemMetrics(SM_CXSCREEN);
int screenHeight = GetSystemMetrics(SM_CYSCREEN);
INPUT input = {};
input.type = INPUT_MOUSE;
input.mi.dwFlags = MOUSEEVENTF_LEFTDOWN;
SendInput(1, &input, sizeof(INPUT));
// Convert to normalized absolute coordinates (0-65535)
LONG absX = static_cast<LONG>((screenX * 65535) / screenWidth);
LONG absY = static_cast<LONG>((screenY * 65535) / screenHeight);
std::this_thread::sleep_for(std::chrono::milliseconds(50));
// Use SendInput with absolute move + click
INPUT inputs[3] = {};
input.mi.dwFlags = MOUSEEVENTF_LEFTUP;
SendInput(1, &input, sizeof(INPUT));
// Move cursor
inputs[0].type = INPUT_MOUSE;
inputs[0].mi.dx = absX;
inputs[0].mi.dy = absY;
inputs[0].mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE;
// Mouse down
inputs[1].type = INPUT_MOUSE;
inputs[1].mi.dwFlags = MOUSEEVENTF_LEFTDOWN;
// Mouse up
inputs[2].type = INPUT_MOUSE;
inputs[2].mi.dwFlags = MOUSEEVENTF_LEFTUP;
SendInput(3, inputs, sizeof(INPUT));
std::cout << "ClickFromBottom at client(" << clientX << "," << clientY
<< ") -> screen(" << screenX << "," << screenY << ")" << std::endl;

41
designer/main.cpp
View File

@@ -156,10 +156,30 @@ static void MouseButtonCallback(GLFWwindow* window, int button, int action, int
double xpos, ypos;
glfwGetCursorPos(window, &xpos, &ypos);
// GLFW cursor callbacks report coordinates in screen/window coordinates (logical pixels)
// which match the RmlUi context dimensions, so no scaling needed
int mouseX = static_cast<int>(xpos);
int mouseY = static_cast<int>(ypos);
// glfwGetCursorPos returns position in screen coordinates (same as window size)
// which may differ from framebuffer size on high-DPI displays.
// We need to scale to match the RmlUi context (which matches framebuffer).
int winWidth, winHeight;
glfwGetWindowSize(window, &winWidth, &winHeight);
int fbWidth, fbHeight;
glfwGetFramebufferSize(window, &fbWidth, &fbHeight);
// Scale cursor position: screen coords -> framebuffer coords -> RmlUi context
// On high DPI: winWidth=432, fbWidth=540, g_width=540
// Cursor in screen space needs to scale to framebuffer/context space
int mouseX = static_cast<int>(xpos * fbWidth / winWidth);
int mouseY = static_cast<int>(ypos * fbHeight / winHeight);
// Debug logging for click events
std::cout << "MouseButton: " << (action == GLFW_PRESS ? "DOWN" : "UP")
<< " at raw(" << xpos << "," << ypos << ") -> scaled(" << mouseX << "," << mouseY << ")"
<< " win=" << winWidth << "x" << winHeight << " fb=" << fbWidth << "x" << fbHeight << std::endl;
if (g_log_file.is_open()) {
g_log_file << "[DEBUG] MouseButton: " << (action == GLFW_PRESS ? "DOWN" : "UP")
<< " at raw(" << xpos << "," << ypos << ") -> scaled(" << mouseX << "," << mouseY << ")" << std::endl;
g_log_file.flush();
}
int key_modifier = 0;
if (mods & GLFW_MOD_CONTROL) key_modifier |= Rml::Input::KM_CTRL;
@@ -187,10 +207,15 @@ static void MouseButtonCallback(GLFWwindow* window, int button, int action, int
static void CursorPosCallback(GLFWwindow* window, double xpos, double ypos) {
if (!g_context) return;
// GLFW cursor callbacks report coordinates in screen/window coordinates (logical pixels)
// which match the RmlUi context dimensions, so no scaling needed
int mouseX = static_cast<int>(xpos);
int mouseY = static_cast<int>(ypos);
// Scale from screen coordinates to framebuffer/RmlUi context coordinates
int winWidth, winHeight;
glfwGetWindowSize(window, &winWidth, &winHeight);
int fbWidth, fbHeight;
glfwGetFramebufferSize(window, &fbWidth, &fbHeight);
int mouseX = static_cast<int>(xpos * fbWidth / winWidth);
int mouseY = static_cast<int>(ypos * fbHeight / winHeight);
g_context->ProcessMouseMove(mouseX, mouseY, 0);
}

54
src/main/cpp/kernel.cpp
View File

@@ -257,13 +257,65 @@ static int LuaLoadScreen(lua_State* L)
return 1;
}
// Lua function to go back to home screen
static int LuaGoHome(lua_State* L)
{
if (!g_context)
{
lua_pushboolean(L, false);
return 1;
}
Logger::Log("goHome called - returning to home screen");
// Stop any running third-party app
if (g_sandbox_manager) {
auto running_apps = g_sandbox_manager->GetRunningApps();
for (const auto& app_id : running_apps) {
// Don't stop system apps
if (app_id.find("com.mosis.") == 0 && app_id != "com.mosis.home") {
Logger::Log(std::format("Stopping app: {}", app_id));
g_sandbox_manager->StopApp(app_id);
}
}
}
// Load home screen
const char* home_path = "apps/home/home.rml";
// Unload current document
if (g_document)
{
g_context->UnloadDocument(g_document);
g_document = nullptr;
}
// Load home document
g_document = g_context->LoadDocument(home_path);
if (g_document)
{
g_document->Show();
Logger::Log("Returned to home screen");
lua_pushboolean(L, true);
}
else
{
Logger::Log("Failed to load home screen");
lua_pushboolean(L, false);
}
return 1;
}
// Register Lua functions for navigation
static void RegisterLuaFunctions(const std::string& current_app_id, bool is_system_app)
{
lua_State* L = Rml::Lua::Interpreter::GetLuaState();
lua_pushcfunction(L, LuaLoadScreen);
lua_setglobal(L, "loadScreen");
Logger::Log("Registered Lua loadScreen function");
lua_pushcfunction(L, LuaGoHome);
lua_setglobal(L, "goHome");
Logger::Log("Registered Lua loadScreen and goHome functions");
// Register app management APIs
if (g_app_manager && g_update_service) {

4
test-apps/com.mosis.sandbox-test/main.rml
View File

@@ -6,8 +6,8 @@
</head>
<body>
<div class="app-bar">
<div class="app-bar-nav btn-icon" onclick="goBack()">
<span class="icon">←</span>
<div class="app-bar-nav btn-icon" onclick="goHome()">
<span class="icon">&lt;</span>
</div>
<div class="app-bar-title">Sandbox Test</div>
</div>