Check OpenGL readback byte counts

docs/modernization/build-inventory.md

@@ -186,7 +186,7 @@ Known local toolchain state:
   RGBA pixel-format mapping used by `RTT` texture allocation. It also validates
   image channel-count to OpenGL texture format mapping, including
   invalid channel counts rejected by `Texture2D::create(Image)`, RGBA8/RGBA32F
-  readback formats, byte-count math, and PBO pixel-buffer target/usage/access
+  readback formats, checked byte-count math, and PBO pixel-buffer target/usage/access
   mapping used by `RTT` and `PBO` readbacks, and framebuffer status naming
   used by `RTT` and `Texture2D` diagnostics. It also owns the 2D texture target,
   framebuffer setup, readback format, mipmap target, and update component-type

docs/modernization/roadmap.md

@@ -444,7 +444,7 @@ format mapping for `Texture2D` image uploads and framebuffer status naming for
 mipmap generation, framebuffer readback setup, and update component-type tokens
 now delegate to `pp_renderer_gl`. `TextureCube` cube-map binding, allocation
 face targets, RGBA allocation format, and unsigned-byte component type also
-delegate to `pp_renderer_gl`. RGBA8/RGBA32F readback formats, byte-count math, and PBO
+delegate to `pp_renderer_gl`. RGBA8/RGBA32F readback formats, checked byte-count math, and PBO
 pixel-buffer target/usage/access tokens used by `RTT` and `PBO` readbacks now
 live in `pp_renderer_gl`. The framebuffer blit color mask and linear/nearest
 filter tokens used by `RTT::resize` and `RTT::copy`, plus the default

src/renderer_gl/opengl_capabilities.cpp

@@ -1,6 +1,7 @@
 #include "renderer_gl/opengl_capabilities.h"
 
 #include <array>
+#include <limits>
 
 namespace pp::renderer::gl {
 
@@ -284,7 +285,22 @@ std::uint64_t readback_byte_count(
     std::uint32_t width,
     std::uint32_t height) noexcept
 {
-    return static_cast<std::uint64_t>(width) * height * format.bytes_per_pixel;
+    if (format.bytes_per_pixel == 0U) {
+        return 0U;
+    }
+
+    const auto wide_width = static_cast<std::uint64_t>(width);
+    const auto wide_height = static_cast<std::uint64_t>(height);
+    if (wide_width != 0U && wide_height > std::numeric_limits<std::uint64_t>::max() / wide_width) {
+        return 0U;
+    }
+
+    const auto pixels = wide_width * wide_height;
+    if (pixels != 0U && format.bytes_per_pixel > std::numeric_limits<std::uint64_t>::max() / pixels) {
+        return 0U;
+    }
+
+    return pixels * format.bytes_per_pixel;
 }
 
 std::uint32_t pixel_pack_buffer_target() noexcept

tests/renderer_gl/capabilities_tests.cpp

@@ -5,6 +5,7 @@
 #include <array>
 #include <cstdint>
 #include <cstring>
+#include <limits>
 #include <string_view>
 
 namespace {
@@ -152,11 +153,30 @@ void maps_readback_formats(pp::tests::Harness& h)
     PP_EXPECT(h, rgba8.component_type == 0x1401U);
     PP_EXPECT(h, rgba8.bytes_per_pixel == 4U);
     PP_EXPECT(h, pp::renderer::gl::readback_byte_count(rgba8, 3U, 5U) == 60U);
+    PP_EXPECT(h, pp::renderer::gl::readback_byte_count(rgba8, 0U, 5U) == 0U);
 
     PP_EXPECT(h, rgba32f.pixel_format == 0x1908U);
     PP_EXPECT(h, rgba32f.component_type == 0x1406U);
     PP_EXPECT(h, rgba32f.bytes_per_pixel == 16U);
     PP_EXPECT(h, pp::renderer::gl::readback_byte_count(rgba32f, 3U, 5U) == 240U);
+
+    const auto invalid_format = pp::renderer::gl::OpenGlReadbackFormat {
+        .pixel_format = rgba8.pixel_format,
+        .component_type = rgba8.component_type,
+        .bytes_per_pixel = 0U,
+    };
+    const auto overflowing_format = pp::renderer::gl::OpenGlReadbackFormat {
+        .pixel_format = rgba8.pixel_format,
+        .component_type = rgba8.component_type,
+        .bytes_per_pixel = std::numeric_limits<std::uint32_t>::max(),
+    };
+
+    PP_EXPECT(h, pp::renderer::gl::readback_byte_count(invalid_format, 1U, 1U) == 0U);
+    PP_EXPECT(h, pp::renderer::gl::readback_byte_count(
+                     overflowing_format,
+                     std::numeric_limits<std::uint32_t>::max(),
+                     std::numeric_limits<std::uint32_t>::max())
+                     == 0U);
 }
 
 void maps_pixel_buffer_parameters(pp::tests::Harness& h)