Route canvas blend gate through paint renderer

src/canvas.cpp

@@ -4,6 +4,7 @@
 #include "app.h"
 #include "texture.h"
 #include "node_progress_bar.h"
+#include "paint_renderer/compositor.h"
 #include "renderer_gl/opengl_capabilities.h"
 #include <thread>
 #include <algorithm>
@@ -43,6 +44,104 @@ GLenum rgba_pixel_format()
     return static_cast<GLenum>(pp::renderer::gl::rgba_pixel_format());
 }
 
+bool to_paint_blend_mode(int value, pp::paint::BlendMode& out) noexcept
+{
+    switch (value) {
+    case 0: out = pp::paint::BlendMode::normal; return true;
+    case 1: out = pp::paint::BlendMode::multiply; return true;
+    case 2: out = pp::paint::BlendMode::screen; return true;
+    case 3: out = pp::paint::BlendMode::color_dodge; return true;
+    case 4: out = pp::paint::BlendMode::overlay; return true;
+    default: return false;
+    }
+}
+
+bool to_stroke_blend_mode(int value, pp::paint::StrokeBlendMode& out) noexcept
+{
+    switch (value) {
+    case 0: out = pp::paint::StrokeBlendMode::normal; return true;
+    case 1: out = pp::paint::StrokeBlendMode::multiply; return true;
+    case 2: out = pp::paint::StrokeBlendMode::subtract; return true;
+    case 3: out = pp::paint::StrokeBlendMode::darken; return true;
+    case 4: out = pp::paint::StrokeBlendMode::overlay; return true;
+    case 5: out = pp::paint::StrokeBlendMode::color_dodge; return true;
+    case 6: out = pp::paint::StrokeBlendMode::color_burn; return true;
+    case 7: out = pp::paint::StrokeBlendMode::linear_burn; return true;
+    case 8: out = pp::paint::StrokeBlendMode::hard_mix; return true;
+    case 9: out = pp::paint::StrokeBlendMode::linear_height; return true;
+    case 10: out = pp::paint::StrokeBlendMode::height; return true;
+    default: return false;
+    }
+}
+
+pp::renderer::RenderDeviceFeatures canvas_stroke_composite_features() noexcept
+{
+    return pp::renderer::RenderDeviceFeatures {
+        .framebuffer_fetch = ShaderManager::ext_framebuffer_fetch,
+        .texture_copy = !ShaderManager::ext_framebuffer_fetch,
+    };
+}
+
+bool stroke_composite_plan_needs_shader_blend(
+    int width,
+    int height,
+    pp::paint::BlendMode layer_blend_mode,
+    pp::paint::StrokeBlendMode stroke_blend_mode) noexcept
+{
+    const auto plan = pp::paint_renderer::plan_stroke_composite(
+        canvas_stroke_composite_features(),
+        pp::paint_renderer::StrokeCompositeRequest {
+            .extent = pp::renderer::Extent2D {
+                .width = static_cast<std::uint32_t>(std::max(width, 0)),
+                .height = static_cast<std::uint32_t>(std::max(height, 0)),
+            },
+            .layer_blend_mode = layer_blend_mode,
+            .stroke_blend_mode = stroke_blend_mode,
+        });
+    return plan ? plan.value().complex_blend : true;
+}
+
+bool draw_merge_needs_shader_blend(
+    int width,
+    int height,
+    const std::vector<std::shared_ptr<Layer>>& layers,
+    const Brush* brush) noexcept
+{
+    for (const auto& layer : layers) {
+        if (!layer) {
+            continue;
+        }
+        pp::paint::BlendMode layer_blend = pp::paint::BlendMode::normal;
+        if (!to_paint_blend_mode(layer->m_blend_mode, layer_blend)) {
+            if (layer->m_blend_mode != 0) {
+                return true;
+            }
+            continue;
+        }
+        if (stroke_composite_plan_needs_shader_blend(
+            width,
+            height,
+            layer_blend,
+            pp::paint::StrokeBlendMode::normal)) {
+            return true;
+        }
+    }
+
+    if (brush) {
+        pp::paint::StrokeBlendMode stroke_blend = pp::paint::StrokeBlendMode::normal;
+        if (!to_stroke_blend_mode(brush->m_blend_mode, stroke_blend)) {
+            return brush->m_blend_mode != 0;
+        }
+        return stroke_composite_plan_needs_shader_blend(
+            width,
+            height,
+            pp::paint::BlendMode::normal,
+            stroke_blend);
+    }
+
+    return false;
+}
+
 GLenum unsigned_byte_component_type()
 {
     return static_cast<GLenum>(pp::renderer::gl::unsigned_byte_component_type());
@@ -1086,14 +1185,11 @@ void Canvas::draw_merge(bool draw_checkerboard, std::array<bool, 6> faces /*= SI
     auto ortho = glm::ortho<float>(-0.5f, 0.5f, -0.5f, 0.5f, -1.f, 1.f);
     const auto& b = m_current_stroke->m_brush;
 
-    // check if any layer use blend, otherwise draw directly on main framebuffer
-    bool use_blend = false;
-    for (auto& l : m_layers)
-    {
-        use_blend |= l->m_blend_mode != 0;
-    }
-    if (Canvas::I->m_current_stroke)
-        use_blend |= Canvas::I->m_current_stroke->m_brush->m_blend_mode != 0;
+    const bool use_blend = draw_merge_needs_shader_blend(
+        m_width,
+        m_height,
+        m_layers,
+        m_current_stroke ? m_current_stroke->m_brush.get() : nullptr);
 
     // if not using shader blend, use gl rasterizer blend
     glDisable(depth_test_state());