implement flood fill algorithm

src/canvas.cpp

@@ -1340,6 +1340,149 @@ void Canvas::layer_merge(int source_idx, int dest_idx) // m_layer index
     ActionManager::add(action);
 */
 }
+
+void Canvas::flood_fill(int layer, int plane, std::vector<glm::ivec2> pos, std::map<int, std::unique_ptr<bool[]>>& plane_mask,
+    float threshold, glm::vec4 dest_color, std::unique_ptr<glm::vec4>& source_color)
+{
+    struct adj_t
+    {
+        int plane;
+        bool flipx;
+        bool flipy;
+        bool flipcoord;
+        adj_t(int plane, bool flipx, bool flipy, int flipcoord) :
+            plane(plane), flipx(flipx), flipy(flipy), flipcoord(flipcoord) { }
+        glm::ivec2 compute(glm::ivec2 p, glm::ivec2 sz)
+        {
+            glm::ivec2 ret;
+            ret[flipcoord] = flipx ? sz.x - p.x : p.x;
+            ret[1 - flipcoord] = flipy ? sz.y - p.y : p.y;
+            return ret;
+        }
+    };
+
+    LOG("flood_fill plane %d", plane);
+
+    auto& rtt = m_layers[layer]->m_rtt[plane];
+    auto sz = glm::ivec2(rtt.getWidth(), rtt.getHeight());
+    auto rgb = reinterpret_cast<glm::u8vec4*>(m_layers[layer]->m_rtt[plane].readTextureData());
+    
+    if (plane_mask.find(plane) == plane_mask.end())
+        plane_mask.insert({ plane, std::make_unique<bool[]>((size_t)sz.x * sz.y) });
+    auto& mask = plane_mask[plane];
+
+    if (!source_color)
+        source_color = std::make_unique<glm::vec4>(rgb[pos.back().y * sz.x + pos.back().x]);
+    glm::vec3 c = *source_color;
+
+    std::array<std::vector<glm::ivec2>, 4> edges;
+    std::array<adj_t, 4> adj[6] = {
+        // front - ok
+        {
+            adj_t(3, 1, 0, 0),
+            adj_t(4, 1, 1, 0),
+            adj_t(1, 0, 0, 0),
+            adj_t(5, 1, 0, 0),
+        },
+        // right - ok
+        {
+            adj_t(0, 1, 0, 0),
+            adj_t(4, 1, 0, 1),
+            adj_t(2, 0, 0, 0),
+            adj_t(5, 0, 0, 1),
+        },
+        // back - ok
+        {
+            adj_t(1, 1, 0, 0),
+            adj_t(4, 0, 0, 0),
+            adj_t(3, 0, 0, 0),
+            adj_t(5, 0, 1, 0),
+        },
+        // left - ok
+        {
+            adj_t(2, 1, 0, 0),
+            adj_t(4, 0, 1, 1),
+            adj_t(0, 0, 0, 0),
+            adj_t(5, 1, 1, 1),
+        },
+        // top - ok
+        {
+            adj_t(1, 1, 1, 1),
+            adj_t(0, 1, 1, 0),
+            adj_t(3, 1, 0, 1),
+            adj_t(2, 0, 1, 0),
+        },
+        // bottom
+        {
+            adj_t(1, 0, 0, 1),
+            adj_t(2, 0, 0, 0),
+            adj_t(3, 0, 1, 1),
+            adj_t(0, 1, 0, 0),
+        },
+    };
+
+    auto test = [&](glm::ivec2 p, bool set_color) -> bool
+    {
+        int i = p.y * sz.x + p.x;
+        if (p.x < 0)
+        {
+            edges[0].push_back(adj[plane][0].compute({ -p.x, p.y }, sz));
+            return false;
+        }
+        else if (p.x >= sz.x)
+        {
+            edges[2].push_back(adj[plane][2].compute({ sz.x - p.x + 1, p.y }, sz));
+            return false;
+        }
+        else if (p.y < 0)
+        {
+            edges[3].push_back(adj[plane][3].compute({ p.x, -p.y }, sz));
+            return false;
+        }
+        else if (p.y >= sz.y)
+        {
+            edges[1].push_back(adj[plane][1].compute({  p.x, sz.y - p.y + 1 }, sz));
+            return false;
+        }
+        if (!mask[i] && rgb[i].a > 0 && glm::distance(c, glm::vec3(rgb[i])) < threshold)
+        {
+            if (set_color)
+            {
+                mask[i] = true;
+                rgb[i] = dest_color * 255.f;
+            }
+            pos.push_back(p);
+            return true;
+        }
+        return false;
+    };
+
+    while (!pos.empty())
+    {
+        auto p = pos.back();
+        pos.pop_back();
+        if(!test(p + glm::ivec2( 0, 0), true))
+            continue;
+        test(p + glm::ivec2(-1, 0), false);
+        test(p + glm::ivec2( 1, 0), false);
+        test(p + glm::ivec2( 0, 1), false);
+        test(p + glm::ivec2( 0,-1), false);
+    }
+
+    rtt.bindTexture();
+    glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, sz.x, sz.y, GL_RGBA, GL_UNSIGNED_BYTE, rgb);
+    rtt.unbindTexture();
+
+    for (int i = 0; i < 4; i++)
+    {
+        if (!edges[i].empty())
+        {
+            flood_fill(layer, adj[plane][i].plane, edges[i], plane_mask, threshold, dest_color, source_color);
+            //LOG("continue to plane %d -> %d", plane, adj[plane][i].plane);
+        }
+    }
+}
+
 void Canvas::resize(int width, int height)
 {
     m_width = width;

src/canvas.h

@@ -166,6 +166,8 @@ public:
     void layer_add(std::string name, std::shared_ptr<Layer> layer = nullptr, int index = 0);
     void layer_order(int idx, int pos);
     void layer_merge(int source_idx, int dest_idx);
+    void flood_fill(int layer, int plane, std::vector<glm::ivec2> pos, std::map<int, std::unique_ptr<bool[]>>& plane_mask,
+        float threshold, glm::vec4 dest_color, std::unique_ptr<glm::vec4>& source_color);
     void stroke_start(glm::vec3 point, float pressure);
     void stroke_update(glm::vec3 point, float pressure);
     void stroke_draw_mix(const glm::vec2& bb_min, const glm::vec2& bb_sz);