fix brush projection to eliminate seams, still problems with big brushes and need to implement erase

engine/util.cpp

@@ -7,6 +7,59 @@ bool point_in_rect(const glm::vec2& p, const glm::vec4& r)
     return p.x > r.x && p.x < r.x+r.z && p.y > r.y && p.y < r.y+r.w;
 }
 
+// params and returns {origin, size} form
+glm::vec4 rect_intersection(glm::vec4 a, glm::vec4 b)
+{
+    // convert from [x,y,w,h] to [x1,y1,x2,y1]
+    a = glm::vec4(a.xy(), a.xy() + a.zw());
+    b = glm::vec4(b.xy(), b.xy() + b.zw());
+    // compute intersection
+    auto o = glm::vec4(glm::max(a.xy(), b.xy()), glm::min(a.zw(), b.zw()));
+    // back to rect form
+    o = glm::vec4(o.xy(), glm::max({ 0, 0 }, o.zw() - o.xy()));
+    return o;
+}
+
+// params and returns {origin, size} form
+glm::vec4 rect_union(glm::vec4 a, glm::vec4 b)
+{
+    // convert from rect [x,y,w,h] to bb [x1,y1,x2,y1]
+    a = glm::vec4(a.xy(), a.xy() + a.zw());
+    b = glm::vec4(b.xy(), b.xy() + b.zw());
+    // compute union
+    glm::vec4 o = { glm::min(a.xy(), b.xy()), glm::max(a.zw(), b.zw()) };
+    // back to rect form
+    o = glm::vec4(o.xy(), glm::max({ 0, 0 }, o.zw() - o.xy()));
+    return o;
+}
+
+// see: https://stackoverflow.com/questions/563198/how-do-you-detect-where-two-line-segments-intersect
+bool segments_intersect(const glm::vec2& p0a, const glm::vec2& p0b,
+    const glm::vec2& p1a, const glm::vec2& p1b, glm::vec2& out_pt)
+{
+    auto cross2d = [](const glm::vec2& v, const glm::vec2& w)
+        { return (v.x * w.y) - (v.y * w.x); };
+    auto p = p0a;
+    auto r = p0b - p0a;
+    auto q = p1a;
+    auto s = p1b - p1a;
+    float den = cross2d(r, s);
+    if (den == 0.f)
+    {
+        glm::vec4 is = rect_intersection({p, r}, {q, s});
+        out_pt = is.xy + is.zw * 0.5f;
+        return glm::all(glm::greaterThan(is.zw(), glm::vec2(0, 0)));
+    }
+    float t = cross2d(q - p, s) / den;
+    float u = cross2d(q - p, r) / den;
+    if (t >= 0 && t <= 1 && u >= 0 && u <= 1)
+    {
+        out_pt = p + t * r;
+        return true;
+    }
+    return false;
+}
+
 glm::vec4 rand_color()
 {
     float r = (rand() % 256) / 256.f;