add timelapse video generation on iOS and mac , fix history memory

engine/app.cpp

@@ -3,6 +3,7 @@
 #include "app.h"
 #include "node_icon.h"
 #include "node_dialog_open.h"
+#include "node_progress_bar.h"
 
 #ifdef __APPLE__
 #include <Foundation/Foundation.h>
@@ -57,6 +58,13 @@ void App::initLog()
     NSArray* paths = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES);
     NSString* docpath = [paths objectAtIndex:0];
     data_path = [docpath cStringUsingEncoding:NSASCIIStringEncoding];
+    NSString* recpath = [docpath stringByAppendingString:@"/rec"];
+    rec_path = [recpath cStringUsingEncoding:NSASCIIStringEncoding];
+    NSError* recerr = nil;
+    if (![[NSFileManager defaultManager] createDirectoryAtPath:recpath withIntermediateDirectories:YES attributes:nil error:&recerr])
+    {
+        LOG("error creating rec path: %s", [[recerr localizedDescription] cStringUsingEncoding:NSASCIIStringEncoding]);
+    }
 #elif _WIN32
     //CHAR my_documents[MAX_PATH];
     //HRESULT result = SHGetFolderPathA(NULL, CSIDL_PERSONAL, NULL, SHGFP_TYPE_CURRENT, my_documents);
@@ -277,6 +285,8 @@ void App::async_end()
 
 void App::update(float dt)
 {
+    static float rec_timer = 0.f;
+    
     static std::mutex m;
     std::lock_guard<std::mutex> _lock(m);
 
@@ -335,6 +345,26 @@ void App::update(float dt)
     //msgbox->watch(observer);
     glDisable(GL_SCISSOR_TEST);
     
+    if (rec_running)
+    {
+        rec_timer += dt;
+        if (rec_timer > 1.f)
+        {
+            LOG("rec tick");
+            rec_timer = 0.f;
+            
+            auto data = new uint8_t[width * height * 4];
+            glReadBuffer(GL_BACK);
+            glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, data);
+            {
+                std::lock_guard<std::mutex> lock(rec_mutex);
+                rec_frames.emplace_back(data);
+                rec_cv.notify_all();
+            }
+            update_rec_frames();
+        }
+    }
+    
     redraw = false;
 }
 
@@ -349,6 +379,7 @@ void App::terminate()
     layers->clear_context();
     color->clear_context();
     stroke->clear_context();
+    rec_stop();
 }
 
 void App::update_memory_usage(size_t bytes)
@@ -361,3 +392,280 @@ void App::update_memory_usage(size_t bytes)
         layout[main_id]->update();
     }
 }
+
+void App::update_rec_frames()
+{
+    if (auto txt = layout[main_id]->find<NodeText>("txt-rec"))
+    {
+        if (rec_running)
+        {
+            static char buffer[128];
+            sprintf(buffer, "Recorder %d frames", rec_count);
+            txt->set_text(buffer);
+        }
+        else
+        {
+            txt->set_text("");
+        }
+        layout[main_id]->update();
+    }
+}
+
+void App::rec_clear()
+{
+    rec_stop();
+#if defined(__IOS__) || defined(__OSX__)
+    NSString *path = [NSString stringWithUTF8String:rec_path.c_str()];
+    NSDirectoryEnumerator* en = [[NSFileManager defaultManager] enumeratorAtPath:path];
+    NSError* err = nil;
+    BOOL res;
+    NSString* file;
+    while (file = [en nextObject])
+    {
+        NSString* file_path = [path stringByAppendingPathComponent:file];
+        [[NSFileManager defaultManager] removeItemAtPath:file_path error:nil];
+        NSLog(@"delete: %@", file_path);
+    }
+#endif
+    rec_count = 0;
+    update_rec_frames();
+}
+
+void App::rec_start()
+{
+    if (!rec_running)
+    {
+        NSString* path = [NSString stringWithUTF8String:rec_path.c_str()];
+        NSArray *dirFiles = [[NSFileManager defaultManager] contentsOfDirectoryAtPath:path error:nil];
+        NSArray *jpgFiles = [dirFiles filteredArrayUsingPredicate:[NSPredicate predicateWithFormat:@"self ENDSWITH '.jpg'"]];
+        rec_count = (int)[jpgFiles count];
+        update_rec_frames();
+        rec_thread = std::thread(&App::rec_loop, this);
+    }
+}
+
+void App::rec_stop()
+{
+    if (rec_running)
+    {
+        rec_running = false;
+        rec_cv.notify_all();
+        if (rec_thread.joinable())
+            rec_thread.join();
+        update_rec_frames();
+    }
+}
+
+void stillImageDataReleaseCallback(void *releaseRefCon, const void *baseAddress)
+{
+    free((void *)baseAddress);
+}
+
+void App::rec_export(std::string path)
+{
+    int progress = 0;
+    int tot = rec_count;
+    auto pb = layout[main_id]->add_child<NodeProgressBar>();
+    pb->m_progress->SetWidthP(0);
+    pb->m_title->set_text("Exporting MP4 movie");
+    async_update();
+
+#if defined(__IOS__) || defined(__OSX__)
+    NSString* mov_path = [NSString stringWithFormat:@"%s/out.mp4", rec_path.c_str()];
+    if ([[NSFileManager defaultManager] fileExistsAtPath:mov_path])
+    {
+        NSLog(@"remove existing mp4");
+        [[NSFileManager defaultManager] removeItemAtPath:mov_path error:nil];
+    }
+    NSURL* url = [NSURL fileURLWithPath:[NSString stringWithFormat:@"%s/out.mp4", rec_path.c_str()]];
+    AVAssetWriter* writer = [AVAssetWriter assetWriterWithURL:url fileType:AVFileTypeMPEG4 error:nil];
+    writer.shouldOptimizeForNetworkUse = NO;
+    NSDictionary *videoCompressionSettings = @{
+        AVVideoCodecKey                  : AVVideoCodecH264,
+        AVVideoWidthKey                  : @(width),
+        AVVideoHeightKey                 : @(height),
+        AVVideoCompressionPropertiesKey  : @{ AVVideoAverageBitRateKey : @(8<<20) }
+    };
+    if (![writer canApplyOutputSettings:videoCompressionSettings forMediaType:AVMediaTypeVideo])
+    {
+        NSLog(@"Couldn't add asset writer video input.");
+        return;
+    }
+    AVAssetWriterInput* input = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo
+                                                              outputSettings:videoCompressionSettings
+                                                            sourceFormatHint:nil];
+    input.expectsMediaDataInRealTime = YES;
+  
+    NSDictionary *adaptorDict = @{
+                                (id)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA),
+                                (id)kCVPixelBufferWidthKey : @(width),
+                                (id)kCVPixelBufferHeightKey : @(height)
+                                };
+
+    AVAssetWriterInputPixelBufferAdaptor* _pixelBufferAdaptor = [[AVAssetWriterInputPixelBufferAdaptor alloc]
+                         initWithAssetWriterInput:input
+                         sourcePixelBufferAttributes:adaptorDict];
+
+
+    // Add asset writer input to asset writer
+    if (![writer canAddInput:input]) {
+        NSLog(@"Couldn't add input to writer.");
+        return;
+    }
+
+    [writer addInput:input];
+
+    CMTime t;
+    t.timescale = 30;
+    t.flags = kCMTimeFlags_Valid;
+    t.epoch = 0;
+    t.value = 0;
+    //[writer startSessionAtSourceTime:t];
+    
+    CVPixelBufferRef buff = NULL;
+    uint8_t* data = (uint8_t*)calloc(1, width * height * 4);
+    CVPixelBufferCreateWithBytes(kCFAllocatorDefault, width, height, kCVPixelFormatType_32RGBA, data,
+        width * 4, stillImageDataReleaseCallback, nil, nil, &buff);
+    OSStatus err = CVPixelBufferPoolCreatePixelBuffer(nil, _pixelBufferAdaptor.pixelBufferPool, &buff);
+
+
+    if (writer.status == AVAssetWriterStatusUnknown)
+    {
+        // If the asset writer status is unknown, implies writing hasn't started yet, hence start writing with start time as the buffer's presentation timestamp
+        if ([writer startWriting])
+        {
+            [writer startSessionAtSourceTime:t];
+        }
+    }
+
+    if (writer.status == AVAssetWriterStatusWriting)
+    {
+        for (int i = 0; i < tot; i++)
+        {
+            // If the asset writer status is writing, append sample buffer to its corresponding asset writer input
+            if (input.readyForMoreMediaData)
+            {
+                char path[256];
+                snprintf(path, sizeof(path), "%s/%04d.jpg", rec_path.c_str(), i);
+                NSString* img_path = [NSString stringWithUTF8String:path];
+                NSLog(@"frame: %@", img_path);
+#if __OSX__
+                NSImage *image = [[NSImage alloc] initWithContentsOfFile:img_path];
+                if (!image)
+                    break;
+                NSRect imageRect = NSMakeRect(0, 0, image.size.width, image.size.height);
+                CGImageRef cgImage = [image CGImageForProposedRect:&imageRect context:NULL hints:nil];
+#elif __IOS__
+                UIImage* image = [UIImage imageNamed:img_path];
+                if (!image)
+                    break;
+                CGImageRef cgImage = image.CGImage;
+#endif
+
+                NSDictionary *options = [NSDictionary dictionaryWithObjectsAndKeys:
+                             [NSNumber numberWithBool:YES], kCVPixelBufferCGImageCompatibilityKey,
+                             [NSNumber numberWithBool:YES], kCVPixelBufferCGBitmapContextCompatibilityKey, nil];
+                CGSize sz = image.size;
+                CVPixelBufferRef pxbuffer = NULL;
+                {
+                    CGImageRef image = cgImage;
+                    CGSize size = sz;
+                    CVReturn status = CVPixelBufferCreate(kCFAllocatorDefault, size.width, size.height, kCVPixelFormatType_32ARGB, (__bridge CFDictionaryRef) options, &pxbuffer);
+                    // CVReturn status = CVPixelBufferPoolCreatePixelBuffer(NULL, adaptor.pixelBufferPool, &pxbuffer);
+
+                    //NSParameterAssert(status == kCVReturnSuccess && pxbuffer != NULL);
+
+                    CVPixelBufferLockBaseAddress(pxbuffer, 0);
+                    void *pxdata = CVPixelBufferGetBaseAddress(pxbuffer);
+                    //NSParameterAssert(pxdata != NULL);
+
+                    CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB();
+                    CGContextRef context = CGBitmapContextCreate(pxdata, size.width, size.height, 8, 4*size.width, rgbColorSpace, kCGImageAlphaPremultipliedFirst);
+                    //NSParameterAssert(context);
+
+                    CGContextDrawImage(context, CGRectMake(0, 0, CGImageGetWidth(image), CGImageGetHeight(image)), image);
+
+                    CGColorSpaceRelease(rgbColorSpace);
+                    CGContextRelease(context);
+
+                    CVPixelBufferUnlockBaseAddress(pxbuffer, 0);
+                }
+
+                t.value = i;
+                if (![_pixelBufferAdaptor appendPixelBuffer:pxbuffer withPresentationTime:t])
+                {
+                    NSLog(@"error %@", [writer.error localizedFailureReason]);
+                }
+                CFRelease(pxbuffer);
+                
+                progress++;
+                pb->m_progress->SetWidthP((float)progress / tot * 100.f);
+                async_update();
+            }
+        }
+        [input markAsFinished];
+        [writer finishWritingWithCompletionHandler:^{
+            NSString* path = [NSString stringWithFormat:@"%s/out.mp4", rec_path.c_str()];
+            NSLog(@"saved video %@", path);
+#if __IOS__
+            if (UIVideoAtPathIsCompatibleWithSavedPhotosAlbum(path))
+            {
+                NSLog(@"saving to camera roll");
+                UISaveVideoAtPathToSavedPhotosAlbum(path, nil, nil, nil);
+            }
+#endif
+        }];
+    }
+
+    if (writer.status == AVAssetWriterStatusFailed)
+    {
+        NSLog(@"failed");
+    }
+    
+#endif
+    
+    pb->destroy();
+    async_update();
+
+}
+
+void App::rec_loop()
+{
+    rec_running = true;
+    while(rec_running)
+    {
+        std::unique_lock<std::mutex> lock(rec_mutex);
+        rec_cv.wait(lock);
+        if (!rec_running)
+            break;
+        if (!rec_frames.empty())
+        {
+            if (rec_frames.front())
+            {
+                auto inverted = std::make_unique<uint8_t[]>(width*height*4);
+                for (int y = height - 1, y1 = 0; y >= 0; y--, y1++)
+                {
+                    uint8_t* dst = &inverted[y * width * 4];
+                    uint8_t* src = &rec_frames.front()[y1 * width * 4];
+                    std::copy_n(src, (int)width * 4, dst);
+                }
+                char path[256];
+                snprintf(path, sizeof(path), "%s/%04d.jpg", rec_path.c_str(), rec_count);
+                LOG("writing %s", path);
+                jpge::params params;
+                params.m_quality = 75;
+                bool saved = jpge::compress_image_to_jpeg_file(path, width, height, 4, inverted.get(), params);
+                if (!saved)
+                {
+                    LOG("error writing the frame");
+                    rec_running = false;
+                }
+                else
+                {
+                    rec_count++;
+                }
+            }
+            rec_frames.pop_front();
+        }
+    }
+}

engine/app.h

@@ -36,6 +36,15 @@ class App
 public:
     static App I;
     std::string data_path{ "." };
+    
+    std::string rec_path{ "." };
+    std::thread rec_thread;
+    bool rec_running = false;
+    int rec_count = 0;
+    std::mutex rec_mutex;
+    std::condition_variable rec_cv;
+    std::deque<std::unique_ptr<uint8_t[]>> rec_frames;
+    
     Sampler sampler;
     Texture2D tex;
     LayoutManager layout;
@@ -94,7 +103,6 @@ public:
     void create();
     void terminate();
     void clear();
-    void update_memory_usage(size_t bytes);
     void update(float dt);
     void async_start();
     void async_update();
@@ -113,12 +121,19 @@ public:
     bool key_up(kKey key);
     bool key_char(char key);
 
+    void rec_clear();
+    void rec_loop();
+    void rec_start();
+    void rec_stop();
+    void rec_export(std::string path);
+    
     void init_toolbar_main();
     void init_toolbar_draw();
     void init_sidebar();
     void init_menu_file();
     void init_menu_edit();
     void init_menu_layer();
+    void init_menu_timelapse();
     void dialog_newdoc();
     void dialog_save();
     void dialog_save_ver();
@@ -137,6 +152,8 @@ public:
 
     void brush_update();
     void title_update(std::string name, int resolution);
+    void update_memory_usage(size_t bytes);
+    void update_rec_frames();
 
     void cmd_convert(std::string pano_path, std::string out_path);
 };

engine/app_layout.cpp

@@ -430,6 +430,51 @@ void App::init_menu_edit()
     }
 }
 
+void App::init_menu_timelapse()
+{
+    if (auto* menu_file = layout[main_id]->find<NodeButtonCustom>("menu-timelapse"))
+    {
+        menu_file->on_click = [=](Node*) {
+            glm::vec2 pos = menu_file->m_pos + glm::vec2(0, menu_file->m_size.y);
+            popup = (NodePopupMenu*)layout[const_hash("timelapse-menu")]->m_children[0]->clone();
+            popup->SetPositioning(YGPositionTypeAbsolute);
+            popup->SetPosition(pos.x, pos.y);
+            layout[main_id]->add_child(popup);
+            layout[main_id]->update();
+            popup->mouse_capture();
+            popup->m_mouse_ignore = false;
+            popup->m_flood_events = true;
+            popup->m_capture_children = false;
+            
+            if (auto item = popup->find<NodeButtonCustom>("timelapse-start"))
+            {
+                if (auto text = popup->find<NodeText>("menu-label"))
+                {
+                    text->set_text(App::I.rec_running ? "Stop Recording" : "Start Recording");
+                }
+            }
+
+            popup->find<NodeButtonCustom>("timelapse-start")->on_click = [this](Node*) {
+                App::I.rec_running ? App::I.rec_stop() : App::I.rec_start();
+                popup->mouse_release();
+                popup->destroy();
+            };
+
+            popup->find<NodeButtonCustom>("timelapse-clear")->on_click = [this](Node*) {
+                App::I.rec_clear();
+                popup->mouse_release();
+                popup->destroy();
+            };
+
+            popup->find<NodeButtonCustom>("timelapse-export")->on_click = [this](Node*) {
+                popup->mouse_release();
+                popup->destroy();
+                App::I.rec_export("");
+            };
+        };
+    }
+}
+
 void App::brush_update()
 {
 //     brushes->select_brush(canvas->m_brush.id);
@@ -549,6 +594,7 @@ void App::initLayout()
         init_menu_file();
         init_menu_edit();
         init_menu_layer();
+        init_menu_timelapse();
 
         Brush b;
         b.m_tex_id = brushes->get_texture_id(0);

engine/canvas.h

@@ -163,11 +163,11 @@ class ActionStroke : public Action
 {
 public:
     std::unique_ptr<Stroke> m_stroke;
-    std::unique_ptr<uint8_t[]> m_image[6];
-    glm::ivec4 m_old_box[6];
-    bool m_old_dirty[6];
-    glm::ivec4 m_box[6];
-    bool m_dirty[6];
+    std::unique_ptr<uint8_t[]> m_image[6] = SIXPLETTE(nullptr);
+    glm::ivec4 m_old_box[6] = SIXPLETTE(glm::ivec4(0));
+    bool m_old_dirty[6] = SIXPLETTE(false);
+    glm::ivec4 m_box[6] = SIXPLETTE(glm::ivec4(0));
+    bool m_dirty[6] = SIXPLETTE(false);
     bool clear_layer = false;
     int m_layer_idx;
     Canvas* m_canvas;

engine/node.cpp

@@ -56,6 +56,8 @@ void Node::watch(std::function<void(Node*)> observer)
 void Node::destroy()
 {
     m_destroyed = true;
+    mouse_release();
+    key_release();
 }
 
 Node* Node::root()

engine/node_button_custom.cpp

@@ -54,9 +54,9 @@ kEventResult NodeButtonCustom::handle_event(Event* e)
         #else
             m_color = m_mouse_inside ? color_hover : color_normal;
         #endif
+        mouse_release();
         if (m_mouse_inside && on_click != nullptr)
             on_click(this);
-        mouse_release();
         break;
     case kEventType::MouseCancel:
         m_color = color_normal;

engine/pch.h

@@ -11,7 +11,8 @@
         #define __IOS__ 1
         #include <CoreFoundation/CoreFoundation.h>
         #ifdef __OBJC__
-            #include <Foundation/Foundation.h>
+            #import <Foundation/Foundation.h>
+            #import <AVFoundation/AVFoundation.h>
             #import <UIKit/UIKit.h>
             #import <GLKit/GLKit.h>
         #endif
@@ -23,7 +24,8 @@
         #define __IOS__ 1
         #include <CoreFoundation/CoreFoundation.h>
         #ifdef __OBJC__
-            #include <Foundation/Foundation.h>
+            #import <Foundation/Foundation.h>
+            #import <AVFoundation/AVFoundation.h>
             #import <UIKit/UIKit.h>
             #import <GLKit/GLKit.h>
         #endif
@@ -33,6 +35,10 @@
     #else
         #define TARGET_OS_OSX 1
         #define __OSX__ 1
+        #ifdef __OBJC__
+            #import <Foundation/Foundation.h>
+            #import <AVFoundation/AVFoundation.h>
+        #endif
         #include <OpenGL/gl3.h>
         #include <OpenGL/gl3ext.h>
         #define SHADER_VERSION "#version 150\n"