From 9eecf60219da9c46bd209cde2eeec394a7ddbdef Mon Sep 17 00:00:00 2001 From: omigamedev Date: Thu, 31 Oct 2019 23:22:09 +0100 Subject: [PATCH] add MP4Encoder class and test timelapse exporting --- PanoPainter.vcxproj | 10 +- PanoPainter.vcxproj.filters | 6 ++ src/app.cpp | 47 ++++----- src/app.h | 14 +-- src/app_dialogs.cpp | 5 +- src/app_layout.cpp | 8 +- src/binary_stream.h | 12 ++- src/mp4enc.cpp | 192 ++++++++++++++++++++++++++++++++++++ src/mp4enc.h | 33 +++++++ src/pch.cpp | 1 + src/version.cpp | 2 +- 11 files changed, 283 insertions(+), 47 deletions(-) create mode 100644 src/mp4enc.cpp create mode 100644 src/mp4enc.h diff --git a/PanoPainter.vcxproj b/PanoPainter.vcxproj index 93c66de..38659d7 100644 --- a/PanoPainter.vcxproj +++ b/PanoPainter.vcxproj @@ -77,8 +77,8 @@ true - libs\glm;libs\glew-2.0.0\include;libs\stb;libs\tinyxml2;libs\yoga;libs\curl-win\include;libs\jpeg;libs\wacom;libs\bugtrap-client\include;libs\poly2tri\poly2tri;libs\base64;libs\sqlite3;libs\openvr\headers;libs\nanort;libs\hash-library;libs\fmt\include;libs\glad\include;libs\openh264\include;libs\mp4v2\include;$(IncludePath) - libs\curl-win\lib\dll-$(Configuration)-$(PlatformShortName);libs\glew-2.0.0\lib\Release\$(Platform);libs\bugtrap-client\lib;libs\openvr\lib\win64;libs\openh264\lib;libs\mp4v2\lib\win;$(LibraryPath) + libs\glm;libs\glew-2.0.0\include;libs\stb;libs\tinyxml2;libs\yoga;libs\curl-win\include;libs\jpeg;libs\wacom;libs\bugtrap-client\include;libs\poly2tri\poly2tri;libs\base64;libs\sqlite3;libs\openvr\headers;libs\nanort;libs\hash-library;libs\fmt\include;libs\glad\include;libs\openh264\include;libs\mp4v2\include;libs\libyuv\include;$(IncludePath) + libs\curl-win\lib\dll-$(Configuration)-$(PlatformShortName);libs\glew-2.0.0\lib\Release\$(Platform);libs\bugtrap-client\lib;libs\openvr\lib\win64;libs\openh264\lib;libs\mp4v2\lib\win;libs\libyuv\lib\win;$(LibraryPath) false @@ -87,8 +87,8 @@ false - libs\glm;libs\glew-2.0.0\include;libs\stb;libs\tinyxml2;libs\yoga;libs\curl-win\include;libs\jpeg;libs\wacom;libs\bugtrap-client\include;libs\poly2tri\poly2tri;libs\base64;libs\sqlite3;libs\openvr\headers;libs\nanort;libs\hash-library;libs\fmt\include;libs\glad\include;libs\openh264\include;libs\mp4v2\include;$(IncludePath) - libs\curl-win\lib\dll-$(Configuration)-$(PlatformShortName);libs\glew-2.0.0\lib\Release\$(Platform);libs\bugtrap-client\lib;libs\openvr\lib\win64;libs\openh264\lib;libs\mp4v2\lib\win;$(LibraryPath) + libs\glm;libs\glew-2.0.0\include;libs\stb;libs\tinyxml2;libs\yoga;libs\curl-win\include;libs\jpeg;libs\wacom;libs\bugtrap-client\include;libs\poly2tri\poly2tri;libs\base64;libs\sqlite3;libs\openvr\headers;libs\nanort;libs\hash-library;libs\fmt\include;libs\glad\include;libs\openh264\include;libs\mp4v2\include;libs\libyuv\include;$(IncludePath) + libs\curl-win\lib\dll-$(Configuration)-$(PlatformShortName);libs\glew-2.0.0\lib\Release\$(Platform);libs\bugtrap-client\lib;libs\openvr\lib\win64;libs\openh264\lib;libs\mp4v2\lib\win;libs\libyuv\lib\win;$(LibraryPath) @@ -346,6 +346,7 @@ + @@ -475,6 +476,7 @@ + diff --git a/PanoPainter.vcxproj.filters b/PanoPainter.vcxproj.filters index 4a7d568..cdae9e9 100644 --- a/PanoPainter.vcxproj.filters +++ b/PanoPainter.vcxproj.filters @@ -381,6 +381,9 @@ libs\yoga + + Source Files + @@ -632,6 +635,9 @@ Source Files\ui + + Header Files + diff --git a/src/app.cpp b/src/app.cpp index cf5b1c0..cbfda38 100644 --- a/src/app.cpp +++ b/src/app.cpp @@ -4,6 +4,7 @@ #include "node_icon.h" #include "node_dialog_open.h" #include "node_progress_bar.h" +#include "mp4enc.h" #ifdef __APPLE__ #include @@ -616,14 +617,15 @@ void App::draw(float dt) { static float rec_timer = 0.f; rec_timer += dt; - if (rec_timer > 1.f && canvas->m_canvas->m_dirty_stroke) + if (rec_timer > 0.3f && canvas->m_canvas->m_dirty_stroke) { canvas->m_canvas->m_dirty_stroke = false; LOG("rec tick"); rec_timer = 0.f; - auto data = new uint8_t[(int)width * (int)height * 4]; + auto img = std::make_unique(); + img->create(width, height); #if __IOS__ [ios_view->glview bindDrawable]; #else @@ -636,10 +638,10 @@ void App::draw(float dt) if (dfbo != rfbo) LOG("DIFFERENT FB"); - glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, data); + glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, img->m_data.get()); { std::lock_guard lock(rec_mutex); - rec_frames.emplace_back(data); + rec_frames.emplace_back(std::move(img)); rec_cv.notify_all(); } update_rec_frames(); @@ -805,11 +807,14 @@ void App::rec_export(std::string path) pb->m_progress->SetWidthP(0); pb->m_title->set_text("Exporting MP4 movie"); +/* #if defined(__IOS__) || defined(__OSX__) export_mp4(rec_path, width, height, rec_count, ^(float) { pb->m_progress->SetWidthP((float)progress / tot * 100.f); }); #endif +*/ + m_encoder->write_mp4(data_path + "/export.mp4"); pb->destroy(); } @@ -818,8 +823,14 @@ void App::rec_loop() { BT_SetTerminate(); rec_running = true; + if (!m_encoder) + { + m_encoder = std::make_unique(); + m_encoder->init(512, 512, 30, 500 << 10); + } while(rec_running) { + std::unique_ptr frame; std::unique_lock lock(rec_mutex); rec_cv.wait(lock); if (!rec_running) @@ -828,32 +839,14 @@ void App::rec_loop() { if (rec_frames.front()) { - auto inverted = std::make_unique(width*height*4); - for (int y = height - 1, y1 = 0; y >= 0; y--, y1++) - { - uint8_t* dst = &inverted[y * width * 4]; - uint8_t* src = &rec_frames.front()[y1 * width * 4]; - std::copy_n(src, (int)width * 4, dst); - } - char path[256]; - snprintf(path, sizeof(path), "%s/%04d.jpg", rec_path.c_str(), rec_count); - LOG("writing %s", path); - jpge::params params; - params.m_quality = 75; - bool saved = jpge::compress_image_to_jpeg_file(path, width, height, 4, inverted.get(), params); - if (!saved) - { - LOG("error writing the frame"); - rec_running = false; - } - else - { - rec_count++; - redraw = true; - } + rec_count++; + frame = std::move(rec_frames.front()); } rec_frames.pop_front(); } + lock.unlock(); + if (frame) + m_encoder->encode(*frame); } } diff --git a/src/app.h b/src/app.h index 3b41cfb..1186e74 100644 --- a/src/app.h +++ b/src/app.h @@ -7,6 +7,8 @@ #include "layout.h" #include "font.h" #include "rtt.h" +#include "image.h" +#include "mp4enc.h" #include "node_message_box.h" #include "node_settings.h" #include "node_popup_menu.h" @@ -18,6 +20,10 @@ #include "node_canvas.h" #include "node_dialog_layer_rename.h" #include "node_progress_bar.h" +#include "node_panel_grid.h" +#include "node_panel_quick.h" +#include "node_input_box.h" +#include "node_panel_animation.h" #if defined(__OBJC__) && defined(__IOS__) #import @@ -37,11 +43,6 @@ #include #endif -#include "node_panel_grid.h" -#include "node_panel_quick.h" -#include "node_input_box.h" -#include "node_panel_animation.h" - struct VRController { enum class kButton : uint8_t @@ -94,7 +95,8 @@ public: int rec_count = 0; std::mutex rec_mutex; std::condition_variable rec_cv; - std::deque> rec_frames; + std::deque> rec_frames; + std::unique_ptr m_encoder; RTT uirtt; Sampler sampler; diff --git a/src/app_dialogs.cpp b/src/app_dialogs.cpp index e8bf1ea..631aac0 100644 --- a/src/app_dialogs.cpp +++ b/src/app_dialogs.cpp @@ -786,10 +786,7 @@ void App::dialog_export_mp4() { int nalu_sz = info.sLayerInfo[layer].pNalLengthInByte[nal]; uint8_t* data = info.sLayerInfo[layer].pBsBuf + bs_size; - data[0] = (nalu_sz - 4) >> 24; - data[1] = (nalu_sz - 4) >> 16; - data[2] = (nalu_sz - 4) >> 8; - data[3] = (nalu_sz - 4) & 0xff; + *(uint32_t*)data = BinaryStream::htonx(nalu_sz - 4); bool sync = false; if (nalu_bytes[4] == 0x65) // I-frame sync = true; diff --git a/src/app_layout.cpp b/src/app_layout.cpp index 0ab3a4b..610cbec 100644 --- a/src/app_layout.cpp +++ b/src/app_layout.cpp @@ -729,20 +729,20 @@ void App::init_menu_tools() popup_time->find("timelapse-start")->on_click = [this, popup_time, popup_exp](Node*) { rec_running ? rec_stop() : rec_start(); - popup_time->destroy(); popup_exp->destroy(); + popup_time->destroy(); }; popup_time->find("timelapse-clear")->on_click = [this, popup_time, popup_exp](Node*) { rec_clear(); - popup_time->destroy(); popup_exp->destroy(); + popup_time->destroy(); }; popup_time->find("timelapse-export")->on_click = [this, popup_time, popup_exp](Node*) { - popup_time->destroy(); - popup_exp->destroy(); rec_export(""); + popup_exp->destroy(); + popup_time->destroy(); }; } }; diff --git a/src/binary_stream.h b/src/binary_stream.h index e4eed84..4984e45 100644 --- a/src/binary_stream.h +++ b/src/binary_stream.h @@ -15,7 +15,17 @@ public: return bint.c[0] == 1 ? ByteOrder::BigEndian : ByteOrder::LittleEndian; } - template T swap(T x) + template static T htonx(T x) + { + static auto sys = sys_order(); + return sys == ByteOrder::BigEndian ? x : swap(x); + } + template static T ntohx(T x) + { + static auto sys = sys_order(); + return sys == ByteOrder::BigEndian ? x : swap(x); + } + template static T swap(T x) { #if _MSC_VER >= 1400 diff --git a/src/mp4enc.cpp b/src/mp4enc.cpp new file mode 100644 index 0000000..6d4c35a --- /dev/null +++ b/src/mp4enc.cpp @@ -0,0 +1,192 @@ +#include "pch.h" +#include "mp4enc.h" + +#include +#include + +#define MP4V2_NO_STDINT_DEFS +#include + +static void encoder_trace_callback(void* context, int level, const char* message) +{ + LOG("ENCODER: %s", message); +} + +bool MP4Encoder::init(int width, int height, int fps, int bitrate) noexcept +{ + if (WelsCreateSVCEncoder(&m_encoder) != 0) + return false; + + m_width = width; + m_height = height; + m_framerate = fps; + m_bitrate = bitrate; + + //Encoder params + SEncParamExt param; + m_encoder->GetDefaultParams(¶m); + param.iUsageType = CAMERA_VIDEO_REAL_TIME; + param.fMaxFrameRate = m_framerate; + param.iLtrMarkPeriod = 75; + param.iPicWidth = m_width; + param.iPicHeight = m_height; + param.iTargetBitrate = m_bitrate; + param.bEnableDenoise = false; + param.iSpatialLayerNum = 1; + param.bUseLoadBalancing = false; + param.bEnableSceneChangeDetect = false; + param.bEnableBackgroundDetection = false; + param.bEnableAdaptiveQuant = false; + param.bEnableFrameSkip = false; + param.iMultipleThreadIdc = 0; + //param.uiIntraPeriod = 10; + + for (int i = 0; i < param.iSpatialLayerNum; i++) + { + param.sSpatialLayers[i].iVideoWidth = m_width >> (param.iSpatialLayerNum - 1 - i); + param.sSpatialLayers[i].iVideoHeight = m_height >> (param.iSpatialLayerNum - 1 - i); + param.sSpatialLayers[i].fFrameRate = m_framerate; + param.sSpatialLayers[i].iSpatialBitrate = m_bitrate; + param.sSpatialLayers[i].uiProfileIdc = PRO_BASELINE; + param.sSpatialLayers[i].uiLevelIdc = LEVEL_4_2; + param.sSpatialLayers[i].iDLayerQp = 42; + + //SSliceArgument sliceArg; + //sliceArg.uiSliceMode = SM_FIXEDSLCNUM_SLICE; + //sliceArg.uiSliceNum = 0; + //param.sSpatialLayers[i].sSliceArgument = sliceArg; + } + + param.uiMaxNalSize = 1500; + param.iTargetBitrate *= param.iSpatialLayerNum; + m_encoder->InitializeExt(¶m); + + int trace_level = WELS_LOG_ERROR; + m_encoder->SetOption(ENCODER_OPTION_TRACE_LEVEL, &trace_level); + m_encoder->SetOption(ENCODER_OPTION_TRACE_CALLBACK, encoder_trace_callback); + m_encoder->SetOption(ENCODER_OPTION_TRACE_CALLBACK_CONTEXT, this); + int videoFormat = videoFormatI420; + m_encoder->SetOption(ENCODER_OPTION_DATAFORMAT, &videoFormat); +} + +bool MP4Encoder::encode(const Image& rgba) noexcept +{ + // lazy allocation of YUV buffer + if (m_yuv_buffer.empty()) + m_yuv_buffer.resize(m_width * m_height * 3 / 2); + + SFrameBSInfo info = { 0 }; + SSourcePicture pic = { 0 }; + pic.iPicWidth = m_width; + pic.iPicHeight = m_height; + pic.iColorFormat = videoFormatI420; + pic.iStride[0] = pic.iPicWidth; + pic.iStride[1] = pic.iStride[2] = pic.iPicWidth >> 1; + pic.pData[0] = m_yuv_buffer.data(); + pic.pData[1] = pic.pData[0] + ((size_t)m_width * m_height); + pic.pData[2] = pic.pData[1] + ((size_t)m_width * m_height / 4); + + if (rgba.width != m_width || rgba.height != m_height) + { + Image resized = rgba.resize(m_width, m_height); + libyuv::ABGRToI420(resized.data(), resized.width * 4, pic.pData[0], m_width, + pic.pData[1], m_width / 2, pic.pData[2], m_width / 2, m_width, m_height); + } + else + { + libyuv::ABGRToI420(rgba.data(), rgba.width * 4, pic.pData[0], m_width, + pic.pData[1], m_width / 2, pic.pData[2], m_width / 2, m_width, m_height); + } + + if (m_encoder->EncodeFrame(&pic, &info)) + { + LOG("EcncodeFrame failed"); + return false; + } + + // No frames generated at this time + if (info.eFrameType == videoFrameTypeSkip) + return false; + + for (int layer = 0; layer < info.iLayerNum; layer++) + { + size_t bs_size = 0; + for (int nal = 0; nal < info.sLayerInfo[layer].iNalCount; nal++) + { + std::array nalu_bytes; + for (int i = 0; i < nalu_bytes.size(); i++) + nalu_bytes[i] = info.sLayerInfo[layer].pBsBuf[bs_size + i]; + if (nalu_bytes[4] == 0x67) // SPS + { + uint8_t avc_profile = info.sLayerInfo[layer].pBsBuf[bs_size + 5]; + uint8_t avc_profile_compat = info.sLayerInfo[layer].pBsBuf[bs_size + 6]; + uint8_t avc_level = info.sLayerInfo[layer].pBsBuf[bs_size + 7]; + uint8_t* ptr = info.sLayerInfo[layer].pBsBuf + bs_size + 4; + size_t sz = info.sLayerInfo[layer].pNalLengthInByte[nal] - 4ull; + m_header.SPS_data = std::vector(ptr, ptr + sz); + } + else if (nalu_bytes[4] == 0x68) // PPS + { + uint8_t* ptr = info.sLayerInfo[layer].pBsBuf + bs_size + 4; + size_t sz = info.sLayerInfo[layer].pNalLengthInByte[nal] - 4ull; + m_header.PPS_data = std::vector(ptr, ptr + sz); + } + else + { + uint32_t nalu_sz = info.sLayerInfo[layer].pNalLengthInByte[nal]; + uint8_t* data = info.sLayerInfo[layer].pBsBuf + bs_size; + *(uint32_t*)data = BinaryStream::htonx(nalu_sz - 4); + Frame::kType frame_type; + if (nalu_bytes[4] == 0x65) // 0x65 I-frame + frame_type = Frame::kType::IFrame; + else if (nalu_bytes[4] == 0x61) // 0x61 P-frame + frame_type = Frame::kType::PFrame; + else // something else, don't care + frame_type = Frame::kType::Unknown; + m_frames.emplace_back(); + m_frames.back().type = frame_type; + m_frames.back().data = std::vector(data, data + nalu_sz); + } + //printf("nalu %x\n", nalu_bytes[4]); + bs_size += info.sLayerInfo[layer].pNalLengthInByte[nal]; + } + } + + return true; +} + +bool MP4Encoder::write_mp4(const std::string& filename) const noexcept +{ + MP4FileHandle mp4 = MP4Create(filename.c_str()); + MP4SetTimeScale(mp4, 90000); + MP4SetVideoProfileLevel(mp4, 1); + MP4TrackId mp4_track = MP4AddH264VideoTrack(mp4, 90000, 90000 / m_framerate, m_width, m_height, + m_header.avc_profile, m_header.avc_profile_compat, m_header.avc_level, 3); + MP4AddH264SequenceParameterSet(mp4, mp4_track, m_header.SPS_data.data(), m_header.SPS_data.size()); + MP4AddH264PictureParameterSet(mp4, mp4_track, m_header.PPS_data.data(), m_header.PPS_data.size()); + for (const auto& f : m_frames) + { + if (!MP4WriteSample(mp4, mp4_track, f.data.data(), f.data.size(), + MP4_INVALID_DURATION, 0, f.type == Frame::kType::IFrame)) + { + LOG("failed to write mp4 sample"); + MP4Close(mp4); + return false; + } + } + MP4Close(mp4); + return true; +} + +void MP4Encoder::destroy() noexcept +{ + if (m_encoder) + { + m_encoder->Uninitialize(); + WelsDestroySVCEncoder(m_encoder); + } + m_encoder = nullptr; + m_frames.clear(); + m_header.PPS_data.clear(); + m_header.SPS_data.clear(); +} diff --git a/src/mp4enc.h b/src/mp4enc.h new file mode 100644 index 0000000..62f6f05 --- /dev/null +++ b/src/mp4enc.h @@ -0,0 +1,33 @@ +#pragma once +#include "image.h" + +class MP4Encoder +{ + struct Frame + { + enum class kType : uint8_t { IFrame, PFrame, Unknown }; + kType type; + std::vector data; + }; + struct Header + { + std::vector SPS_data; + std::vector PPS_data; + uint8_t avc_profile; + uint8_t avc_profile_compat; + uint8_t avc_level; + }; + class ISVCEncoder* m_encoder = nullptr; + int m_width = 0; + int m_height = 0; + int m_bitrate = 1000 << 10; + float m_framerate = 0; + Header m_header; + std::vector m_frames; + std::vector m_yuv_buffer; +public: + bool init(int width, int height, int fps, int bitrate) noexcept; + bool encode(const Image& rgba) noexcept; + bool write_mp4(const std::string& filename) const noexcept; + void destroy() noexcept; +}; diff --git a/src/pch.cpp b/src/pch.cpp index b863eb7..e3fe1a5 100644 --- a/src/pch.cpp +++ b/src/pch.cpp @@ -23,4 +23,5 @@ #pragma comment(lib, "openvr_api.lib") #pragma comment(lib, "openh264-2.0.0-win64.lib") #pragma comment(lib, "libmp4v2.lib") + #pragma comment(lib, "yuv.lib") #endif // _WIN32 diff --git a/src/version.cpp b/src/version.cpp index 067a6cb..d7a20f7 100644 --- a/src/version.cpp +++ b/src/version.cpp @@ -10,7 +10,7 @@ const int g_version_fix = PP_VERSION_FIX; const int g_version_build = PP_VERSION_BUILD; #ifdef _WIN32 -#include "windows.h" +#include const wchar_t* g_version_w = TEXT(PP_VERSION_STRING); const wchar_t* g_version_number_w = TEXT(PP_VERSION_NUMBER_STRING); const wchar_t* g_window_title_w = L"PanoPainter " TEXT(PP_VERSION_NUMBER_STRING);