omar/panopainter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add MP4Encoder class and test timelapse exporting

Browse Source
This commit is contained in:
omigamedev
2019-10-31 23:22:09 +01:00
parent 8e94d6b401
commit 9eecf60219
11 changed files with 283 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
PanoPainter.vcxproj
View File

@@ -77,8 +77,8 @@
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>libs\glm;libs\glew-2.0.0\include;libs\stb;libs\tinyxml2;libs\yoga;libs\curl-win\include;libs\jpeg;libs\wacom;libs\bugtrap-client\include;libs\poly2tri\poly2tri;libs\base64;libs\sqlite3;libs\openvr\headers;libs\nanort;libs\hash-library;libs\fmt\include;libs\glad\include;libs\openh264\include;libs\mp4v2\include;$(IncludePath)</IncludePath>
<LibraryPath>libs\curl-win\lib\dll-$(Configuration)-$(PlatformShortName);libs\glew-2.0.0\lib\Release\$(Platform);libs\bugtrap-client\lib;libs\openvr\lib\win64;libs\openh264\lib;libs\mp4v2\lib\win;$(LibraryPath)</LibraryPath>
<IncludePath>libs\glm;libs\glew-2.0.0\include;libs\stb;libs\tinyxml2;libs\yoga;libs\curl-win\include;libs\jpeg;libs\wacom;libs\bugtrap-client\include;libs\poly2tri\poly2tri;libs\base64;libs\sqlite3;libs\openvr\headers;libs\nanort;libs\hash-library;libs\fmt\include;libs\glad\include;libs\openh264\include;libs\mp4v2\include;libs\libyuv\include;$(IncludePath)</IncludePath>
<LibraryPath>libs\curl-win\lib\dll-$(Configuration)-$(PlatformShortName);libs\glew-2.0.0\lib\Release\$(Platform);libs\bugtrap-client\lib;libs\openvr\lib\win64;libs\openh264\lib;libs\mp4v2\lib\win;libs\libyuv\lib\win;$(LibraryPath)</LibraryPath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
@@ -87,8 +87,8 @@
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>libs\glm;libs\glew-2.0.0\include;libs\stb;libs\tinyxml2;libs\yoga;libs\curl-win\include;libs\jpeg;libs\wacom;libs\bugtrap-client\include;libs\poly2tri\poly2tri;libs\base64;libs\sqlite3;libs\openvr\headers;libs\nanort;libs\hash-library;libs\fmt\include;libs\glad\include;libs\openh264\include;libs\mp4v2\include;$(IncludePath)</IncludePath>
<LibraryPath>libs\curl-win\lib\dll-$(Configuration)-$(PlatformShortName);libs\glew-2.0.0\lib\Release\$(Platform);libs\bugtrap-client\lib;libs\openvr\lib\win64;libs\openh264\lib;libs\mp4v2\lib\win;$(LibraryPath)</LibraryPath>
<IncludePath>libs\glm;libs\glew-2.0.0\include;libs\stb;libs\tinyxml2;libs\yoga;libs\curl-win\include;libs\jpeg;libs\wacom;libs\bugtrap-client\include;libs\poly2tri\poly2tri;libs\base64;libs\sqlite3;libs\openvr\headers;libs\nanort;libs\hash-library;libs\fmt\include;libs\glad\include;libs\openh264\include;libs\mp4v2\include;libs\libyuv\include;$(IncludePath)</IncludePath>
<LibraryPath>libs\curl-win\lib\dll-$(Configuration)-$(PlatformShortName);libs\glew-2.0.0\lib\Release\$(Platform);libs\bugtrap-client\lib;libs\openvr\lib\win64;libs\openh264\lib;libs\mp4v2\lib\win;libs\libyuv\lib\win;$(LibraryPath)</LibraryPath>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
@@ -346,6 +346,7 @@
<ClCompile Include="src\layout.cpp" />
<ClCompile Include="src\log.cpp" />
<ClCompile Include="src\main.cpp" />
<ClCompile Include="src\mp4enc.cpp" />
<ClCompile Include="src\node.cpp" />
<ClCompile Include="src\node_about.cpp" />
<ClCompile Include="src\node_border.cpp" />
@@ -475,6 +476,7 @@
<ClInclude Include="src\keymap.h" />
<ClInclude Include="src\layout.h" />
<ClInclude Include="src\log.h" />
<ClInclude Include="src\mp4enc.h" />
<ClInclude Include="src\node.h" />
<ClInclude Include="src\node_about.h" />
<ClInclude Include="src\node_border.h" />

6
PanoPainter.vcxproj.filters
View File

@@ -381,6 +381,9 @@
<ClCompile Include="libs\yoga\yoga\event\event.cpp">
<Filter>libs\yoga</Filter>
</ClCompile>
<ClCompile Include="src\mp4enc.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="libs\jpeg\jpgd.h">
@@ -632,6 +635,9 @@
<ClInclude Include="src\node_panel_animation.h">
<Filter>Source Files\ui</Filter>
</ClInclude>
<ClInclude Include="src\mp4enc.h">
<Filter>Header Files</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="PanoPainter.rc">

47
src/app.cpp
View File

@@ -4,6 +4,7 @@
#include "node_icon.h"
#include "node_dialog_open.h"
#include "node_progress_bar.h"
#include "mp4enc.h"
#ifdef __APPLE__
#include <Foundation/Foundation.h>
@@ -616,14 +617,15 @@ void App::draw(float dt)
{
static float rec_timer = 0.f;
rec_timer += dt;
if (rec_timer > 1.f && canvas->m_canvas->m_dirty_stroke)
if (rec_timer > 0.3f && canvas->m_canvas->m_dirty_stroke)
{
canvas->m_canvas->m_dirty_stroke = false;
LOG("rec tick");
rec_timer = 0.f;
auto data = new uint8_t[(int)width * (int)height * 4];
auto img = std::make_unique<Image>();
img->create(width, height);
#if __IOS__
[ios_view->glview bindDrawable];
#else
@@ -636,10 +638,10 @@ void App::draw(float dt)
if (dfbo != rfbo)
LOG("DIFFERENT FB");
glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, data);
glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, img->m_data.get());
{
std::lock_guard<std::mutex> lock(rec_mutex);
rec_frames.emplace_back(data);
rec_frames.emplace_back(std::move(img));
rec_cv.notify_all();
}
update_rec_frames();
@@ -805,11 +807,14 @@ void App::rec_export(std::string path)
pb->m_progress->SetWidthP(0);
pb->m_title->set_text("Exporting MP4 movie");
/*
#if defined(__IOS__) || defined(__OSX__)
export_mp4(rec_path, width, height, rec_count, ^(float) {
pb->m_progress->SetWidthP((float)progress / tot * 100.f);
});
#endif
*/
m_encoder->write_mp4(data_path + "/export.mp4");
pb->destroy();
}
@@ -818,8 +823,14 @@ void App::rec_loop()
{
BT_SetTerminate();
rec_running = true;
if (!m_encoder)
{
m_encoder = std::make_unique<MP4Encoder>();
m_encoder->init(512, 512, 30, 500 << 10);
}
while(rec_running)
{
std::unique_ptr<Image> frame;
std::unique_lock<std::mutex> lock(rec_mutex);
rec_cv.wait(lock);
if (!rec_running)
@@ -828,32 +839,14 @@ void App::rec_loop()
{
if (rec_frames.front())
{
auto inverted = std::make_unique<uint8_t[]>(width*height*4);
for (int y = height - 1, y1 = 0; y >= 0; y--, y1++)
{
uint8_t* dst = &inverted[y * width * 4];
uint8_t* src = &rec_frames.front()[y1 * width * 4];
std::copy_n(src, (int)width * 4, dst);
}
char path[256];
snprintf(path, sizeof(path), "%s/%04d.jpg", rec_path.c_str(), rec_count);
LOG("writing %s", path);
jpge::params params;
params.m_quality = 75;
bool saved = jpge::compress_image_to_jpeg_file(path, width, height, 4, inverted.get(), params);
if (!saved)
{
LOG("error writing the frame");
rec_running = false;
}
else
{
rec_count++;
redraw = true;
}
rec_count++;
frame = std::move(rec_frames.front());
}
rec_frames.pop_front();
}
lock.unlock();
if (frame)
m_encoder->encode(*frame);
}
}

14
src/app.h
View File

@@ -7,6 +7,8 @@
#include "layout.h"
#include "font.h"
#include "rtt.h"
#include "image.h"
#include "mp4enc.h"
#include "node_message_box.h"
#include "node_settings.h"
#include "node_popup_menu.h"
@@ -18,6 +20,10 @@
#include "node_canvas.h"
#include "node_dialog_layer_rename.h"
#include "node_progress_bar.h"
#include "node_panel_grid.h"
#include "node_panel_quick.h"
#include "node_input_box.h"
#include "node_panel_animation.h"
#if defined(__OBJC__) && defined(__IOS__)
#import <Foundation/Foundation.h>
@@ -37,11 +43,6 @@
#include <GLFW/glfw3.h>
#endif
#include "node_panel_grid.h"
#include "node_panel_quick.h"
#include "node_input_box.h"
#include "node_panel_animation.h"
struct VRController
{
enum class kButton : uint8_t
@@ -94,7 +95,8 @@ public:
int rec_count = 0;
std::mutex rec_mutex;
std::condition_variable rec_cv;
std::deque<std::unique_ptr<uint8_t[]>> rec_frames;
std::deque<std::unique_ptr<Image>> rec_frames;
std::unique_ptr<MP4Encoder> m_encoder;
RTT uirtt;
Sampler sampler;

5
src/app_dialogs.cpp
View File

@@ -786,10 +786,7 @@ void App::dialog_export_mp4()
{
int nalu_sz = info.sLayerInfo[layer].pNalLengthInByte[nal];
uint8_t* data = info.sLayerInfo[layer].pBsBuf + bs_size;
data[0] = (nalu_sz - 4) >> 24;
data[1] = (nalu_sz - 4) >> 16;
data[2] = (nalu_sz - 4) >> 8;
data[3] = (nalu_sz - 4) & 0xff;
*(uint32_t*)data = BinaryStream::htonx(nalu_sz - 4);
bool sync = false;
if (nalu_bytes[4] == 0x65) // I-frame
sync = true;

8
src/app_layout.cpp
View File

@@ -729,20 +729,20 @@ void App::init_menu_tools()
popup_time->find<NodeButtonCustom>("timelapse-start")->on_click = [this, popup_time, popup_exp](Node*) {
rec_running ? rec_stop() : rec_start();
popup_time->destroy();
popup_exp->destroy();
popup_time->destroy();
};
popup_time->find<NodeButtonCustom>("timelapse-clear")->on_click = [this, popup_time, popup_exp](Node*) {
rec_clear();
popup_time->destroy();
popup_exp->destroy();
popup_time->destroy();
};
popup_time->find<NodeButtonCustom>("timelapse-export")->on_click = [this, popup_time, popup_exp](Node*) {
popup_time->destroy();
popup_exp->destroy();
rec_export("");
popup_exp->destroy();
popup_time->destroy();
};
}
};

12
src/binary_stream.h
View File

@@ -15,7 +15,17 @@ public:
return bint.c[0] == 1 ? ByteOrder::BigEndian : ByteOrder::LittleEndian;
}
template<typename T> T swap(T x)
template<typename T> static T htonx(T x)
{
static auto sys = sys_order();
return sys == ByteOrder::BigEndian ? x : swap(x);
}
template<typename T> static T ntohx(T x)
{
static auto sys = sys_order();
return sys == ByteOrder::BigEndian ? x : swap(x);
}
template<typename T> static T swap(T x)
{
#if _MSC_VER >= 1400

192
src/mp4enc.cpp Normal file
View File

@@ -0,0 +1,192 @@
#include "pch.h"
#include "mp4enc.h"
#include <codec_api.h>
#include <libyuv.h>
#define MP4V2_NO_STDINT_DEFS
#include <mp4v2/mp4v2.h>
static void encoder_trace_callback(void* context, int level, const char* message)
{
LOG("ENCODER: %s", message);
}
bool MP4Encoder::init(int width, int height, int fps, int bitrate) noexcept
{
if (WelsCreateSVCEncoder(&m_encoder) != 0)
return false;
m_width = width;
m_height = height;
m_framerate = fps;
m_bitrate = bitrate;
//Encoder params
SEncParamExt param;
m_encoder->GetDefaultParams(&param);
param.iUsageType = CAMERA_VIDEO_REAL_TIME;
param.fMaxFrameRate = m_framerate;
param.iLtrMarkPeriod = 75;
param.iPicWidth = m_width;
param.iPicHeight = m_height;
param.iTargetBitrate = m_bitrate;
param.bEnableDenoise = false;
param.iSpatialLayerNum = 1;
param.bUseLoadBalancing = false;
param.bEnableSceneChangeDetect = false;
param.bEnableBackgroundDetection = false;
param.bEnableAdaptiveQuant = false;
param.bEnableFrameSkip = false;
param.iMultipleThreadIdc = 0;
//param.uiIntraPeriod = 10;
for (int i = 0; i < param.iSpatialLayerNum; i++)
{
param.sSpatialLayers[i].iVideoWidth = m_width >> (param.iSpatialLayerNum - 1 - i);
param.sSpatialLayers[i].iVideoHeight = m_height >> (param.iSpatialLayerNum - 1 - i);
param.sSpatialLayers[i].fFrameRate = m_framerate;
param.sSpatialLayers[i].iSpatialBitrate = m_bitrate;
param.sSpatialLayers[i].uiProfileIdc = PRO_BASELINE;
param.sSpatialLayers[i].uiLevelIdc = LEVEL_4_2;
param.sSpatialLayers[i].iDLayerQp = 42;
//SSliceArgument sliceArg;
//sliceArg.uiSliceMode = SM_FIXEDSLCNUM_SLICE;
//sliceArg.uiSliceNum = 0;
//param.sSpatialLayers[i].sSliceArgument = sliceArg;
}
param.uiMaxNalSize = 1500;
param.iTargetBitrate *= param.iSpatialLayerNum;
m_encoder->InitializeExt(&param);
int trace_level = WELS_LOG_ERROR;
m_encoder->SetOption(ENCODER_OPTION_TRACE_LEVEL, &trace_level);
m_encoder->SetOption(ENCODER_OPTION_TRACE_CALLBACK, encoder_trace_callback);
m_encoder->SetOption(ENCODER_OPTION_TRACE_CALLBACK_CONTEXT, this);
int videoFormat = videoFormatI420;
m_encoder->SetOption(ENCODER_OPTION_DATAFORMAT, &videoFormat);
}
bool MP4Encoder::encode(const Image& rgba) noexcept
{
// lazy allocation of YUV buffer
if (m_yuv_buffer.empty())
m_yuv_buffer.resize(m_width * m_height * 3 / 2);
SFrameBSInfo info = { 0 };
SSourcePicture pic = { 0 };
pic.iPicWidth = m_width;
pic.iPicHeight = m_height;
pic.iColorFormat = videoFormatI420;
pic.iStride[0] = pic.iPicWidth;
pic.iStride[1] = pic.iStride[2] = pic.iPicWidth >> 1;
pic.pData[0] = m_yuv_buffer.data();
pic.pData[1] = pic.pData[0] + ((size_t)m_width * m_height);
pic.pData[2] = pic.pData[1] + ((size_t)m_width * m_height / 4);
if (rgba.width != m_width || rgba.height != m_height)
{
Image resized = rgba.resize(m_width, m_height);
libyuv::ABGRToI420(resized.data(), resized.width * 4, pic.pData[0], m_width,
pic.pData[1], m_width / 2, pic.pData[2], m_width / 2, m_width, m_height);
}
else
{
libyuv::ABGRToI420(rgba.data(), rgba.width * 4, pic.pData[0], m_width,
pic.pData[1], m_width / 2, pic.pData[2], m_width / 2, m_width, m_height);
}
if (m_encoder->EncodeFrame(&pic, &info))
{
LOG("EcncodeFrame failed");
return false;
}
// No frames generated at this time
if (info.eFrameType == videoFrameTypeSkip)
return false;
for (int layer = 0; layer < info.iLayerNum; layer++)
{
size_t bs_size = 0;
for (int nal = 0; nal < info.sLayerInfo[layer].iNalCount; nal++)
{
std::array<uint8_t, 5> nalu_bytes;
for (int i = 0; i < nalu_bytes.size(); i++)
nalu_bytes[i] = info.sLayerInfo[layer].pBsBuf[bs_size + i];
if (nalu_bytes[4] == 0x67) // SPS
{
uint8_t avc_profile = info.sLayerInfo[layer].pBsBuf[bs_size + 5];
uint8_t avc_profile_compat = info.sLayerInfo[layer].pBsBuf[bs_size + 6];
uint8_t avc_level = info.sLayerInfo[layer].pBsBuf[bs_size + 7];
uint8_t* ptr = info.sLayerInfo[layer].pBsBuf + bs_size + 4;
size_t sz = info.sLayerInfo[layer].pNalLengthInByte[nal] - 4ull;
m_header.SPS_data = std::vector<uint8_t>(ptr, ptr + sz);
}
else if (nalu_bytes[4] == 0x68) // PPS
{
uint8_t* ptr = info.sLayerInfo[layer].pBsBuf + bs_size + 4;
size_t sz = info.sLayerInfo[layer].pNalLengthInByte[nal] - 4ull;
m_header.PPS_data = std::vector<uint8_t>(ptr, ptr + sz);
}
else
{
uint32_t nalu_sz = info.sLayerInfo[layer].pNalLengthInByte[nal];
uint8_t* data = info.sLayerInfo[layer].pBsBuf + bs_size;
*(uint32_t*)data = BinaryStream::htonx(nalu_sz - 4);
Frame::kType frame_type;
if (nalu_bytes[4] == 0x65) // 0x65 I-frame
frame_type = Frame::kType::IFrame;
else if (nalu_bytes[4] == 0x61) // 0x61 P-frame
frame_type = Frame::kType::PFrame;
else // something else, don't care
frame_type = Frame::kType::Unknown;
m_frames.emplace_back();
m_frames.back().type = frame_type;
m_frames.back().data = std::vector<uint8_t>(data, data + nalu_sz);
}
//printf("nalu %x\n", nalu_bytes[4]);
bs_size += info.sLayerInfo[layer].pNalLengthInByte[nal];
}
}
return true;
}
bool MP4Encoder::write_mp4(const std::string& filename) const noexcept
{
MP4FileHandle mp4 = MP4Create(filename.c_str());
MP4SetTimeScale(mp4, 90000);
MP4SetVideoProfileLevel(mp4, 1);
MP4TrackId mp4_track = MP4AddH264VideoTrack(mp4, 90000, 90000 / m_framerate, m_width, m_height,
m_header.avc_profile, m_header.avc_profile_compat, m_header.avc_level, 3);
MP4AddH264SequenceParameterSet(mp4, mp4_track, m_header.SPS_data.data(), m_header.SPS_data.size());
MP4AddH264PictureParameterSet(mp4, mp4_track, m_header.PPS_data.data(), m_header.PPS_data.size());
for (const auto& f : m_frames)
{
if (!MP4WriteSample(mp4, mp4_track, f.data.data(), f.data.size(),
MP4_INVALID_DURATION, 0, f.type == Frame::kType::IFrame))
{
LOG("failed to write mp4 sample");
MP4Close(mp4);
return false;
}
}
MP4Close(mp4);
return true;
}
void MP4Encoder::destroy() noexcept
{
if (m_encoder)
{
m_encoder->Uninitialize();
WelsDestroySVCEncoder(m_encoder);
}
m_encoder = nullptr;
m_frames.clear();
m_header.PPS_data.clear();
m_header.SPS_data.clear();
}

33
src/mp4enc.h Normal file
View File

@@ -0,0 +1,33 @@
#pragma once
#include "image.h"
class MP4Encoder
{
struct Frame
{
enum class kType : uint8_t { IFrame, PFrame, Unknown };
kType type;
std::vector<uint8_t> data;
};
struct Header
{
std::vector<uint8_t> SPS_data;
std::vector<uint8_t> PPS_data;
uint8_t avc_profile;
uint8_t avc_profile_compat;
uint8_t avc_level;
};
class ISVCEncoder* m_encoder = nullptr;
int m_width = 0;
int m_height = 0;
int m_bitrate = 1000 << 10;
float m_framerate = 0;
Header m_header;
std::vector<Frame> m_frames;
std::vector<uint8_t> m_yuv_buffer;
public:
bool init(int width, int height, int fps, int bitrate) noexcept;
bool encode(const Image& rgba) noexcept;
bool write_mp4(const std::string& filename) const noexcept;
void destroy() noexcept;
};

1
src/pch.cpp
View File

@@ -23,4 +23,5 @@
#pragma comment(lib, "openvr_api.lib")
#pragma comment(lib, "openh264-2.0.0-win64.lib")
#pragma comment(lib, "libmp4v2.lib")
#pragma comment(lib, "yuv.lib")
#endif // _WIN32

2
src/version.cpp
View File

@@ -10,7 +10,7 @@ const int g_version_fix = PP_VERSION_FIX;
const int g_version_build = PP_VERSION_BUILD;
#ifdef _WIN32
#include "windows.h"
#include <windows.h>
const wchar_t* g_version_w = TEXT(PP_VERSION_STRING);
const wchar_t* g_version_number_w = TEXT(PP_VERSION_NUMBER_STRING);
const wchar_t* g_window_title_w = L"PanoPainter " TEXT(PP_VERSION_NUMBER_STRING);