OpenShot Library | libopenshot  0.3.2
FFmpegWriter.cpp
Go to the documentation of this file.
1 
12 // Copyright (c) 2008-2024 OpenShot Studios, LLC, Fabrice Bellard
13 //
14 // SPDX-License-Identifier: LGPL-3.0-or-later
15 
16 #include <algorithm>
17 #include <iostream>
18 #include <cmath>
19 #include <ctime>
20 #include <unistd.h>
21 
22 #include "FFmpegUtilities.h"
23 
24 #include "FFmpegWriter.h"
25 #include "Exceptions.h"
26 #include "Frame.h"
27 #include "OpenMPUtilities.h"
28 #include "Settings.h"
29 #include "ZmqLogger.h"
30 
31 using namespace openshot;
32 
33 // Multiplexer parameters temporary storage
34 AVDictionary *mux_dict = NULL;
35 
36 #if USE_HW_ACCEL
37 int hw_en_on = 1; // Is set in UI
38 int hw_en_supported = 0; // Is set by FFmpegWriter
39 AVPixelFormat hw_en_av_pix_fmt = AV_PIX_FMT_NONE;
40 AVHWDeviceType hw_en_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
41 static AVBufferRef *hw_device_ctx = NULL;
42 AVFrame *hw_frame = NULL;
43 
44 static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int64_t width, int64_t height)
45 {
46  AVBufferRef *hw_frames_ref;
47  AVHWFramesContext *frames_ctx = NULL;
48  int err = 0;
49 
50  if (!(hw_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx))) {
51  std::clog << "Failed to create HW frame context.\n";
52  return -1;
53  }
54  frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data);
55  frames_ctx->format = hw_en_av_pix_fmt;
56  frames_ctx->sw_format = AV_PIX_FMT_NV12;
57  frames_ctx->width = width;
58  frames_ctx->height = height;
59  frames_ctx->initial_pool_size = 20;
60  if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0) {
61  std::clog << "Failed to initialize HW frame context. " <<
62  "Error code: " << av_err2string(err) << "\n";
63  av_buffer_unref(&hw_frames_ref);
64  return err;
65  }
66  ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
67  if (!ctx->hw_frames_ctx)
68  err = AVERROR(ENOMEM);
69 
70  av_buffer_unref(&hw_frames_ref);
71  return err;
72 }
73 #endif // USE_HW_ACCEL
74 
75 FFmpegWriter::FFmpegWriter(const std::string& path) :
76  path(path), oc(NULL), audio_st(NULL), video_st(NULL), samples(NULL),
77  audio_outbuf(NULL), audio_outbuf_size(0), audio_input_frame_size(0), audio_input_position(0),
78  initial_audio_input_frame_size(0), img_convert_ctx(NULL), num_of_rescalers(1),
79  rescaler_position(0), video_codec_ctx(NULL), audio_codec_ctx(NULL), is_writing(false), video_timestamp(0), audio_timestamp(0),
80  original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false),
81  write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL) {
82 
83  // Disable audio & video (so they can be independently enabled)
84  info.has_audio = false;
85  info.has_video = false;
86 
87  // Initialize FFMpeg, and register all formats and codecs
88  AV_REGISTER_ALL
89 
90  // auto detect format
91  auto_detect_format();
92 }
93 
94 // Open the writer
95 void FFmpegWriter::Open() {
96  if (!is_open) {
97  // Open the writer
98  is_open = true;
99 
100  // Prepare streams (if needed)
101  if (!prepare_streams)
102  PrepareStreams();
103 
104  // Now that all the parameters are set, we can open the audio and video codecs and allocate the necessary encode buffers
105  if (info.has_video && video_st)
106  open_video(oc, video_st);
107  if (info.has_audio && audio_st)
108  open_audio(oc, audio_st);
109 
110  // Write header (if needed)
111  if (!write_header)
112  WriteHeader();
113  }
114 }
115 
116 // auto detect format (from path)
117 void FFmpegWriter::auto_detect_format() {
118 
119  // Allocate the output media context
120  AV_OUTPUT_CONTEXT(&oc, path.c_str());
121  if (!oc) {
122  throw OutOfMemory(
123  "Could not allocate memory for AVFormatContext.", path);
124  }
125 
126  // Determine what format to use when encoding this output filename
127  oc->oformat = av_guess_format(NULL, path.c_str(), NULL);
128  if (oc->oformat == nullptr) {
129  throw InvalidFormat(
130  "Could not deduce output format from file extension.", path);
131  }
132 
133  // Update video codec name
134  if (oc->oformat->video_codec != AV_CODEC_ID_NONE && info.has_video)
135  info.vcodec = avcodec_find_encoder(oc->oformat->video_codec)->name;
136 
137  // Update audio codec name
138  if (oc->oformat->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
139  info.acodec = avcodec_find_encoder(oc->oformat->audio_codec)->name;
140 }
141 
142 // initialize streams
143 void FFmpegWriter::initialize_streams() {
144  ZmqLogger::Instance()->AppendDebugMethod(
145  "FFmpegWriter::initialize_streams",
146  "oc->oformat->video_codec", oc->oformat->video_codec,
147  "oc->oformat->audio_codec", oc->oformat->audio_codec,
148  "AV_CODEC_ID_NONE", AV_CODEC_ID_NONE);
149 
150  // Add the audio and video streams using the default format codecs and initialize the codecs
151  video_st = NULL;
152  audio_st = NULL;
153  if (oc->oformat->video_codec != AV_CODEC_ID_NONE && info.has_video)
154  // Add video stream
155  video_st = add_video_stream();
156 
157  if (oc->oformat->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
158  // Add audio stream
159  audio_st = add_audio_stream();
160 }
161 
162 // Set video export options
163 void FFmpegWriter::SetVideoOptions(bool has_video, std::string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate) {
164  // Set the video options
165  if (codec.length() > 0) {
166  const AVCodec *new_codec;
167  // Check if the codec selected is a hardware accelerated codec
168 #if USE_HW_ACCEL
169 #if defined(__linux__)
170  if (strstr(codec.c_str(), "_vaapi") != NULL) {
171  new_codec = avcodec_find_encoder_by_name(codec.c_str());
172  hw_en_on = 1;
173  hw_en_supported = 1;
174  hw_en_av_pix_fmt = AV_PIX_FMT_VAAPI;
175  hw_en_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
176  } else if (strstr(codec.c_str(), "_nvenc") != NULL) {
177  new_codec = avcodec_find_encoder_by_name(codec.c_str());
178  hw_en_on = 1;
179  hw_en_supported = 1;
180  hw_en_av_pix_fmt = AV_PIX_FMT_CUDA;
181  hw_en_av_device_type = AV_HWDEVICE_TYPE_CUDA;
182  } else {
183  new_codec = avcodec_find_encoder_by_name(codec.c_str());
184  hw_en_on = 0;
185  hw_en_supported = 0;
186  }
187 #elif defined(_WIN32)
188  if (strstr(codec.c_str(), "_dxva2") != NULL) {
189  new_codec = avcodec_find_encoder_by_name(codec.c_str());
190  hw_en_on = 1;
191  hw_en_supported = 1;
192  hw_en_av_pix_fmt = AV_PIX_FMT_DXVA2_VLD;
193  hw_en_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
194  } else if (strstr(codec.c_str(), "_nvenc") != NULL) {
195  new_codec = avcodec_find_encoder_by_name(codec.c_str());
196  hw_en_on = 1;
197  hw_en_supported = 1;
198  hw_en_av_pix_fmt = AV_PIX_FMT_CUDA;
199  hw_en_av_device_type = AV_HWDEVICE_TYPE_CUDA;
200  } else {
201  new_codec = avcodec_find_encoder_by_name(codec.c_str());
202  hw_en_on = 0;
203  hw_en_supported = 0;
204  }
205 #elif defined(__APPLE__)
206  if (strstr(codec.c_str(), "_videotoolbox") != NULL) {
207  new_codec = avcodec_find_encoder_by_name(codec.c_str());
208  hw_en_on = 1;
209  hw_en_supported = 1;
210  hw_en_av_pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX;
211  hw_en_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
212  } else {
213  new_codec = avcodec_find_encoder_by_name(codec.c_str());
214  hw_en_on = 0;
215  hw_en_supported = 0;
216  }
217 #else // unknown OS
218  new_codec = avcodec_find_encoder_by_name(codec.c_str());
219 #endif //__linux__/_WIN32/__APPLE__
220 #else // USE_HW_ACCEL
221  new_codec = avcodec_find_encoder_by_name(codec.c_str());
222 #endif // USE_HW_ACCEL
223  if (new_codec == NULL)
224  throw InvalidCodec("A valid video codec could not be found for this file.", path);
225  else {
226  // Set video codec
227  info.vcodec = new_codec->name;
228  }
229  }
230  if (fps.num > 0) {
231  // Set frames per second (if provided)
232  info.fps.num = fps.num;
233  info.fps.den = fps.den;
234 
235  // Set the timebase (inverse of fps)
236  info.video_timebase.num = info.fps.den;
237  info.video_timebase.den = info.fps.num;
238  }
239  if (width >= 1)
240  info.width = width;
241  if (height >= 1)
242  info.height = height;
243  if (pixel_ratio.num > 0) {
244  info.pixel_ratio.num = pixel_ratio.num;
245  info.pixel_ratio.den = pixel_ratio.den;
246  }
247  if (bit_rate >= 1000) // bit_rate is the bitrate in b/s
248  info.video_bit_rate = bit_rate;
249  if ((bit_rate >= 0) && (bit_rate < 256)) // bit_rate is the bitrate in crf
250  info.video_bit_rate = bit_rate;
251 
252  info.interlaced_frame = interlaced;
253  info.top_field_first = top_field_first;
254 
255  // Calculate the DAR (display aspect ratio)
256  Fraction size(info.width * info.pixel_ratio.num, info.height * info.pixel_ratio.den);
257 
258  // Reduce size fraction
259  size.Reduce();
260 
261  // Set the ratio based on the reduced fraction
262  info.display_ratio.num = size.num;
263  info.display_ratio.den = size.den;
264 
265  ZmqLogger::Instance()->AppendDebugMethod(
266  "FFmpegWriter::SetVideoOptions (" + codec + ")",
267  "width", width, "height", height,
268  "size.num", size.num, "size.den", size.den,
269  "fps.num", fps.num, "fps.den", fps.den);
270 
271  // Enable / Disable video
272  info.has_video = has_video;
273 }
274 
275 // Set video export options (overloaded function)
276 void FFmpegWriter::SetVideoOptions(std::string codec, int width, int height, Fraction fps, int bit_rate) {
277  // Call full signature with some default parameters
278  FFmpegWriter::SetVideoOptions(
279  true, codec, fps, width, height,
280  openshot::Fraction(1, 1), false, true, bit_rate
281  );
282 }
283 
284 
285 // Set audio export options
286 void FFmpegWriter::SetAudioOptions(bool has_audio, std::string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate) {
287  // Set audio options
288  if (codec.length() > 0) {
289  const AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str());
290  if (new_codec == NULL)
291  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
292  else {
293  // Set audio codec
294  info.acodec = new_codec->name;
295  }
296  }
297  if (sample_rate > 7999)
298  info.sample_rate = sample_rate;
299  if (channels > 0)
300  info.channels = channels;
301  if (bit_rate > 999)
302  info.audio_bit_rate = bit_rate;
303  info.channel_layout = channel_layout;
304 
305  // init resample options (if zero)
306  if (original_sample_rate == 0)
307  original_sample_rate = info.sample_rate;
308  if (original_channels == 0)
309  original_channels = info.channels;
310 
311  ZmqLogger::Instance()->AppendDebugMethod(
312  "FFmpegWriter::SetAudioOptions (" + codec + ")",
313  "sample_rate", sample_rate,
314  "channels", channels,
315  "bit_rate", bit_rate);
316 
317  // Enable / Disable audio
318  info.has_audio = has_audio;
319 }
320 
321 
322 // Set audio export options (overloaded function)
323 void FFmpegWriter::SetAudioOptions(std::string codec, int sample_rate, int bit_rate) {
324  // Call full signature with some default parameters
325  FFmpegWriter::SetAudioOptions(
326  true, codec, sample_rate, 2,
327  openshot::LAYOUT_STEREO, bit_rate
328  );
329 }
330 
331 
332 // Set custom options (some codecs accept additional params)
333 void FFmpegWriter::SetOption(StreamType stream, std::string name, std::string value) {
334  // Declare codec context
335  AVCodecContext *c = NULL;
336  AVStream *st = NULL;
337  std::stringstream convert(value);
338 
339  if (info.has_video && stream == VIDEO_STREAM && video_st) {
340  st = video_st;
341  // Get codec context
342  c = AV_GET_CODEC_PAR_CONTEXT(st, video_codec_ctx);
343  // Was a codec / stream found?
344  if (c) {
345  if (info.interlaced_frame) {
346  c->field_order = info.top_field_first ? AV_FIELD_TT : AV_FIELD_BB;
347  // We only use these two version and ignore AV_FIELD_TB and AV_FIELD_BT
348  // Otherwise we would need to change the whole export window
349  }
350  }
351  } else if (info.has_audio && stream == AUDIO_STREAM && audio_st) {
352  st = audio_st;
353  // Get codec context
354  c = AV_GET_CODEC_PAR_CONTEXT(st, audio_codec_ctx);
355  } else
356  throw NoStreamsFound("The stream was not found. Be sure to call PrepareStreams() first.", path);
357 
358  // Init AVOption
359  const AVOption *option = NULL;
360 
361  // Was a codec / stream found?
362  if (c)
363  // Find AVOption (if it exists)
364  option = AV_OPTION_FIND(c->priv_data, name.c_str());
365 
366  // Was option found?
367  if (option || (name == "g" || name == "qmin" || name == "qmax" || name == "max_b_frames" || name == "mb_decision" ||
368  name == "level" || name == "profile" || name == "slices" || name == "rc_min_rate" || name == "rc_max_rate" ||
369  name == "rc_buffer_size" || name == "crf" || name == "cqp" || name == "qp")) {
370  // Check for specific named options
371  if (name == "g")
372  // Set gop_size
373  convert >> c->gop_size;
374 
375  else if (name == "qmin")
376  // Minimum quantizer
377  convert >> c->qmin;
378 
379  else if (name == "qmax")
380  // Maximum quantizer
381  convert >> c->qmax;
382 
383  else if (name == "max_b_frames")
384  // Maximum number of B-frames between non-B-frames
385  convert >> c->max_b_frames;
386 
387  else if (name == "mb_decision")
388  // Macroblock decision mode
389  convert >> c->mb_decision;
390 
391  else if (name == "level")
392  // Set codec level
393  convert >> c->level;
394 
395  else if (name == "profile")
396  // Set codec profile
397  convert >> c->profile;
398 
399  else if (name == "slices")
400  // Indicates number of picture subdivisions
401  convert >> c->slices;
402 
403  else if (name == "rc_min_rate")
404  // Minimum bitrate
405  convert >> c->rc_min_rate;
406 
407  else if (name == "rc_max_rate")
408  // Maximum bitrate
409  convert >> c->rc_max_rate;
410 
411  else if (name == "rc_buffer_size")
412  // Buffer size
413  convert >> c->rc_buffer_size;
414 
415  else if (name == "cqp") {
416  // encode quality and special settings like lossless
417  // This might be better in an extra methods as more options
418  // and way to set quality are possible
419 #if USE_HW_ACCEL
420  if (hw_en_on) {
421  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),63), 0); // 0-63
422  } else
423 #endif // USE_HW_ACCEL
424  {
425  switch (c->codec_id) {
426 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
427  // FFmpeg 4.0+
428  case AV_CODEC_ID_AV1 :
429  c->bit_rate = 0;
430  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),63), 0); // 0-63
431  break;
432 #endif
433  case AV_CODEC_ID_VP8 :
434  c->bit_rate = 10000000;
435  av_opt_set_int(c->priv_data, "qp", std::max(std::min(std::stoi(value), 63), 4), 0); // 4-63
436  break;
437  case AV_CODEC_ID_VP9 :
438  c->bit_rate = 0; // Must be zero!
439  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 63), 0); // 0-63
440  if (std::stoi(value) == 0) {
441  av_opt_set(c->priv_data, "preset", "veryslow", 0);
442  av_opt_set_int(c->priv_data, "lossless", 1, 0);
443  }
444  break;
445  case AV_CODEC_ID_H264 :
446  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 51), 0); // 0-51
447  if (std::stoi(value) == 0) {
448  av_opt_set(c->priv_data, "preset", "veryslow", 0);
449  c->pix_fmt = PIX_FMT_YUV444P; // no chroma subsampling
450  }
451  break;
452  case AV_CODEC_ID_HEVC :
453  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 51), 0); // 0-51
454  if (std::stoi(value) == 0) {
455  av_opt_set(c->priv_data, "preset", "veryslow", 0);
456  av_opt_set_int(c->priv_data, "lossless", 1, 0);
457  }
458  break;
459  default:
460  // For all other codecs assume a range of 0-63
461  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 63), 0); // 0-63
462  c->bit_rate = 0;
463  }
464  }
465  } else if (name == "crf") {
466  // encode quality and special settings like lossless
467  // This might be better in an extra methods as more options
468  // and way to set quality are possible
469 #if USE_HW_ACCEL
470  if (hw_en_on) {
471  double mbs = 15000000.0;
472  if (info.video_bit_rate > 0) {
473  if (info.video_bit_rate > 42) {
474  mbs = 380000.0;
475  }
476  else {
477  mbs *= std::pow(0.912,info.video_bit_rate);
478  }
479  }
480  c->bit_rate = (int)(mbs);
481  } else
482 #endif // USE_HW_ACCEL
483  {
484  switch (c->codec_id) {
485 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
486  // FFmpeg 4.0+
487  case AV_CODEC_ID_AV1 :
488  c->bit_rate = 0;
489  // AV1 only supports "crf" quality values
490  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value),63), 0);
491  break;
492 #endif
493  case AV_CODEC_ID_VP8 :
494  c->bit_rate = 10000000;
495  av_opt_set_int(c->priv_data, "crf", std::max(std::min(std::stoi(value), 63), 4), 0); // 4-63
496  break;
497  case AV_CODEC_ID_VP9 :
498  c->bit_rate = 0; // Must be zero!
499  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value), 63), 0); // 0-63
500  if (std::stoi(value) == 0) {
501  av_opt_set(c->priv_data, "preset", "veryslow", 0);
502  av_opt_set_int(c->priv_data, "lossless", 1, 0);
503  }
504  break;
505  case AV_CODEC_ID_H264 :
506  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value), 51), 0); // 0-51
507  if (std::stoi(value) == 0) {
508  av_opt_set(c->priv_data, "preset", "veryslow", 0);
509  c->pix_fmt = PIX_FMT_YUV444P; // no chroma subsampling
510  }
511  break;
512  case AV_CODEC_ID_HEVC :
513  if (strstr(info.vcodec.c_str(), "svt_hevc") != NULL) {
514  av_opt_set_int(c->priv_data, "preset", 7, 0);
515  av_opt_set_int(c->priv_data, "forced-idr",1,0);
516  av_opt_set_int(c->priv_data, "qp",std::min(std::stoi(value), 51),0);
517  }
518  else {
519  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value), 51), 0); // 0-51
520  }
521  if (std::stoi(value) == 0) {
522  av_opt_set(c->priv_data, "preset", "veryslow", 0);
523  av_opt_set_int(c->priv_data, "lossless", 1, 0);
524  }
525  break;
526  default:
527  // If this codec doesn't support crf calculate a bitrate
528  // TODO: find better formula
529  double mbs = 15000000.0;
530  if (info.video_bit_rate > 0) {
531  if (info.video_bit_rate > 42) {
532  mbs = 380000.0;
533  } else {
534  mbs *= std::pow(0.912, info.video_bit_rate);
535  }
536  }
537  c->bit_rate = (int) (mbs);
538  }
539  }
540  } else if (name == "qp") {
541  // encode quality and special settings like lossless
542  // This might be better in an extra methods as more options
543  // and way to set quality are possible
544 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
545  // FFmpeg 4.0+
546  switch (c->codec_id) {
547  case AV_CODEC_ID_AV1 :
548  c->bit_rate = 0;
549  if (strstr(info.vcodec.c_str(), "svtav1") != NULL) {
550  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),63), 0);
551  }
552  else if (strstr(info.vcodec.c_str(), "rav1e") != NULL) {
553  // Set number of tiles to a fixed value
554  // TODO Let user choose number of tiles
555  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),255), 0);
556  }
557  else if (strstr(info.vcodec.c_str(), "aom") != NULL) {
558  // Set number of tiles to a fixed value
559  // TODO Let user choose number of tiles
560  // libaom doesn't have qp only crf
561  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value),63), 0);
562  }
563  else {
564  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value),63), 0);
565  }
566  case AV_CODEC_ID_HEVC :
567  c->bit_rate = 0;
568  if (strstr(info.vcodec.c_str(), "svt_hevc") != NULL) {
569  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),51), 0);
570  av_opt_set_int(c->priv_data, "preset", 7, 0);
571  av_opt_set_int(c->priv_data, "forced-idr",1,0);
572  }
573  break;
574  }
575 #endif // FFmpeg 4.0+
576  } else {
577  // Set AVOption
578  AV_OPTION_SET(st, c->priv_data, name.c_str(), value.c_str(), c);
579  }
580 
581  ZmqLogger::Instance()->AppendDebugMethod(
582  "FFmpegWriter::SetOption (" + (std::string)name + ")",
583  "stream == VIDEO_STREAM", stream == VIDEO_STREAM);
584 
585  // Muxing dictionary is not part of the codec context.
586  // Just reusing SetOption function to set popular multiplexing presets.
587  } else if (name == "muxing_preset") {
588  if (value == "mp4_faststart") {
589  // 'moov' box to the beginning; only for MOV, MP4
590  av_dict_set(&mux_dict, "movflags", "faststart", 0);
591  } else if (value == "mp4_fragmented") {
592  // write selfcontained fragmented file, minimum length of the fragment 8 sec; only for MOV, MP4
593  av_dict_set(&mux_dict, "movflags", "frag_keyframe", 0);
594  av_dict_set(&mux_dict, "min_frag_duration", "8000000", 0);
595  }
596  } else {
597  throw InvalidOptions("The option is not valid for this codec.", path);
598  }
599 
600 }
601 
603 bool FFmpegWriter::IsValidCodec(std::string codec_name) {
604  // Initialize FFMpeg, and register all formats and codecs
605  AV_REGISTER_ALL
606 
607  // Find the codec (if any)
608  if (avcodec_find_encoder_by_name(codec_name.c_str()) == NULL)
609  return false;
610  else
611  return true;
612 }
613 
614 // Prepare & initialize streams and open codecs
615 void FFmpegWriter::PrepareStreams() {
616  if (!info.has_audio && !info.has_video)
617  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
618 
619  ZmqLogger::Instance()->AppendDebugMethod(
620  "FFmpegWriter::PrepareStreams [" + path + "]",
621  "info.has_audio", info.has_audio,
622  "info.has_video", info.has_video);
623 
624  // Initialize the streams (i.e. add the streams)
625  initialize_streams();
626 
627  // Mark as 'prepared'
628  prepare_streams = true;
629 }
630 
631 // Write the file header (after the options are set)
632 void FFmpegWriter::WriteHeader() {
633  if (!info.has_audio && !info.has_video)
634  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
635 
636  // Open the output file, if needed
637  if (!(oc->oformat->flags & AVFMT_NOFILE)) {
638  if (avio_open(&oc->pb, path.c_str(), AVIO_FLAG_WRITE) < 0)
639  throw InvalidFile("Could not open or write file.", path);
640  }
641 
642  // Force the output filename (which doesn't always happen for some reason)
643  AV_SET_FILENAME(oc, path.c_str());
644 
645  // Add general metadata (if any)
646  for (std::map<std::string, std::string>::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) {
647  av_dict_set(&oc->metadata, iter->first.c_str(), iter->second.c_str(), 0);
648  }
649 
650  // Set multiplexing parameters
651  AVDictionary *dict = NULL;
652 
653  bool is_mp4 = strcmp(oc->oformat->name, "mp4");
654  bool is_mov = strcmp(oc->oformat->name, "mov");
655  // Set dictionary preset only for MP4 and MOV files
656  if (is_mp4 || is_mov)
657  av_dict_copy(&dict, mux_dict, 0);
658 
659  // Write the stream header
660  if (avformat_write_header(oc, &dict) != 0) {
661  ZmqLogger::Instance()->AppendDebugMethod(
662  "FFmpegWriter::WriteHeader (avformat_write_header)");
663  throw InvalidFile("Could not write header to file.", path);
664  };
665 
666  // Free multiplexing dictionaries sets
667  if (dict) av_dict_free(&dict);
668  if (mux_dict) av_dict_free(&mux_dict);
669 
670  // Mark as 'written'
671  write_header = true;
672 
673  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteHeader");
674 }
675 
676 // Add a frame to the queue waiting to be encoded.
677 void FFmpegWriter::WriteFrame(std::shared_ptr<openshot::Frame> frame) {
678  // Check for open reader (or throw exception)
679  if (!is_open)
680  throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path);
681 
682  ZmqLogger::Instance()->AppendDebugMethod(
683  "FFmpegWriter::WriteFrame",
684  "frame->number", frame->number,
685  "is_writing", is_writing);
686 
687  // Write frames to video file
688  write_frame(frame);
689 
690  // Keep track of the last frame added
691  last_frame = frame;
692 }
693 
694 // Write all frames in the queue to the video file.
695 void FFmpegWriter::write_frame(std::shared_ptr<Frame> frame) {
696  // Flip writing flag
697  is_writing = true;
698 
699  // Create blank exception
700  bool has_error_encoding_video = false;
701 
702  // Process audio frame
703  if (info.has_audio && audio_st)
704  write_audio_packets(false, frame);
705 
706  // Process video frame
707  if (info.has_video && video_st)
708  process_video_packet(frame);
709 
710  if (info.has_video && video_st) {
711  // Does this frame's AVFrame still exist
712  if (av_frames.count(frame)) {
713  // Get AVFrame
714  AVFrame *frame_final = av_frames[frame];
715 
716  // Write frame to video file
717  if (!write_video_packet(frame, frame_final)) {
718  has_error_encoding_video = true;
719  }
720 
721  // Deallocate buffer and AVFrame
722  av_freep(&(frame_final->data[0]));
723  AV_FREE_FRAME(&frame_final);
724  av_frames.erase(frame);
725  }
726  }
727 
728  // Done writing
729  is_writing = false;
730 
731  // Raise exception from main thread
732  if (has_error_encoding_video)
733  throw ErrorEncodingVideo("Error while writing raw video frame", -1);
734 }
735 
736 // Write a block of frames from a reader
737 void FFmpegWriter::WriteFrame(ReaderBase *reader, int64_t start, int64_t length) {
738  ZmqLogger::Instance()->AppendDebugMethod(
739  "FFmpegWriter::WriteFrame (from Reader)",
740  "start", start,
741  "length", length);
742 
743  // Loop through each frame (and encoded it)
744  for (int64_t number = start; number <= length; number++) {
745  // Get the frame
746  std::shared_ptr<Frame> f = reader->GetFrame(number);
747 
748  // Encode frame
749  WriteFrame(f);
750  }
751 }
752 
753 // Write the file trailer (after all frames are written)
754 void FFmpegWriter::WriteTrailer() {
755  // Process final audio frame (if any)
756  if (info.has_audio && audio_st)
757  write_audio_packets(true, NULL);
758 
759  // Flush encoders (who sometimes hold on to frames)
760  flush_encoders();
761 
762  /* write the trailer, if any. The trailer must be written
763  * before you close the CodecContexts open when you wrote the
764  * header; otherwise write_trailer may try to use memory that
765  * was freed on av_codec_close() */
766  av_write_trailer(oc);
767 
768  // Mark as 'written'
769  write_trailer = true;
770 
771  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteTrailer");
772 }
773 
774 // Flush encoders
775 void FFmpegWriter::flush_encoders() {
776  if (info.has_audio && audio_codec_ctx && AV_GET_CODEC_TYPE(audio_st) == AVMEDIA_TYPE_AUDIO && AV_GET_CODEC_ATTRIBUTES(audio_st, audio_codec_ctx)->frame_size <= 1)
777  return;
778 #if (LIBAVFORMAT_VERSION_MAJOR < 58)
779  // FFmpeg < 4.0
780  if (info.has_video && video_codec_ctx && AV_GET_CODEC_TYPE(video_st) == AVMEDIA_TYPE_VIDEO && (oc->oformat->flags & AVFMT_RAWPICTURE) && AV_FIND_DECODER_CODEC_ID(video_st) == AV_CODEC_ID_RAWVIDEO)
781  return;
782 #else
783  if (info.has_video && video_codec_ctx && AV_GET_CODEC_TYPE(video_st) == AVMEDIA_TYPE_VIDEO && AV_FIND_DECODER_CODEC_ID(video_st) == AV_CODEC_ID_RAWVIDEO)
784  return;
785 #endif
786 
787  // FLUSH VIDEO ENCODER
788  if (info.has_video) {
789  for (;;) {
790 
791  // Increment PTS (in frames and scaled to the codec's timebase)
792  video_timestamp += av_rescale_q(1, av_make_q(info.fps.den, info.fps.num), video_codec_ctx->time_base);
793 
794 #if IS_FFMPEG_3_2
795  AVPacket* pkt = av_packet_alloc();
796 #else
797  AVPacket* pkt;
798  av_init_packet(pkt);
799 #endif
800  pkt->data = NULL;
801  pkt->size = 0;
802 
803  /* encode the image */
804  int got_packet = 0;
805  int error_code = 0;
806 
807 #if IS_FFMPEG_3_2
808  // Encode video packet (latest version of FFmpeg)
809  error_code = avcodec_send_frame(video_codec_ctx, NULL);
810  got_packet = 0;
811  while (error_code >= 0) {
812  error_code = avcodec_receive_packet(video_codec_ctx, pkt);
813  if (error_code == AVERROR(EAGAIN)|| error_code == AVERROR_EOF) {
814  got_packet = 0;
815  // Write packet
816  avcodec_flush_buffers(video_codec_ctx);
817  break;
818  }
819  av_packet_rescale_ts(pkt, video_codec_ctx->time_base, video_st->time_base);
820  pkt->stream_index = video_st->index;
821  error_code = av_interleaved_write_frame(oc, pkt);
822  }
823 #else // IS_FFMPEG_3_2
824 
825  // Encode video packet (older than FFmpeg 3.2)
826  error_code = avcodec_encode_video2(video_codec_ctx, pkt, NULL, &got_packet);
827 
828 #endif // IS_FFMPEG_3_2
829 
830  if (error_code < 0) {
831  ZmqLogger::Instance()->AppendDebugMethod(
832  "FFmpegWriter::flush_encoders ERROR ["
833  + av_err2string(error_code) + "]",
834  "error_code", error_code);
835  }
836  if (!got_packet) {
837  break;
838  }
839 
840  // set the timestamp
841  av_packet_rescale_ts(pkt, video_codec_ctx->time_base, video_st->time_base);
842  pkt->stream_index = video_st->index;
843 
844  // Write packet
845  error_code = av_interleaved_write_frame(oc, pkt);
846  if (error_code < 0) {
847  ZmqLogger::Instance()->AppendDebugMethod(
848  "FFmpegWriter::flush_encoders ERROR ["
849  + av_err2string(error_code) + "]",
850  "error_code", error_code);
851  }
852  }
853  }
854 
855  // FLUSH AUDIO ENCODER
856  if (info.has_audio) {
857  for (;;) {
858 #if IS_FFMPEG_3_2
859  AVPacket* pkt = av_packet_alloc();
860 #else
861  AVPacket* pkt;
862  av_init_packet(pkt);
863 #endif
864  pkt->data = NULL;
865  pkt->size = 0;
866  pkt->pts = pkt->dts = audio_timestamp;
867 
868  /* encode the image */
869  int error_code = 0;
870  int got_packet = 0;
871 #if IS_FFMPEG_3_2
872  error_code = avcodec_send_frame(audio_codec_ctx, NULL);
873 #else
874  error_code = avcodec_encode_audio2(audio_codec_ctx, pkt, NULL, &got_packet);
875 #endif
876  if (error_code < 0) {
877  ZmqLogger::Instance()->AppendDebugMethod(
878  "FFmpegWriter::flush_encoders ERROR ["
879  + av_err2string(error_code) + "]",
880  "error_code", error_code);
881  }
882  if (!got_packet) {
883  break;
884  }
885 
886  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
887  // but it fixes lots of PTS related issues when I do this.
888  pkt->pts = pkt->dts = audio_timestamp;
889 
890  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
891  av_packet_rescale_ts(pkt, audio_codec_ctx->time_base, audio_st->time_base);
892 
893  // set stream
894  pkt->stream_index = audio_st->index;
895  pkt->flags |= AV_PKT_FLAG_KEY;
896 
897  // Write packet
898  error_code = av_interleaved_write_frame(oc, pkt);
899  if (error_code < 0) {
900  ZmqLogger::Instance()->AppendDebugMethod(
901  "FFmpegWriter::flush_encoders ERROR ["
902  + av_err2string(error_code) + "]",
903  "error_code", error_code);
904  }
905 
906  // Increment PTS by duration of packet
907  audio_timestamp += pkt->duration;
908 
909  // deallocate memory for packet
910  AV_FREE_PACKET(pkt);
911  }
912  }
913 
914 }
915 
916 // Close the video codec
917 void FFmpegWriter::close_video(AVFormatContext *oc, AVStream *st)
918 {
919 #if USE_HW_ACCEL
920  if (hw_en_on && hw_en_supported) {
921  if (hw_device_ctx) {
922  av_buffer_unref(&hw_device_ctx);
923  hw_device_ctx = NULL;
924  }
925  }
926 #endif // USE_HW_ACCEL
927 
928  // Free any previous memory allocations
929  if (video_codec_ctx != nullptr) {
930  AV_FREE_CONTEXT(video_codec_ctx);
931  av_free(video_codec_ctx);
932  }
933 }
934 
935 // Close the audio codec
936 void FFmpegWriter::close_audio(AVFormatContext *oc, AVStream *st)
937 {
938  // Clear buffers
939  delete[] samples;
940  delete[] audio_outbuf;
941  delete[] audio_encoder_buffer;
942  samples = NULL;
943  audio_outbuf = NULL;
944  audio_encoder_buffer = NULL;
945 
946  // Deallocate resample buffer
947  if (avr) {
948  SWR_CLOSE(avr);
949  SWR_FREE(&avr);
950  avr = NULL;
951  }
952 
953  if (avr_planar) {
954  SWR_CLOSE(avr_planar);
955  SWR_FREE(&avr_planar);
956  avr_planar = NULL;
957  }
958 
959  // Free any previous memory allocations
960  if (audio_codec_ctx != nullptr) {
961  AV_FREE_CONTEXT(audio_codec_ctx);
962  av_free(audio_codec_ctx);
963  }
964 }
965 
966 // Close the writer
967 void FFmpegWriter::Close() {
968  // Write trailer (if needed)
969  if (!write_trailer)
970  WriteTrailer();
971 
972  // Close each codec
973  if (video_st)
974  close_video(oc, video_st);
975  if (audio_st)
976  close_audio(oc, audio_st);
977 
978  // Deallocate image scalers
979  if (image_rescalers.size() > 0)
980  RemoveScalers();
981 
982  if (!(oc->oformat->flags & AVFMT_NOFILE)) {
983  /* close the output file */
984  avio_close(oc->pb);
985  }
986 
987  // Reset frame counters
988  video_timestamp = 0;
989  audio_timestamp = 0;
990 
991  // Free the context which frees the streams too
992  avformat_free_context(oc);
993  oc = NULL;
994 
995  // Close writer
996  is_open = false;
997  prepare_streams = false;
998  write_header = false;
999  write_trailer = false;
1000 
1001  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::Close");
1002 }
1003 
1004 // Add an AVFrame to the cache
1005 void FFmpegWriter::add_avframe(std::shared_ptr<Frame> frame, AVFrame *av_frame) {
1006  // Add AVFrame to map (if it does not already exist)
1007  if (!av_frames.count(frame)) {
1008  // Add av_frame
1009  av_frames[frame] = av_frame;
1010  } else {
1011  // Do not add, and deallocate this AVFrame
1012  AV_FREE_FRAME(&av_frame);
1013  }
1014 }
1015 
1016 // Add an audio output stream
1017 AVStream *FFmpegWriter::add_audio_stream() {
1018  // Find the audio codec
1019  const AVCodec *codec = avcodec_find_encoder_by_name(info.acodec.c_str());
1020  if (codec == NULL)
1021  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
1022 
1023  // Free any previous memory allocations
1024  if (audio_codec_ctx != nullptr) {
1025  AV_FREE_CONTEXT(audio_codec_ctx);
1026  }
1027 
1028  // Create a new audio stream
1029  AVStream* st = avformat_new_stream(oc, codec);
1030  if (!st)
1031  throw OutOfMemory("Could not allocate memory for the video stream.", path);
1032 
1033  // Allocate a new codec context for the stream
1034  ALLOC_CODEC_CTX(audio_codec_ctx, codec, st)
1035 #if (LIBAVFORMAT_VERSION_MAJOR >= 58)
1036  st->codecpar->codec_id = codec->id;
1037 #endif
1038  AVCodecContext* c = audio_codec_ctx;
1039 
1040  c->codec_id = codec->id;
1041  c->codec_type = AVMEDIA_TYPE_AUDIO;
1042 
1043  // Set the sample parameters
1044  c->bit_rate = info.audio_bit_rate;
1045 #if !HAVE_CH_LAYOUT
1046  c->channels = info.channels;
1047 #endif
1048 
1049  // Set valid sample rate (or throw error)
1050  if (codec->supported_samplerates) {
1051  int i;
1052  for (i = 0; codec->supported_samplerates[i] != 0; i++)
1053  if (info.sample_rate == codec->supported_samplerates[i]) {
1054  // Set the valid sample rate
1055  c->sample_rate = info.sample_rate;
1056  break;
1057  }
1058  if (codec->supported_samplerates[i] == 0)
1059  throw InvalidSampleRate("An invalid sample rate was detected for this codec.", path);
1060  } else
1061  // Set sample rate
1062  c->sample_rate = info.sample_rate;
1063 
1064 uint64_t channel_layout = info.channel_layout;
1065 #if HAVE_CH_LAYOUT
1066  // Set a valid number of channels (or throw error)
1067  AVChannelLayout ch_layout;
1068  av_channel_layout_from_mask(&ch_layout, info.channel_layout);
1069  if (codec->ch_layouts) {
1070  int i;
1071  for (i = 0; av_channel_layout_check(&codec->ch_layouts[i]); i++)
1072  if (av_channel_layout_compare(&ch_layout, &codec->ch_layouts[i])) {
1073  // Set valid channel layout
1074  av_channel_layout_copy(&c->ch_layout, &ch_layout);
1075  break;
1076  }
1077  if (!av_channel_layout_check(&codec->ch_layouts[i]))
1078  throw InvalidChannels("An invalid channel layout was detected (i.e. MONO / STEREO).", path);
1079  } else
1080  // Set valid channel layout
1081  av_channel_layout_copy(&c->ch_layout, &ch_layout);
1082 #else
1083  // Set a valid number of channels (or throw error)
1084  if (codec->channel_layouts) {
1085  int i;
1086  for (i = 0; codec->channel_layouts[i] != 0; i++)
1087  if (channel_layout == codec->channel_layouts[i]) {
1088  // Set valid channel layout
1089  c->channel_layout = channel_layout;
1090  break;
1091  }
1092  if (codec->channel_layouts[i] == 0)
1093  throw InvalidChannels("An invalid channel layout was detected (i.e. MONO / STEREO).", path);
1094  } else
1095  // Set valid channel layout
1096  c->channel_layout = channel_layout;
1097 #endif
1098 
1099  // Choose a valid sample_fmt
1100  if (codec->sample_fmts) {
1101  for (int i = 0; codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; i++) {
1102  // Set sample format to 1st valid format (and then exit loop)
1103  c->sample_fmt = codec->sample_fmts[i];
1104  break;
1105  }
1106  }
1107  if (c->sample_fmt == AV_SAMPLE_FMT_NONE) {
1108  // Default if no sample formats found
1109  c->sample_fmt = AV_SAMPLE_FMT_S16;
1110  }
1111 
1112  // some formats want stream headers to be separate
1113  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
1114 #if (LIBAVCODEC_VERSION_MAJOR >= 57)
1115  // FFmpeg 3.0+
1116  c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
1117 #else
1118  c->flags |= CODEC_FLAG_GLOBAL_HEADER;
1119 #endif
1120 
1121  AV_COPY_PARAMS_FROM_CONTEXT(st, c);
1122 
1123 int nb_channels;
1124 const char* nb_channels_label;
1125 const char* channel_layout_label;
1126 
1127 #if HAVE_CH_LAYOUT
1128  nb_channels = c->ch_layout.nb_channels;
1129  channel_layout = c->ch_layout.u.mask;
1130  nb_channels_label = "c->ch_layout.nb_channels";
1131  channel_layout_label = "c->ch_layout.u.mask";
1132 #else
1133  nb_channels = c->channels;
1134  nb_channels_label = "c->channels";
1135  channel_layout_label = "c->channel_layout";
1136 #endif
1137 
1138  ZmqLogger::Instance()->AppendDebugMethod(
1139  "FFmpegWriter::add_audio_stream",
1140  "c->codec_id", c->codec_id,
1141  "c->bit_rate", c->bit_rate,
1142  nb_channels_label, nb_channels,
1143  "c->sample_fmt", c->sample_fmt,
1144  channel_layout_label, channel_layout,
1145  "c->sample_rate", c->sample_rate);
1146 
1147  return st;
1148 }
1149 
1150 // Add a video output stream
1151 AVStream *FFmpegWriter::add_video_stream() {
1152  // Find the video codec
1153  const AVCodec *codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
1154  if (codec == NULL)
1155  throw InvalidCodec("A valid video codec could not be found for this file.", path);
1156 
1157  // Free any previous memory allocations
1158  if (video_codec_ctx != nullptr) {
1159  AV_FREE_CONTEXT(video_codec_ctx);
1160  }
1161 
1162  // Create a new video stream
1163  AVStream* st = avformat_new_stream(oc, codec);
1164  if (!st)
1165  throw OutOfMemory("Could not allocate memory for the video stream.", path);
1166 
1167  // Allocate a new codec context for the stream
1168  ALLOC_CODEC_CTX(video_codec_ctx, codec, st)
1169 #if (LIBAVFORMAT_VERSION_MAJOR >= 58)
1170  st->codecpar->codec_id = codec->id;
1171 #endif
1172 
1173  AVCodecContext* c = video_codec_ctx;
1174 
1175  c->codec_id = codec->id;
1176  c->codec_type = AVMEDIA_TYPE_VIDEO;
1177 
1178  // Set sample aspect ratio
1179  c->sample_aspect_ratio.num = info.pixel_ratio.num;
1180  c->sample_aspect_ratio.den = info.pixel_ratio.den;
1181 
1182  /* Init video encoder options */
1183  if (info.video_bit_rate >= 1000
1184 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
1185  && c->codec_id != AV_CODEC_ID_AV1
1186 #endif
1187  ) {
1188  c->bit_rate = info.video_bit_rate;
1189  if (info.video_bit_rate >= 1500000) {
1190  if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
1191  c->qmin = 2;
1192  c->qmax = 30;
1193  }
1194  }
1195  // Here should be the setting for low fixed bitrate
1196  // Defaults are used because mpeg2 otherwise had problems
1197  } else {
1198  // Check if codec supports crf or qp
1199  switch (c->codec_id) {
1200 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
1201  // FFmpeg 4.0+
1202  case AV_CODEC_ID_AV1 :
1203  // TODO: Set `crf` or `qp` according to bitrate, as bitrate is not supported by these encoders yet.
1204  if (info.video_bit_rate >= 1000) {
1205  c->bit_rate = 0;
1206  if (strstr(info.vcodec.c_str(), "aom") != NULL) {
1207  int calculated_quality = 35;
1208  if (info.video_bit_rate < 500000) calculated_quality = 50;
1209  if (info.video_bit_rate > 5000000) calculated_quality = 10;
1210  av_opt_set_int(c->priv_data, "crf", calculated_quality, 0);
1211  info.video_bit_rate = calculated_quality;
1212  } else {
1213  int calculated_quality = 50;
1214  if (info.video_bit_rate < 500000) calculated_quality = 60;
1215  if (info.video_bit_rate > 5000000) calculated_quality = 15;
1216  av_opt_set_int(c->priv_data, "qp", calculated_quality, 0);
1217  info.video_bit_rate = calculated_quality;
1218  } // medium
1219  }
1220  if (strstr(info.vcodec.c_str(), "svtav1") != NULL) {
1221  av_opt_set_int(c->priv_data, "preset", 6, 0);
1222  av_opt_set_int(c->priv_data, "forced-idr",1,0);
1223  }
1224  else if (strstr(info.vcodec.c_str(), "rav1e") != NULL) {
1225  av_opt_set_int(c->priv_data, "speed", 7, 0);
1226  av_opt_set_int(c->priv_data, "tile-rows", 2, 0);
1227  av_opt_set_int(c->priv_data, "tile-columns", 4, 0);
1228  }
1229  else if (strstr(info.vcodec.c_str(), "aom") != NULL) {
1230  // Set number of tiles to a fixed value
1231  // TODO: Allow user to chose their own number of tiles
1232  av_opt_set_int(c->priv_data, "tile-rows", 1, 0); // log2 of number of rows
1233  av_opt_set_int(c->priv_data, "tile-columns", 2, 0); // log2 of number of columns
1234  av_opt_set_int(c->priv_data, "row-mt", 1, 0); // use multiple cores
1235  av_opt_set_int(c->priv_data, "cpu-used", 3, 0); // default is 1, usable is 4
1236  }
1237  //break;
1238 #endif
1239  case AV_CODEC_ID_VP9 :
1240  case AV_CODEC_ID_HEVC :
1241  case AV_CODEC_ID_VP8 :
1242  case AV_CODEC_ID_H264 :
1243  if (info.video_bit_rate < 40) {
1244  c->qmin = 0;
1245  c->qmax = 63;
1246  } else {
1247  c->qmin = info.video_bit_rate - 5;
1248  c->qmax = 63;
1249  }
1250  break;
1251  default:
1252  // Here should be the setting for codecs that don't support crf
1253  // For now defaults are used
1254  break;
1255  }
1256  }
1257 
1258  //TODO: Implement variable bitrate feature (which actually works). This implementation throws
1259  //invalid bitrate errors and rc buffer underflow errors, etc...
1260  //c->rc_min_rate = info.video_bit_rate;
1261  //c->rc_max_rate = info.video_bit_rate;
1262  //c->rc_buffer_size = FFMAX(c->rc_max_rate, 15000000) * 112L / 15000000 * 16384;
1263  //if ( !c->rc_initial_buffer_occupancy )
1264  // c->rc_initial_buffer_occupancy = c->rc_buffer_size * 3/4;
1265 
1266  /* resolution must be a multiple of two */
1267  // TODO: require /2 height and width
1268  c->width = info.width;
1269  c->height = info.height;
1270 
1271  /* time base: this is the fundamental unit of time (in seconds) in terms
1272  of which frame timestamps are represented. for fixed-fps content,
1273  timebase should be 1/framerate and timestamp increments should be
1274  identically 1. */
1275  c->time_base.num = info.video_timebase.num;
1276  c->time_base.den = info.video_timebase.den;
1277 // AVCodecContext->framerate was added in FFmpeg 2.6
1278 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 26, 0)
1279  c->framerate = av_inv_q(c->time_base);
1280 #endif
1281  st->avg_frame_rate = av_inv_q(c->time_base);
1282  st->time_base.num = info.video_timebase.num;
1283  st->time_base.den = info.video_timebase.den;
1284 
1285  c->gop_size = 12; /* TODO: add this to "info"... emit one intra frame every twelve frames at most */
1286  c->max_b_frames = 10;
1287  if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO)
1288  /* just for testing, we also add B frames */
1289  c->max_b_frames = 2;
1290  if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO)
1291  /* Needed to avoid using macroblocks in which some coeffs overflow.
1292  This does not happen with normal video, it just happens here as
1293  the motion of the chroma plane does not match the luma plane. */
1294  c->mb_decision = 2;
1295  // some formats want stream headers to be separate
1296  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
1297 #if (LIBAVCODEC_VERSION_MAJOR >= 57)
1298  // FFmpeg 3.0+
1299  c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
1300 #else
1301  c->flags |= CODEC_FLAG_GLOBAL_HEADER;
1302 #endif
1303 
1304  // Find all supported pixel formats for this codec
1305  const PixelFormat *supported_pixel_formats = codec->pix_fmts;
1306  while (supported_pixel_formats != NULL && *supported_pixel_formats != PIX_FMT_NONE) {
1307  // Assign the 1st valid pixel format (if one is missing)
1308  if (c->pix_fmt == PIX_FMT_NONE)
1309  c->pix_fmt = *supported_pixel_formats;
1310  ++supported_pixel_formats;
1311  }
1312 
1313  // Codec doesn't have any pix formats?
1314  if (c->pix_fmt == PIX_FMT_NONE) {
1315  if (oc->oformat->video_codec == AV_CODEC_ID_RAWVIDEO) {
1316  // Raw video should use RGB24
1317  c->pix_fmt = PIX_FMT_RGB24;
1318 
1319 #if (LIBAVFORMAT_VERSION_MAJOR < 58)
1320  // FFmpeg < 4.0
1321  if (strcmp(oc->oformat->name, "gif") != 0)
1322  // If not GIF format, skip the encoding process
1323  // Set raw picture flag (so we don't encode this video)
1324  oc->oformat->flags |= AVFMT_RAWPICTURE;
1325 #endif
1326  } else {
1327  // Set the default codec
1328  c->pix_fmt = PIX_FMT_YUV420P;
1329  }
1330  }
1331 
1332  AV_COPY_PARAMS_FROM_CONTEXT(st, c);
1333  ZmqLogger::Instance()->AppendDebugMethod(
1334  "FFmpegWriter::add_video_stream ("
1335  + (std::string)oc->oformat->name + " : "
1336  + (std::string)av_get_pix_fmt_name(c->pix_fmt) + ")",
1337  "c->codec_id", c->codec_id,
1338  "c->bit_rate", c->bit_rate,
1339  "c->pix_fmt", c->pix_fmt,
1340  "oc->oformat->flags", oc->oformat->flags);
1341  return st;
1342 }
1343 
1344 // open audio codec
1345 void FFmpegWriter::open_audio(AVFormatContext *oc, AVStream *st) {
1346  const AVCodec *codec;
1347  AV_GET_CODEC_FROM_STREAM(st, audio_codec_ctx)
1348 
1349  // Set number of threads equal to number of processors (not to exceed 16)
1350  audio_codec_ctx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
1351 
1352  // Find the audio encoder
1353  codec = avcodec_find_encoder_by_name(info.acodec.c_str());
1354  if (!codec)
1355  codec = avcodec_find_encoder(audio_codec_ctx->codec_id);
1356  if (!codec)
1357  throw InvalidCodec("Could not find codec", path);
1358 
1359  // Init options
1360  AVDictionary *opts = NULL;
1361  av_dict_set(&opts, "strict", "experimental", 0);
1362 
1363  // Open the codec
1364  if (avcodec_open2(audio_codec_ctx, codec, &opts) < 0)
1365  throw InvalidCodec("Could not open audio codec", path);
1366  AV_COPY_PARAMS_FROM_CONTEXT(st, audio_codec_ctx);
1367 
1368  // Free options
1369  av_dict_free(&opts);
1370 
1371  // Calculate the size of the input frame (i..e how many samples per packet), and the output buffer
1372  // TODO: Ugly hack for PCM codecs (will be removed ASAP with new PCM support to compute the input frame size in samples
1373  if (audio_codec_ctx->frame_size <= 1) {
1374  // No frame size found... so calculate
1375  audio_input_frame_size = 50000 / info.channels;
1376 
1377  int s = AV_FIND_DECODER_CODEC_ID(st);
1378  switch (s) {
1379  case AV_CODEC_ID_PCM_S16LE:
1380  case AV_CODEC_ID_PCM_S16BE:
1381  case AV_CODEC_ID_PCM_U16LE:
1382  case AV_CODEC_ID_PCM_U16BE:
1383  audio_input_frame_size >>= 1;
1384  break;
1385  default:
1386  break;
1387  }
1388  } else {
1389  // Set frame size based on the codec
1390  audio_input_frame_size = audio_codec_ctx->frame_size;
1391  }
1392 
1393  // Set the initial frame size (since it might change during resampling)
1394  initial_audio_input_frame_size = audio_input_frame_size;
1395 
1396  // Allocate array for samples
1397  samples = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE];
1398 
1399  // Set audio output buffer (used to store the encoded audio)
1400  audio_outbuf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE;
1401  audio_outbuf = new uint8_t[audio_outbuf_size];
1402 
1403  // Set audio packet encoding buffer
1404  audio_encoder_buffer_size = AUDIO_PACKET_ENCODING_SIZE;
1405  audio_encoder_buffer = new uint8_t[audio_encoder_buffer_size];
1406 
1407  // Add audio metadata (if any)
1408  for (std::map<std::string, std::string>::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) {
1409  av_dict_set(&st->metadata, iter->first.c_str(), iter->second.c_str(), 0);
1410  }
1411 
1412  ZmqLogger::Instance()->AppendDebugMethod(
1413  "FFmpegWriter::open_audio",
1414  "audio_codec_ctx->thread_count", audio_codec_ctx->thread_count,
1415  "audio_input_frame_size", audio_input_frame_size,
1416  "buffer_size", AVCODEC_MAX_AUDIO_FRAME_SIZE + MY_INPUT_BUFFER_PADDING_SIZE);
1417 }
1418 
1419 // open video codec
1420 void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) {
1421  const AVCodec *codec;
1422  AV_GET_CODEC_FROM_STREAM(st, video_codec_ctx)
1423 
1424  // Set number of threads equal to number of processors (not to exceed 16)
1425  video_codec_ctx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
1426 
1427 #if USE_HW_ACCEL
1428  if (hw_en_on && hw_en_supported) {
1429  //char *dev_hw = NULL;
1430  char adapter[256];
1431  char *adapter_ptr = NULL;
1432  int adapter_num;
1433  // Use the hw device given in the environment variable HW_EN_DEVICE_SET or the default if not set
1434  adapter_num = openshot::Settings::Instance()->HW_EN_DEVICE_SET;
1435  std::clog << "Encoding Device Nr: " << adapter_num << "\n";
1436  if (adapter_num < 3 && adapter_num >=0) {
1437 #if defined(__linux__)
1438  snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128);
1439  // Maybe 127 is better because the first card would be 1?!
1440  adapter_ptr = adapter;
1441 #elif defined(_WIN32) || defined(__APPLE__)
1442  adapter_ptr = NULL;
1443 #endif
1444  }
1445  else {
1446  adapter_ptr = NULL; // Just to be sure
1447  }
1448 // Check if it is there and writable
1449 #if defined(__linux__)
1450  if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) {
1451 #elif defined(_WIN32) || defined(__APPLE__)
1452  if( adapter_ptr != NULL ) {
1453 #endif
1454  ZmqLogger::Instance()->AppendDebugMethod(
1455  "Encode Device present using device",
1456  "adapter", adapter_num);
1457  }
1458  else {
1459  adapter_ptr = NULL; // use default
1460  ZmqLogger::Instance()->AppendDebugMethod(
1461  "Encode Device not present, using default");
1462  }
1463  if (av_hwdevice_ctx_create(&hw_device_ctx,
1464  hw_en_av_device_type, adapter_ptr, NULL, 0) < 0)
1465  {
1466  ZmqLogger::Instance()->AppendDebugMethod(
1467  "FFmpegWriter::open_video ERROR creating hwdevice, Codec name:",
1468  info.vcodec.c_str(), -1);
1469  throw InvalidCodec("Could not create hwdevice", path);
1470  }
1471  }
1472 #endif // USE_HW_ACCEL
1473 
1474  /* find the video encoder */
1475  codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
1476  if (!codec)
1477  codec = avcodec_find_encoder(AV_FIND_DECODER_CODEC_ID(st));
1478  if (!codec)
1479  throw InvalidCodec("Could not find codec", path);
1480 
1481  /* Force max_b_frames to 0 in some cases (i.e. for mjpeg image sequences */
1482  if (video_codec_ctx->max_b_frames && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG4 && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG1VIDEO && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG2VIDEO)
1483  video_codec_ctx->max_b_frames = 0;
1484 
1485  // Init options
1486  AVDictionary *opts = NULL;
1487  av_dict_set(&opts, "strict", "experimental", 0);
1488 
1489 #if USE_HW_ACCEL
1490  if (hw_en_on && hw_en_supported) {
1491  video_codec_ctx->pix_fmt = hw_en_av_pix_fmt;
1492 
1493  // for the list of possible options, see the list of codec-specific options:
1494  // e.g. ffmpeg -h encoder=h264_vaapi or ffmpeg -h encoder=hevc_vaapi
1495  // and "man ffmpeg-codecs"
1496 
1497  // For VAAPI, it is safer to explicitly set rc_mode instead of relying on auto-selection
1498  // which is ffmpeg version-specific.
1499  if (hw_en_av_pix_fmt == AV_PIX_FMT_VAAPI) {
1500  int64_t qp;
1501  if (av_opt_get_int(video_codec_ctx->priv_data, "qp", 0, &qp) != 0 || qp == 0) {
1502  // unless "qp" was set for CQP, switch to VBR RC mode
1503  av_opt_set(video_codec_ctx->priv_data, "rc_mode", "VBR", 0);
1504 
1505  // In the current state (ffmpeg-4.2-4 libva-mesa-driver-19.1.5-1) to use VBR,
1506  // one has to specify both bit_rate and maxrate, otherwise a small low quality file is generated on Intel iGPU).
1507  video_codec_ctx->rc_max_rate = video_codec_ctx->bit_rate;
1508  }
1509  }
1510 
1511  switch (video_codec_ctx->codec_id) {
1512  case AV_CODEC_ID_H264:
1513  video_codec_ctx->max_b_frames = 0; // At least this GPU doesn't support b-frames
1514  video_codec_ctx->profile = FF_PROFILE_H264_BASELINE | FF_PROFILE_H264_CONSTRAINED;
1515  av_opt_set(video_codec_ctx->priv_data, "preset", "slow", 0);
1516  av_opt_set(video_codec_ctx->priv_data, "tune", "zerolatency", 0);
1517  av_opt_set(video_codec_ctx->priv_data, "vprofile", "baseline", AV_OPT_SEARCH_CHILDREN);
1518  break;
1519  case AV_CODEC_ID_HEVC:
1520  // tested to work with defaults
1521  break;
1522  case AV_CODEC_ID_VP9:
1523  // tested to work with defaults
1524  break;
1525  default:
1526  ZmqLogger::Instance()->AppendDebugMethod(
1527  "No codec-specific options defined for this codec. HW encoding may fail",
1528  "codec_id", video_codec_ctx->codec_id);
1529  break;
1530  }
1531 
1532  // set hw_frames_ctx for encoder's AVCodecContext
1533  int err;
1534  if ((err = set_hwframe_ctx(video_codec_ctx, hw_device_ctx, info.width, info.height)) < 0)
1535  {
1536  ZmqLogger::Instance()->AppendDebugMethod(
1537  "FFmpegWriter::open_video (set_hwframe_ctx) ERROR faled to set hwframe context",
1538  "width", info.width,
1539  "height", info.height,
1540  av_err2string(err), -1);
1541  }
1542  }
1543 #endif // USE_HW_ACCEL
1544 
1545  /* open the codec */
1546  if (avcodec_open2(video_codec_ctx, codec, &opts) < 0)
1547  throw InvalidCodec("Could not open video codec", path);
1548  AV_COPY_PARAMS_FROM_CONTEXT(st, video_codec_ctx);
1549 
1550  // Free options
1551  av_dict_free(&opts);
1552 
1553  // Add video metadata (if any)
1554  for (std::map<std::string, std::string>::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) {
1555  av_dict_set(&st->metadata, iter->first.c_str(), iter->second.c_str(), 0);
1556  }
1557 
1558  ZmqLogger::Instance()->AppendDebugMethod(
1559  "FFmpegWriter::open_video",
1560  "video_codec_ctx->thread_count", video_codec_ctx->thread_count);
1561 
1562 }
1563 
1564 // write all queued frames' audio to the video file
1565 void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptr<openshot::Frame> frame) {
1566  if (!frame && !is_final)
1567  return;
1568 
1569  // Init audio buffers / variables
1570  int total_frame_samples = 0;
1571  int frame_position = 0;
1572  int channels_in_frame = 0;
1573  int sample_rate_in_frame = 0;
1574  int samples_in_frame = 0;
1575  ChannelLayout channel_layout_in_frame = LAYOUT_MONO; // default channel layout
1576 
1577  // Create a new array (to hold all S16 audio samples, for the current queued frames
1578  unsigned int all_queued_samples_size = sizeof(int16_t) * AVCODEC_MAX_AUDIO_FRAME_SIZE;
1579  int16_t *all_queued_samples = (int16_t *) av_malloc(all_queued_samples_size);
1580  int16_t *all_resampled_samples = NULL;
1581  int16_t *final_samples_planar = NULL;
1582  int16_t *final_samples = NULL;
1583 
1584  // Get audio sample array
1585  float *frame_samples_float = NULL;
1586 
1587  // Get the audio details from this frame
1588  if (frame) {
1589  sample_rate_in_frame = frame->SampleRate();
1590  samples_in_frame = frame->GetAudioSamplesCount();
1591  channels_in_frame = frame->GetAudioChannelsCount();
1592  channel_layout_in_frame = frame->ChannelsLayout();
1593 
1594  // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
1595  frame_samples_float = frame->GetInterleavedAudioSamples(&samples_in_frame);
1596  }
1597 
1598  // Calculate total samples
1599  total_frame_samples = samples_in_frame * channels_in_frame;
1600 
1601  // Translate audio sample values back to 16 bit integers with saturation
1602  const int16_t max16 = 32767;
1603  const int16_t min16 = -32768;
1604  for (int s = 0; s < total_frame_samples; s++, frame_position++) {
1605  float valF = frame_samples_float[s] * (1 << 15);
1606  int16_t conv;
1607  if (valF > max16) {
1608  conv = max16;
1609  } else if (valF < min16) {
1610  conv = min16;
1611  } else {
1612  conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding
1613  }
1614 
1615  // Copy into buffer
1616  all_queued_samples[frame_position] = conv;
1617  }
1618 
1619  // Deallocate float array
1620  delete[] frame_samples_float;
1621 
1622 
1623  // Update total samples (since we've combined all queued frames)
1624  total_frame_samples = frame_position;
1625  int remaining_frame_samples = total_frame_samples;
1626  int samples_position = 0;
1627 
1628 
1629  ZmqLogger::Instance()->AppendDebugMethod(
1630  "FFmpegWriter::write_audio_packets",
1631  "is_final", is_final,
1632  "total_frame_samples", total_frame_samples,
1633  "channel_layout_in_frame", channel_layout_in_frame,
1634  "channels_in_frame", channels_in_frame,
1635  "samples_in_frame", samples_in_frame,
1636  "LAYOUT_MONO", LAYOUT_MONO);
1637 
1638  // Keep track of the original sample format
1639  AVSampleFormat output_sample_fmt = audio_codec_ctx->sample_fmt;
1640 
1641  AVFrame *audio_frame = NULL;
1642  if (!is_final) {
1643  // Create input frame (and allocate arrays)
1644  audio_frame = AV_ALLOCATE_FRAME();
1645  AV_RESET_FRAME(audio_frame);
1646  audio_frame->nb_samples = total_frame_samples / channels_in_frame;
1647 
1648  // Fill input frame with sample data
1649  int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) all_queued_samples, all_queued_samples_size, 0);
1650  if (error_code < 0) {
1651  ZmqLogger::Instance()->AppendDebugMethod(
1652  "FFmpegWriter::write_audio_packets ERROR ["
1653  + av_err2string(error_code) + "]",
1654  "error_code", error_code);
1655  }
1656 
1657  // Do not convert audio to planar format (yet). We need to keep everything interleaved at this point.
1658  switch (audio_codec_ctx->sample_fmt) {
1659  case AV_SAMPLE_FMT_FLTP: {
1660  output_sample_fmt = AV_SAMPLE_FMT_FLT;
1661  break;
1662  }
1663  case AV_SAMPLE_FMT_S32P: {
1664  output_sample_fmt = AV_SAMPLE_FMT_S32;
1665  break;
1666  }
1667  case AV_SAMPLE_FMT_S16P: {
1668  output_sample_fmt = AV_SAMPLE_FMT_S16;
1669  break;
1670  }
1671  case AV_SAMPLE_FMT_U8P: {
1672  output_sample_fmt = AV_SAMPLE_FMT_U8;
1673  break;
1674  }
1675  default: {
1676  // This is only here to silence unused-enum warnings
1677  break;
1678  }
1679  }
1680 
1681  // Update total samples & input frame size (due to bigger or smaller data types)
1682  total_frame_samples *= (float(info.sample_rate) / sample_rate_in_frame); // adjust for different byte sizes
1683  total_frame_samples *= (float(info.channels) / channels_in_frame); // adjust for different # of channels
1684 
1685  // Create output frame (and allocate arrays)
1686  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1687  AV_RESET_FRAME(audio_converted);
1688  audio_converted->nb_samples = total_frame_samples / channels_in_frame;
1689  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_converted->nb_samples, output_sample_fmt, 0);
1690 
1691  ZmqLogger::Instance()->AppendDebugMethod(
1692  "FFmpegWriter::write_audio_packets (1st resampling)",
1693  "in_sample_fmt", AV_SAMPLE_FMT_S16,
1694  "out_sample_fmt", output_sample_fmt,
1695  "in_sample_rate", sample_rate_in_frame,
1696  "out_sample_rate", info.sample_rate,
1697  "in_channels", channels_in_frame,
1698  "out_channels", info.channels);
1699 
1700  // setup resample context
1701  if (!avr) {
1702  avr = SWR_ALLOC();
1703 #if HAVE_CH_LAYOUT
1704  AVChannelLayout in_chlayout;
1705  AVChannelLayout out_chlayout;
1706  av_channel_layout_from_mask(&in_chlayout, channel_layout_in_frame);
1707  av_channel_layout_from_mask(&out_chlayout, info.channel_layout);
1708  av_opt_set_chlayout(avr, "in_chlayout", &in_chlayout, 0);
1709  av_opt_set_chlayout(avr, "out_chlayout", &out_chlayout, 0);
1710 #else
1711  av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
1712  av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
1713  av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
1714  av_opt_set_int(avr, "out_channels", info.channels, 0);
1715 #endif
1716  av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1717  av_opt_set_int(avr, "out_sample_fmt", output_sample_fmt, 0); // planar not allowed here
1718  av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
1719  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1720  SWR_INIT(avr);
1721  }
1722  // Convert audio samples
1723  int nb_samples = SWR_CONVERT(
1724  avr, // audio resample context
1725  audio_converted->data, // output data pointers
1726  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1727  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1728  audio_frame->data, // input data pointers
1729  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1730  audio_frame->nb_samples // number of input samples to convert
1731  );
1732 
1733  // Set remaining samples
1734  remaining_frame_samples = total_frame_samples;
1735 
1736  // Create a new array (to hold all resampled S16 audio samples)
1737  all_resampled_samples = (int16_t *) av_malloc(
1738  sizeof(int16_t) * nb_samples * info.channels
1739  * (av_get_bytes_per_sample(output_sample_fmt) /
1740  av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) )
1741  );
1742 
1743  // Copy audio samples over original samples
1744  memcpy(all_resampled_samples, audio_converted->data[0],
1745  static_cast<size_t>(nb_samples)
1746  * info.channels
1747  * av_get_bytes_per_sample(output_sample_fmt));
1748 
1749  // Remove converted audio
1750  av_freep(&(audio_frame->data[0]));
1751  AV_FREE_FRAME(&audio_frame);
1752  av_freep(&audio_converted->data[0]);
1753  AV_FREE_FRAME(&audio_converted);
1754  all_queued_samples = NULL; // this array cleared with above call
1755 
1756  ZmqLogger::Instance()->AppendDebugMethod(
1757  "FFmpegWriter::write_audio_packets (Successfully completed 1st resampling)",
1758  "nb_samples", nb_samples,
1759  "remaining_frame_samples", remaining_frame_samples);
1760  }
1761 
1762  // Loop until no more samples
1763  while (remaining_frame_samples > 0 || is_final) {
1764  // Get remaining samples needed for this packet
1765  int remaining_packet_samples = (audio_input_frame_size * info.channels) - audio_input_position;
1766 
1767  // Determine how many samples we need
1768  int diff = 0;
1769  if (remaining_frame_samples >= remaining_packet_samples) {
1770  diff = remaining_packet_samples;
1771  } else {
1772  diff = remaining_frame_samples;
1773  }
1774 
1775  // Copy frame samples into the packet samples array
1776  if (!is_final)
1777  //TODO: Make this more sane
1778  memcpy(
1779  samples + (audio_input_position
1780  * (av_get_bytes_per_sample(output_sample_fmt) /
1781  av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) )
1782  ),
1783  all_resampled_samples + samples_position,
1784  static_cast<size_t>(diff)
1785  * av_get_bytes_per_sample(output_sample_fmt)
1786  );
1787 
1788  // Increment counters
1789  audio_input_position += diff;
1790  samples_position += diff * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16));
1791  remaining_frame_samples -= diff;
1792 
1793  // Do we have enough samples to proceed?
1794  if (audio_input_position < (audio_input_frame_size * info.channels) && !is_final)
1795  // Not enough samples to encode... so wait until the next frame
1796  break;
1797 
1798  // Convert to planar (if needed by audio codec)
1799  AVFrame *frame_final = AV_ALLOCATE_FRAME();
1800  AV_RESET_FRAME(frame_final);
1801  if (av_sample_fmt_is_planar(audio_codec_ctx->sample_fmt)) {
1802  ZmqLogger::Instance()->AppendDebugMethod(
1803  "FFmpegWriter::write_audio_packets (2nd resampling for Planar formats)",
1804  "in_sample_fmt", output_sample_fmt,
1805  "out_sample_fmt", audio_codec_ctx->sample_fmt,
1806  "in_sample_rate", info.sample_rate,
1807  "out_sample_rate", info.sample_rate,
1808  "in_channels", info.channels,
1809  "out_channels", info.channels
1810  );
1811 
1812  // setup resample context
1813  if (!avr_planar) {
1814  avr_planar = SWR_ALLOC();
1815 #if HAVE_CH_LAYOUT
1816  AVChannelLayout layout;
1817  av_channel_layout_from_mask(&layout, info.channel_layout);
1818  av_opt_set_chlayout(avr_planar, "in_chlayout", &layout, 0);
1819  av_opt_set_chlayout(avr_planar, "out_chlayout", &layout, 0);
1820 #else
1821  av_opt_set_int(avr_planar, "in_channel_layout", info.channel_layout, 0);
1822  av_opt_set_int(avr_planar, "out_channel_layout", info.channel_layout, 0);
1823  av_opt_set_int(avr_planar, "in_channels", info.channels, 0);
1824  av_opt_set_int(avr_planar, "out_channels", info.channels, 0);
1825 #endif
1826  av_opt_set_int(avr_planar, "in_sample_fmt", output_sample_fmt, 0);
1827  av_opt_set_int(avr_planar, "out_sample_fmt", audio_codec_ctx->sample_fmt, 0); // planar not allowed here
1828  av_opt_set_int(avr_planar, "in_sample_rate", info.sample_rate, 0);
1829  av_opt_set_int(avr_planar, "out_sample_rate", info.sample_rate, 0);
1830  SWR_INIT(avr_planar);
1831  }
1832 
1833  // Create input frame (and allocate arrays)
1834  audio_frame = AV_ALLOCATE_FRAME();
1835  AV_RESET_FRAME(audio_frame);
1836  audio_frame->nb_samples = audio_input_position / info.channels;
1837 
1838  // Create a new array
1839  final_samples_planar = (int16_t *) av_malloc(
1840  sizeof(int16_t) * audio_frame->nb_samples * info.channels
1841  * (av_get_bytes_per_sample(output_sample_fmt) /
1842  av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) )
1843  );
1844 
1845  // Copy audio into buffer for frame
1846  memcpy(final_samples_planar, samples,
1847  static_cast<size_t>(audio_frame->nb_samples)
1848  * info.channels
1849  * av_get_bytes_per_sample(output_sample_fmt));
1850 
1851  // Fill input frame with sample data
1852  avcodec_fill_audio_frame(audio_frame, info.channels, output_sample_fmt,
1853  (uint8_t *) final_samples_planar, audio_encoder_buffer_size, 0);
1854 
1855  // Create output frame (and allocate arrays)
1856  frame_final->nb_samples = audio_input_frame_size;
1857 #if HAVE_CH_LAYOUT
1858  av_channel_layout_from_mask(&frame_final->ch_layout, info.channel_layout);
1859 #else
1860  frame_final->channels = info.channels;
1861  frame_final->channel_layout = info.channel_layout;
1862 #endif
1863  frame_final->format = audio_codec_ctx->sample_fmt;
1864  av_samples_alloc(frame_final->data, frame_final->linesize, info.channels,
1865  frame_final->nb_samples, audio_codec_ctx->sample_fmt, 0);
1866 
1867  // Convert audio samples
1868  int nb_samples = SWR_CONVERT(
1869  avr_planar, // audio resample context
1870  frame_final->data, // output data pointers
1871  frame_final->linesize[0], // output plane size, in bytes. (0 if unknown)
1872  frame_final->nb_samples, // maximum number of samples that the output buffer can hold
1873  audio_frame->data, // input data pointers
1874  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1875  audio_frame->nb_samples // number of input samples to convert
1876  );
1877 
1878  // Copy audio samples over original samples
1879  const auto copy_length = static_cast<size_t>(nb_samples)
1880  * av_get_bytes_per_sample(audio_codec_ctx->sample_fmt)
1881  * info.channels;
1882 
1883  if (nb_samples > 0)
1884  memcpy(samples, frame_final->data[0], copy_length);
1885 
1886  // deallocate AVFrame
1887  av_freep(&(audio_frame->data[0]));
1888  AV_FREE_FRAME(&audio_frame);
1889  all_queued_samples = NULL; // this array cleared with above call
1890 
1891  ZmqLogger::Instance()->AppendDebugMethod(
1892  "FFmpegWriter::write_audio_packets (Successfully completed 2nd resampling for Planar formats)",
1893  "nb_samples", nb_samples);
1894 
1895  } else {
1896  // Create a new array
1897  const auto buf_size = static_cast<size_t>(audio_input_position)
1898  * (av_get_bytes_per_sample(audio_codec_ctx->sample_fmt) /
1899  av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)
1900  );
1901  final_samples = reinterpret_cast<int16_t*>(
1902  av_malloc(sizeof(int16_t) * buf_size));
1903 
1904  // Copy audio into buffer for frame
1905  memcpy(final_samples, samples,
1906  audio_input_position * av_get_bytes_per_sample(audio_codec_ctx->sample_fmt));
1907 
1908  // Init the nb_samples property
1909  frame_final->nb_samples = audio_input_frame_size;
1910 
1911  // Fill the final_frame AVFrame with audio (non planar)
1912 #if HAVE_CH_LAYOUT
1913  int nb_channels = audio_codec_ctx->ch_layout.nb_channels;
1914 #else
1915  int nb_channels = audio_codec_ctx->channels;
1916 #endif
1917  avcodec_fill_audio_frame(frame_final, nb_channels,
1918  audio_codec_ctx->sample_fmt, (uint8_t *) final_samples,
1919  audio_encoder_buffer_size, 0);
1920  }
1921 
1922  // Set the AVFrame's PTS
1923  frame_final->pts = audio_timestamp;
1924 
1925  // Init the packet
1926 #if IS_FFMPEG_3_2
1927  AVPacket* pkt = av_packet_alloc();
1928 #else
1929  AVPacket* pkt;
1930  av_init_packet(pkt);
1931 #endif
1932  pkt->data = audio_encoder_buffer;
1933  pkt->size = audio_encoder_buffer_size;
1934 
1935  // Set the packet's PTS prior to encoding
1936  pkt->pts = pkt->dts = audio_timestamp;
1937 
1938  /* encode the audio samples */
1939  int got_packet_ptr = 0;
1940 
1941 #if IS_FFMPEG_3_2
1942  // Encode audio (latest version of FFmpeg)
1943  int error_code;
1944  int ret = 0;
1945  int frame_finished = 0;
1946  error_code = ret = avcodec_send_frame(audio_codec_ctx, frame_final);
1947  if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
1948  avcodec_send_frame(audio_codec_ctx, NULL);
1949  }
1950  else {
1951  if (ret >= 0)
1952  pkt->size = 0;
1953  ret = avcodec_receive_packet(audio_codec_ctx, pkt);
1954  if (ret >= 0)
1955  frame_finished = 1;
1956  if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1957  avcodec_flush_buffers(audio_codec_ctx);
1958  ret = 0;
1959  }
1960  if (ret >= 0) {
1961  ret = frame_finished;
1962  }
1963  }
1964  if (!pkt->data && !frame_finished)
1965  {
1966  ret = -1;
1967  }
1968  got_packet_ptr = ret;
1969 #else
1970  // Encode audio (older versions of FFmpeg)
1971  int error_code = avcodec_encode_audio2(audio_codec_ctx, pkt, frame_final, &got_packet_ptr);
1972 #endif
1973  /* if zero size, it means the image was buffered */
1974  if (error_code == 0 && got_packet_ptr) {
1975 
1976  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
1977  // but it fixes lots of PTS related issues when I do this.
1978  pkt->pts = pkt->dts = audio_timestamp;
1979 
1980  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
1981  av_packet_rescale_ts(pkt, audio_codec_ctx->time_base, audio_st->time_base);
1982 
1983  // set stream
1984  pkt->stream_index = audio_st->index;
1985  pkt->flags |= AV_PKT_FLAG_KEY;
1986 
1987  /* write the compressed frame in the media file */
1988  error_code = av_interleaved_write_frame(oc, pkt);
1989  }
1990 
1991  if (error_code < 0) {
1992  ZmqLogger::Instance()->AppendDebugMethod(
1993  "FFmpegWriter::write_audio_packets ERROR ["
1994  + av_err2string(error_code) + "]",
1995  "error_code", error_code);
1996  }
1997 
1998  // Increment PTS (no pkt.duration, so calculate with maths)
1999  audio_timestamp += FFMIN(audio_input_frame_size, audio_input_position);
2000 
2001  // deallocate AVFrame
2002  av_freep(&(frame_final->data[0]));
2003  AV_FREE_FRAME(&frame_final);
2004 
2005  // deallocate memory for packet
2006  AV_FREE_PACKET(pkt);
2007 
2008  // Reset position
2009  audio_input_position = 0;
2010  is_final = false;
2011  }
2012 
2013  // Delete arrays (if needed)
2014  if (all_resampled_samples) {
2015  av_freep(&all_resampled_samples);
2016  all_resampled_samples = NULL;
2017  }
2018  if (all_queued_samples) {
2019  av_freep(&all_queued_samples);
2020  all_queued_samples = NULL;
2021  }
2022 }
2023 
2024 // Allocate an AVFrame object
2025 AVFrame *FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer) {
2026  // Create an RGB AVFrame
2027  AVFrame *new_av_frame = NULL;
2028 
2029  // Allocate an AVFrame structure
2030  new_av_frame = AV_ALLOCATE_FRAME();
2031  if (new_av_frame == NULL)
2032  throw OutOfMemory("Could not allocate AVFrame", path);
2033 
2034  // Determine required buffer size and allocate buffer
2035  *buffer_size = AV_GET_IMAGE_SIZE(pix_fmt, width, height);
2036 
2037  // Create buffer (if not provided)
2038  if (!new_buffer) {
2039  // New Buffer
2040  new_buffer = (uint8_t *) av_malloc(*buffer_size * sizeof(uint8_t));
2041  // Attach buffer to AVFrame
2042  AV_COPY_PICTURE_DATA(new_av_frame, new_buffer, pix_fmt, width, height);
2043  new_av_frame->width = width;
2044  new_av_frame->height = height;
2045  new_av_frame->format = pix_fmt;
2046  }
2047 
2048  // return AVFrame
2049  return new_av_frame;
2050 }
2051 
2052 // process video frame
2053 void FFmpegWriter::process_video_packet(std::shared_ptr<Frame> frame) {
2054  // Determine the height & width of the source image
2055  int source_image_width = frame->GetWidth();
2056  int source_image_height = frame->GetHeight();
2057 
2058  // Do nothing if size is 1x1 (i.e. no image in this frame)
2059  if (source_image_height == 1 && source_image_width == 1)
2060  return;
2061 
2062  // Init rescalers (if not initialized yet)
2063  if (image_rescalers.size() == 0)
2064  InitScalers(source_image_width, source_image_height);
2065 
2066  // Get a unique rescaler (for this thread)
2067  SwsContext *scaler = image_rescalers[rescaler_position];
2068  rescaler_position++;
2069  if (rescaler_position == num_of_rescalers)
2070  rescaler_position = 0;
2071 
2072  // Allocate an RGB frame & final output frame
2073  int bytes_source = 0;
2074  int bytes_final = 0;
2075  AVFrame *frame_source = NULL;
2076  const uchar *pixels = NULL;
2077 
2078  // Get a list of pixels from source image
2079  pixels = frame->GetPixels();
2080 
2081  // Init AVFrame for source image & final (converted image)
2082  frame_source = allocate_avframe(PIX_FMT_RGBA, source_image_width, source_image_height, &bytes_source, (uint8_t *) pixels);
2083 #if IS_FFMPEG_3_2
2084  AVFrame *frame_final;
2085 #if USE_HW_ACCEL
2086  if (hw_en_on && hw_en_supported) {
2087  frame_final = allocate_avframe(AV_PIX_FMT_NV12, info.width, info.height, &bytes_final, NULL);
2088  } else
2089 #endif // USE_HW_ACCEL
2090  {
2091  frame_final = allocate_avframe(
2092  (AVPixelFormat)(video_st->codecpar->format),
2093  info.width, info.height, &bytes_final, NULL
2094  );
2095  }
2096 #else
2097  AVFrame *frame_final = allocate_avframe(video_codec_ctx->pix_fmt, info.width, info.height, &bytes_final, NULL);
2098 #endif // IS_FFMPEG_3_2
2099 
2100  // Fill with data
2101  AV_COPY_PICTURE_DATA(frame_source, (uint8_t *) pixels, PIX_FMT_RGBA, source_image_width, source_image_height);
2102  ZmqLogger::Instance()->AppendDebugMethod(
2103  "FFmpegWriter::process_video_packet",
2104  "frame->number", frame->number,
2105  "bytes_source", bytes_source,
2106  "bytes_final", bytes_final);
2107 
2108  // Resize & convert pixel format
2109  sws_scale(scaler, frame_source->data, frame_source->linesize, 0,
2110  source_image_height, frame_final->data, frame_final->linesize);
2111 
2112  // Add resized AVFrame to av_frames map
2113  add_avframe(frame, frame_final);
2114 
2115  // Deallocate memory
2116  AV_FREE_FRAME(&frame_source);
2117 }
2118 
2119 // write video frame
2120 bool FFmpegWriter::write_video_packet(std::shared_ptr<Frame> frame, AVFrame *frame_final) {
2121 #if (LIBAVFORMAT_VERSION_MAJOR >= 58)
2122  // FFmpeg 4.0+
2123  ZmqLogger::Instance()->AppendDebugMethod(
2124  "FFmpegWriter::write_video_packet",
2125  "frame->number", frame->number,
2126  "oc->oformat->flags", oc->oformat->flags);
2127 
2128  if (AV_GET_CODEC_TYPE(video_st) == AVMEDIA_TYPE_VIDEO && AV_FIND_DECODER_CODEC_ID(video_st) == AV_CODEC_ID_RAWVIDEO) {
2129 #else
2130  // TODO: Should we have moved away from oc->oformat->flags / AVFMT_RAWPICTURE
2131  // on ffmpeg < 4.0 as well?
2132  // Does AV_CODEC_ID_RAWVIDEO not work in ffmpeg 3.x?
2133  ZmqLogger::Instance()->AppendDebugMethod(
2134  "FFmpegWriter::write_video_packet",
2135  "frame->number", frame->number,
2136  "oc->oformat->flags & AVFMT_RAWPICTURE", oc->oformat->flags & AVFMT_RAWPICTURE);
2137 
2138  if (oc->oformat->flags & AVFMT_RAWPICTURE) {
2139 #endif
2140  // Raw video case.
2141 #if IS_FFMPEG_3_2
2142  AVPacket* pkt = av_packet_alloc();
2143 #else
2144  AVPacket* pkt;
2145  av_init_packet(pkt);
2146 #endif
2147 
2148  av_packet_from_data(
2149  pkt, frame_final->data[0],
2150  frame_final->linesize[0] * frame_final->height);
2151 
2152  pkt->flags |= AV_PKT_FLAG_KEY;
2153  pkt->stream_index = video_st->index;
2154 
2155  // Set PTS (in frames and scaled to the codec's timebase)
2156  pkt->pts = video_timestamp;
2157 
2158  /* write the compressed frame in the media file */
2159  int error_code = av_interleaved_write_frame(oc, pkt);
2160  if (error_code < 0) {
2161  ZmqLogger::Instance()->AppendDebugMethod(
2162  "FFmpegWriter::write_video_packet ERROR ["
2163  + av_err2string(error_code) + "]",
2164  "error_code", error_code);
2165  return false;
2166  }
2167 
2168  // Deallocate packet
2169  AV_FREE_PACKET(pkt);
2170 
2171  } else
2172  {
2173 
2174 #if IS_FFMPEG_3_2
2175  AVPacket* pkt = av_packet_alloc();
2176 #else
2177  AVPacket* pkt;
2178  av_init_packet(pkt);
2179 #endif
2180  pkt->data = NULL;
2181  pkt->size = 0;
2182  pkt->pts = pkt->dts = AV_NOPTS_VALUE;
2183 
2184  // Assign the initial AVFrame PTS from the frame counter
2185  frame_final->pts = video_timestamp;
2186 #if USE_HW_ACCEL
2187  if (hw_en_on && hw_en_supported) {
2188  if (!(hw_frame = av_frame_alloc())) {
2189  std::clog << "Error code: av_hwframe_alloc\n";
2190  }
2191  if (av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, hw_frame, 0) < 0) {
2192  std::clog << "Error code: av_hwframe_get_buffer\n";
2193  }
2194  if (!hw_frame->hw_frames_ctx) {
2195  std::clog << "Error hw_frames_ctx.\n";
2196  }
2197  hw_frame->format = AV_PIX_FMT_NV12;
2198  if ( av_hwframe_transfer_data(hw_frame, frame_final, 0) < 0) {
2199  std::clog << "Error while transferring frame data to surface.\n";
2200  }
2201  av_frame_copy_props(hw_frame, frame_final);
2202  }
2203 #endif // USE_HW_ACCEL
2204  /* encode the image */
2205  int got_packet_ptr = 0;
2206  int error_code = 0;
2207 #if IS_FFMPEG_3_2
2208  // Write video packet
2209  int ret;
2210 
2211  #if USE_HW_ACCEL
2212  if (hw_en_on && hw_en_supported) {
2213  ret = avcodec_send_frame(video_codec_ctx, hw_frame); //hw_frame!!!
2214  } else
2215  #endif // USE_HW_ACCEL
2216  {
2217  ret = avcodec_send_frame(video_codec_ctx, frame_final);
2218  }
2219  error_code = ret;
2220  if (ret < 0 ) {
2221  ZmqLogger::Instance()->AppendDebugMethod(
2222  "FFmpegWriter::write_video_packet (Frame not sent)");
2223  if (ret == AVERROR(EAGAIN) ) {
2224  std::clog << "Frame EAGAIN\n";
2225  }
2226  if (ret == AVERROR_EOF ) {
2227  std::clog << "Frame AVERROR_EOF\n";
2228  }
2229  avcodec_send_frame(video_codec_ctx, NULL);
2230  }
2231  else {
2232  while (ret >= 0) {
2233  ret = avcodec_receive_packet(video_codec_ctx, pkt);
2234 
2235  if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
2236  avcodec_flush_buffers(video_codec_ctx);
2237  got_packet_ptr = 0;
2238  break;
2239  }
2240  if (ret == 0) {
2241  got_packet_ptr = 1;
2242  break;
2243  }
2244  }
2245  }
2246 #else
2247  // Write video packet (older than FFmpeg 3.2)
2248  error_code = avcodec_encode_video2(video_codec_ctx, pkt, frame_final, &got_packet_ptr);
2249  if (error_code != 0) {
2250  ZmqLogger::Instance()->AppendDebugMethod(
2251  "FFmpegWriter::write_video_packet ERROR ["
2252  + av_err2string(error_code) + "]",
2253  "error_code", error_code);
2254  }
2255  if (got_packet_ptr == 0) {
2256  ZmqLogger::Instance()->AppendDebugMethod(
2257  "FFmpegWriter::write_video_packet (Frame gotpacket error)");
2258  }
2259 #endif // IS_FFMPEG_3_2
2260 
2261  /* if zero size, it means the image was buffered */
2262  if (error_code == 0 && got_packet_ptr) {
2263  // set the timestamp
2264  av_packet_rescale_ts(pkt, video_codec_ctx->time_base, video_st->time_base);
2265  pkt->stream_index = video_st->index;
2266 
2267  /* write the compressed frame in the media file */
2268  int result = av_interleaved_write_frame(oc, pkt);
2269  if (result < 0) {
2270  ZmqLogger::Instance()->AppendDebugMethod(
2271  "FFmpegWriter::write_video_packet ERROR ["
2272  + av_err2string(result) + "]",
2273  "result", result);
2274  return false;
2275  }
2276  }
2277 
2278  // Deallocate packet
2279  AV_FREE_PACKET(pkt);
2280 #if USE_HW_ACCEL
2281  if (hw_en_on && hw_en_supported) {
2282  if (hw_frame) {
2283  av_frame_free(&hw_frame);
2284  hw_frame = NULL;
2285  }
2286  }
2287 #endif // USE_HW_ACCEL
2288  }
2289 
2290  // Increment PTS (in frames and scaled to the codec's timebase)
2291  video_timestamp += av_rescale_q(1, av_make_q(info.fps.den, info.fps.num), video_codec_ctx->time_base);
2292 
2293  // Success
2294  return true;
2295 }
2296 
2297 // Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
2298 void FFmpegWriter::OutputStreamInfo() {
2299  // output debug info
2300  av_dump_format(oc, 0, path.c_str(), 1);
2301 }
2302 
2303 // Init a collection of software rescalers (thread safe)
2304 void FFmpegWriter::InitScalers(int source_width, int source_height) {
2305  int scale_mode = SWS_FAST_BILINEAR;
2306  if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
2307  scale_mode = SWS_BICUBIC;
2308  }
2309 
2310  // Init software rescalers vector (many of them, one for each thread)
2311  for (int x = 0; x < num_of_rescalers; x++) {
2312  // Init the software scaler from FFMpeg
2313 #if USE_HW_ACCEL
2314  if (hw_en_on && hw_en_supported) {
2315  img_convert_ctx = sws_getContext(source_width, source_height, PIX_FMT_RGBA,
2316  info.width, info.height, AV_PIX_FMT_NV12, scale_mode, NULL, NULL, NULL);
2317  } else
2318 #endif // USE_HW_ACCEL
2319  {
2320  img_convert_ctx = sws_getContext(source_width, source_height, PIX_FMT_RGBA,
2321  info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(video_st, video_st->codec),
2322  scale_mode, NULL, NULL, NULL);
2323  }
2324 
2325  // Add rescaler to vector
2326  image_rescalers.push_back(img_convert_ctx);
2327  }
2328 }
2329 
2330 // Set audio resample options
2331 void FFmpegWriter::ResampleAudio(int sample_rate, int channels) {
2332  original_sample_rate = sample_rate;
2333  original_channels = channels;
2334 }
2335 
2336 // Remove & deallocate all software scalers
2337 void FFmpegWriter::RemoveScalers() {
2338  // Close all rescalers
2339  for (int x = 0; x < num_of_rescalers; x++)
2340  sws_freeContext(image_rescalers[x]);
2341 
2342  // Clear vector
2343  image_rescalers.clear();
2344 }
AUDIO_PACKET_ENCODING_SIZE
#define AUDIO_PACKET_ENCODING_SIZE
Definition: FFmpegUtilities.h:85
Settings.h
Header file for global Settings class.
openshot::AUDIO_STREAM
@ AUDIO_STREAM
An audio stream (used to determine which type of stream)
Definition: FFmpegWriter.h:30
hw_frame
AVFrame * hw_frame
Definition: FFmpegWriter.cpp:42
openshot::InvalidFormat
Exception when no valid format is found for a file.
Definition: Exceptions.h:202
PIX_FMT_RGB24
#define PIX_FMT_RGB24
Definition: FFmpegUtilities.h:111
AV_FIND_DECODER_CODEC_ID
#define AV_FIND_DECODER_CODEC_ID(av_stream)
Definition: FFmpegUtilities.h:206
openshot::InvalidSampleRate
Exception when invalid sample rate is detected during encoding.
Definition: Exceptions.h:247
FFmpegUtilities.h
Header file for FFmpegUtilities.
openshot::WriterInfo::video_bit_rate
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: WriterBase.h:43
FFmpegWriter.h
Header file for FFmpegWriter class.
openshot::InvalidCodec
Exception when no valid codec is found for a file.
Definition: Exceptions.h:172
openshot::FFmpegWriter::ResampleAudio
void ResampleAudio(int sample_rate, int channels)
Set audio resample options.
Definition: FFmpegWriter.cpp:2331
openshot::WriterInfo::display_ratio
openshot::Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3)
Definition: WriterBase.h:45
openshot::WriterClosed
Exception when a writer is closed, and a frame is requested.
Definition: Exceptions.h:415
AV_COPY_PICTURE_DATA
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
Definition: FFmpegUtilities.h:218
openshot::FFmpegWriter::OutputStreamInfo
void OutputStreamInfo()
Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
Definition: FFmpegWriter.cpp:2298
PixelFormat
#define PixelFormat
Definition: FFmpegUtilities.h:102
openshot::ReaderBase::GetFrame
virtual std::shared_ptr< openshot::Frame > GetFrame(int64_t number)=0
AV_ALLOCATE_FRAME
#define AV_ALLOCATE_FRAME()
Definition: FFmpegUtilities.h:198
SWR_CONVERT
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Definition: FFmpegUtilities.h:144
openshot::WriterInfo::fps
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: WriterBase.h:42
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:28
AV_OPTION_SET
#define AV_OPTION_SET(av_stream, priv_data, name, value, avcodec)
Definition: FFmpegUtilities.h:223
FF_NUM_PROCESSORS
#define FF_NUM_PROCESSORS
Definition: OpenMPUtilities.h:24
openshot::WriterInfo::audio_bit_rate
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: WriterBase.h:53
openshot::WriterInfo::channels
int channels
The number of audio channels used in the audio stream.
Definition: WriterBase.h:55
AV_COPY_PARAMS_FROM_CONTEXT
AV_COPY_PARAMS_FROM_CONTEXT(st, video_codec_ctx)
openshot::Fraction
This class represents a fraction.
Definition: Fraction.h:30
AV_GET_CODEC_FROM_STREAM
#define AV_GET_CODEC_FROM_STREAM(av_stream, codec_in)
Definition: FFmpegUtilities.h:212
openshot::FFmpegWriter::FFmpegWriter
FFmpegWriter(const std::string &path)
Constructor for FFmpegWriter. Throws an exception on failure to open path.
Definition: FFmpegWriter.cpp:75
AV_FREE_FRAME
#define AV_FREE_FRAME(av_frame)
Definition: FFmpegUtilities.h:202
AV_FREE_PACKET
#define AV_FREE_PACKET(av_packet)
Definition: FFmpegUtilities.h:203
openshot::FFmpegWriter::Open
void Open()
Open writer.
Definition: FFmpegWriter.cpp:95
openshot::FFmpegWriter::SetVideoOptions
void SetVideoOptions(bool has_video, std::string codec, openshot::Fraction fps, int width, int height, openshot::Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate)
Set video export options.
Definition: FFmpegWriter.cpp:163
openshot::LAYOUT_STEREO
@ LAYOUT_STEREO
Definition: ChannelLayouts.h:31
AV_GET_CODEC_PAR_CONTEXT
#define AV_GET_CODEC_PAR_CONTEXT(av_stream, av_codec)
Definition: FFmpegUtilities.h:211
openshot::WriterInfo::width
int width
The width of the video (in pixels)
Definition: WriterBase.h:40
hw_en_on
int hw_en_on
Definition: FFmpegWriter.cpp:37
openshot::WriterInfo::acodec
std::string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: WriterBase.h:52
openshot::WriterInfo::video_timebase
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: WriterBase.h:49
openshot::LAYOUT_MONO
@ LAYOUT_MONO
Definition: ChannelLayouts.h:30
AV_GET_CODEC_ATTRIBUTES
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
Definition: FFmpegUtilities.h:213
openshot::Settings::HW_EN_DEVICE_SET
int HW_EN_DEVICE_SET
Which GPU to use to encode (0 is the first)
Definition: Settings.h:83
openshot::WriterInfo::pixel_ratio
openshot::Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square)
Definition: WriterBase.h:44
openshot::Fraction::num
int num
Numerator for the fraction.
Definition: Fraction.h:32
openshot::WriterInfo::top_field_first
bool top_field_first
Which interlaced field should be displayed first.
Definition: WriterBase.h:51
if
if(!codec) codec
AV_SET_FILENAME
#define AV_SET_FILENAME(oc, f)
Definition: FFmpegUtilities.h:196
AV_GET_IMAGE_SIZE
#define AV_GET_IMAGE_SIZE(pix_fmt, width, height)
Definition: FFmpegUtilities.h:217
ZmqLogger.h
Header file for ZeroMQ-based Logger class.
mux_dict
AVDictionary * mux_dict
Definition: FFmpegWriter.cpp:34
openshot::ErrorEncodingVideo
Exception when encoding audio packet.
Definition: Exceptions.h:142
openshot::Fraction::den
int den
Denominator for the fraction.
Definition: Fraction.h:33
openshot::FFmpegWriter::SetOption
void SetOption(openshot::StreamType stream, std::string name, std::string value)
Set custom options (some codecs accept additional params). This must be called after the PrepareStrea...
Definition: FFmpegWriter.cpp:333
openshot::Fraction::Reduce
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:65
AV_RESET_FRAME
#define AV_RESET_FRAME(av_frame)
Definition: FFmpegUtilities.h:201
SWR_CLOSE
#define SWR_CLOSE(ctx)
Definition: FFmpegUtilities.h:147
openshot::WriterInfo::channel_layout
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: WriterBase.h:56
openshot::OutOfMemory
Exception when memory could not be allocated.
Definition: Exceptions.h:348
SWR_INIT
#define SWR_INIT(ctx)
Definition: FFmpegUtilities.h:149
hw_en_av_pix_fmt
AVPixelFormat hw_en_av_pix_fmt
Definition: FFmpegWriter.cpp:39
openshot::Settings::Instance
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: Settings.cpp:23
openshot::VIDEO_STREAM
@ VIDEO_STREAM
A video stream (used to determine which type of stream)
Definition: FFmpegWriter.h:29
openshot::WriterInfo::metadata
std::map< std::string, std::string > metadata
An optional map/dictionary of video & audio metadata.
Definition: WriterBase.h:59
path
path
Definition: FFmpegWriter.cpp:1479
Frame.h
Header file for Frame class.
ALLOC_CODEC_CTX
#define ALLOC_CODEC_CTX(ctx, codec, stream)
Definition: FFmpegUtilities.h:226
openshot::InvalidFile
Exception for files that can not be found or opened.
Definition: Exceptions.h:187
openshot::ZmqLogger::Instance
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: ZmqLogger.cpp:35
openshot::FFmpegWriter::WriteFrame
void WriteFrame(std::shared_ptr< openshot::Frame > frame)
Add a frame to the stack waiting to be encoded.
Definition: FFmpegWriter.cpp:677
openshot::ZmqLogger::AppendDebugMethod
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:178
openshot::WriterInfo::has_video
bool has_video
Determines if this file has a video stream.
Definition: WriterBase.h:34
openshot::WriterInfo::has_audio
bool has_audio
Determines if this file has an audio stream.
Definition: WriterBase.h:35
PIX_FMT_YUV420P
#define PIX_FMT_YUV420P
Definition: FFmpegUtilities.h:114
PIX_FMT_YUV444P
#define PIX_FMT_YUV444P
Definition: FFmpegUtilities.h:117
AV_GET_CODEC_TYPE
#define AV_GET_CODEC_TYPE(av_stream)
Definition: FFmpegUtilities.h:205
openshot::FFmpegWriter::Close
void Close()
Close the writer.
Definition: FFmpegWriter.cpp:967
openshot::FFmpegWriter::IsValidCodec
static bool IsValidCodec(std::string codec_name)
Determine if codec name is valid.
Definition: FFmpegWriter.cpp:603
openshot::WriterInfo::height
int height
The height of the video (in pixels)
Definition: WriterBase.h:39
AV_FREE_CONTEXT
#define AV_FREE_CONTEXT(av_context)
Definition: FFmpegUtilities.h:204
PIX_FMT_RGBA
#define PIX_FMT_RGBA
Definition: FFmpegUtilities.h:105
AV_GET_CODEC_PIXEL_FORMAT
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
Definition: FFmpegUtilities.h:214
OpenMPUtilities.h
Header file for OpenMPUtilities (set some common macros)
SWR_FREE
#define SWR_FREE(ctx)
Definition: FFmpegUtilities.h:148
openshot::FFmpegWriter::WriteTrailer
void WriteTrailer()
Write the file trailer (after all frames are written). This is called automatically by the Close() me...
Definition: FFmpegWriter.cpp:754
PIX_FMT_NONE
#define PIX_FMT_NONE
Definition: FFmpegUtilities.h:108
openshot::ReaderBase
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:75
openshot::WriterInfo::interlaced_frame
bool interlaced_frame
Are the contents of this frame interlaced.
Definition: WriterBase.h:50
openshot::WriterInfo::vcodec
std::string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: WriterBase.h:46
openshot::WriterInfo::sample_rate
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: WriterBase.h:54
openshot::InvalidChannels
Exception when an invalid # of audio channels are detected.
Definition: Exceptions.h:157
AV_OPTION_FIND
#define AV_OPTION_FIND(priv_data, name)
Definition: FFmpegUtilities.h:222
codec
codec
Definition: FFmpegWriter.cpp:1475
openshot::ChannelLayout
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
Definition: ChannelLayouts.h:28
hw_en_av_device_type
AVHWDeviceType hw_en_av_device_type
Definition: FFmpegWriter.cpp:40
SWR_ALLOC
#define SWR_ALLOC()
Definition: FFmpegUtilities.h:146
openshot::FFmpegWriter::SetAudioOptions
void SetAudioOptions(bool has_audio, std::string codec, int sample_rate, int channels, openshot::ChannelLayout channel_layout, int bit_rate)
Set audio export options.
Definition: FFmpegWriter.cpp:286
AV_REGISTER_ALL
#define AV_REGISTER_ALL
Definition: FFmpegUtilities.h:193
AV_OUTPUT_CONTEXT
#define AV_OUTPUT_CONTEXT(output_context, path)
Definition: FFmpegUtilities.h:220
hw_en_supported
int hw_en_supported
Definition: FFmpegWriter.cpp:38
openshot::StreamType
StreamType
This enumeration designates the type of stream when encoding (video or audio)
Definition: FFmpegWriter.h:28
openshot::FFmpegWriter::PrepareStreams
void PrepareStreams()
Prepare & initialize streams and open codecs. This method is called automatically by the Open() metho...
Definition: FFmpegWriter.cpp:615
openshot::InvalidOptions
Exception when invalid encoding options are used.
Definition: Exceptions.h:232
openshot::WriterBase::info
WriterInfo info
Information about the current media file.
Definition: WriterBase.h:76
openshot::NoStreamsFound
Exception when no streams are found in the file.
Definition: Exceptions.h:285
openshot::FFmpegWriter::RemoveScalers
void RemoveScalers()
Remove & deallocate all software scalers.
Definition: FFmpegWriter.cpp:2337
MY_INPUT_BUFFER_PADDING_SIZE
#define MY_INPUT_BUFFER_PADDING_SIZE
Definition: FFmpegUtilities.h:197
AVCODEC_MAX_AUDIO_FRAME_SIZE
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
Definition: FFmpegUtilities.h:78
opts
AVDictionary * opts
Definition: FFmpegWriter.cpp:1486
Exceptions.h
Header file for all Exception classes.
openshot::FFmpegWriter::WriteHeader
void WriteHeader()
Write the file header (after the options are set). This method is called automatically by the Open() ...
Definition: FFmpegWriter.cpp:632
Generated by   doxygen 1.8.17