OpenShot Library | libopenshot  0.3.2
FFmpegReader.cpp
Go to the documentation of this file.
1 
12 // Copyright (c) 2008-2024 OpenShot Studios, LLC, Fabrice Bellard
13 //
14 // SPDX-License-Identifier: LGPL-3.0-or-later
15 
16 #include <thread> // for std::this_thread::sleep_for
17 #include <chrono> // for std::chrono::milliseconds
18 #include <unistd.h>
19 
20 #include "FFmpegUtilities.h"
21 
22 #include "FFmpegReader.h"
23 #include "Exceptions.h"
24 #include "Timeline.h"
25 #include "ZmqLogger.h"
26 
27 #define ENABLE_VAAPI 0
28 
29 #if USE_HW_ACCEL
30 #define MAX_SUPPORTED_WIDTH 1950
31 #define MAX_SUPPORTED_HEIGHT 1100
32 
33 #if ENABLE_VAAPI
34 #include "libavutil/hwcontext_vaapi.h"
35 
36 typedef struct VAAPIDecodeContext {
37  VAProfile va_profile;
38  VAEntrypoint va_entrypoint;
39  VAConfigID va_config;
40  VAContextID va_context;
41 
42 #if FF_API_STRUCT_VAAPI_CONTEXT
43  // FF_DISABLE_DEPRECATION_WARNINGS
44  int have_old_context;
45  struct vaapi_context *old_context;
46  AVBufferRef *device_ref;
47  // FF_ENABLE_DEPRECATION_WARNINGS
48 #endif
49 
50  AVHWDeviceContext *device;
51  AVVAAPIDeviceContext *hwctx;
52 
53  AVHWFramesContext *frames;
54  AVVAAPIFramesContext *hwfc;
55 
56  enum AVPixelFormat surface_format;
57  int surface_count;
58  } VAAPIDecodeContext;
59 #endif // ENABLE_VAAPI
60 #endif // USE_HW_ACCEL
61 
62 
63 using namespace openshot;
64 
65 int hw_de_on = 0;
66 #if USE_HW_ACCEL
67  AVPixelFormat hw_de_av_pix_fmt_global = AV_PIX_FMT_NONE;
68  AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE;
69 #endif
70 
71 FFmpegReader::FFmpegReader(const std::string &path, bool inspect_reader)
72  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0), NO_PTS_OFFSET(-99999),
73  path(path), is_video_seek(true), check_interlace(false), check_fps(false), enable_seek(true), is_open(false),
74  seek_audio_frame_found(0), seek_video_frame_found(0),is_duration_known(false), largest_frame_processed(0),
75  current_video_frame(0), packet(NULL), max_concurrent_frames(OPEN_MP_NUM_PROCESSORS), audio_pts(0),
76  video_pts(0), pFormatCtx(NULL), videoStream(-1), audioStream(-1), pCodecCtx(NULL), aCodecCtx(NULL),
77  pStream(NULL), aStream(NULL), pFrame(NULL), previous_packet_location{-1,0},
78  hold_packet(false) {
79 
80  // Initialize FFMpeg, and register all formats and codecs
83 
84  // Init timestamp offsets
85  pts_offset_seconds = NO_PTS_OFFSET;
86  video_pts_seconds = NO_PTS_OFFSET;
87  audio_pts_seconds = NO_PTS_OFFSET;
88 
89  // Init cache
90  working_cache.SetMaxBytesFromInfo(max_concurrent_frames * info.fps.ToDouble() * 2, info.width, info.height, info.sample_rate, info.channels);
91  final_cache.SetMaxBytesFromInfo(max_concurrent_frames * 2, info.width, info.height, info.sample_rate, info.channels);
92 
93  // Open and Close the reader, to populate its attributes (such as height, width, etc...)
94  if (inspect_reader) {
95  Open();
96  Close();
97  }
98 }
99 
101  if (is_open)
102  // Auto close reader if not already done
103  Close();
104 }
105 
106 // This struct holds the associated video frame and starting sample # for an audio packet.
107 bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount) {
108  // Is frame even close to this one?
109  if (abs(location.frame - frame) >= 2)
110  // This is too far away to be considered
111  return false;
112 
113  // Note that samples_per_frame can vary slightly frame to frame when the
114  // audio sampling rate is not an integer multiple of the video fps.
115  int64_t diff = samples_per_frame * (location.frame - frame) + location.sample_start - sample_start;
116  if (abs(diff) <= amount)
117  // close
118  return true;
119 
120  // not close
121  return false;
122 }
123 
124 #if USE_HW_ACCEL
125 
126 // Get hardware pix format
127 static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts)
128 {
129  const enum AVPixelFormat *p;
130 
131  for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) {
132  switch (*p) {
133 #if defined(__linux__)
134  // Linux pix formats
135  case AV_PIX_FMT_VAAPI:
136  hw_de_av_pix_fmt_global = AV_PIX_FMT_VAAPI;
137  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VAAPI;
138  return *p;
139  break;
140  case AV_PIX_FMT_VDPAU:
141  hw_de_av_pix_fmt_global = AV_PIX_FMT_VDPAU;
142  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VDPAU;
143  return *p;
144  break;
145 #endif
146 #if defined(_WIN32)
147  // Windows pix formats
148  case AV_PIX_FMT_DXVA2_VLD:
149  hw_de_av_pix_fmt_global = AV_PIX_FMT_DXVA2_VLD;
150  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_DXVA2;
151  return *p;
152  break;
153  case AV_PIX_FMT_D3D11:
154  hw_de_av_pix_fmt_global = AV_PIX_FMT_D3D11;
155  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_D3D11VA;
156  return *p;
157  break;
158 #endif
159 #if defined(__APPLE__)
160  // Apple pix formats
161  case AV_PIX_FMT_VIDEOTOOLBOX:
162  hw_de_av_pix_fmt_global = AV_PIX_FMT_VIDEOTOOLBOX;
163  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
164  return *p;
165  break;
166 #endif
167  // Cross-platform pix formats
168  case AV_PIX_FMT_CUDA:
169  hw_de_av_pix_fmt_global = AV_PIX_FMT_CUDA;
170  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_CUDA;
171  return *p;
172  break;
173  case AV_PIX_FMT_QSV:
174  hw_de_av_pix_fmt_global = AV_PIX_FMT_QSV;
175  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_QSV;
176  return *p;
177  break;
178  default:
179  // This is only here to silence unused-enum warnings
180  break;
181  }
182  }
183  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::get_hw_dec_format (Unable to decode this file using hardware decode)");
184  return AV_PIX_FMT_NONE;
185 }
186 
187 int FFmpegReader::IsHardwareDecodeSupported(int codecid)
188 {
189  int ret;
190  switch (codecid) {
191  case AV_CODEC_ID_H264:
192  case AV_CODEC_ID_MPEG2VIDEO:
193  case AV_CODEC_ID_VC1:
194  case AV_CODEC_ID_WMV1:
195  case AV_CODEC_ID_WMV2:
196  case AV_CODEC_ID_WMV3:
197  ret = 1;
198  break;
199  default :
200  ret = 0;
201  break;
202  }
203  return ret;
204 }
205 #endif // USE_HW_ACCEL
206 
208  // Open reader if not already open
209  if (!is_open) {
210  // Prevent async calls to the following code
211  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
212 
213  // Initialize format context
214  pFormatCtx = NULL;
215  {
217  ZmqLogger::Instance()->AppendDebugMethod("Decode hardware acceleration settings", "hw_de_on", hw_de_on, "HARDWARE_DECODER", openshot::Settings::Instance()->HARDWARE_DECODER);
218  }
219 
220  // Open video file
221  if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
222  throw InvalidFile("File could not be opened.", path);
223 
224  // Retrieve stream information
225  if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
226  throw NoStreamsFound("No streams found in file.", path);
227 
228  videoStream = -1;
229  audioStream = -1;
230 
231  // Init end-of-file detection variables
232  packet_status.reset(true);
233 
234  // Loop through each stream, and identify the video and audio stream index
235  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
236  // Is this a video stream?
237  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
238  videoStream = i;
239  packet_status.video_eof = false;
240  packet_status.packets_eof = false;
241  packet_status.end_of_file = false;
242  }
243  // Is this an audio stream?
244  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
245  audioStream = i;
246  packet_status.audio_eof = false;
247  packet_status.packets_eof = false;
248  packet_status.end_of_file = false;
249  }
250  }
251  if (videoStream == -1 && audioStream == -1)
252  throw NoStreamsFound("No video or audio streams found in this file.", path);
253 
254  // Is there a video stream?
255  if (videoStream != -1) {
256  // Set the stream index
257  info.video_stream_index = videoStream;
258 
259  // Set the codec and codec context pointers
260  pStream = pFormatCtx->streams[videoStream];
261 
262  // Find the codec ID from stream
263  const AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(pStream);
264 
265  // Get codec and codec context from stream
266  const AVCodec *pCodec = avcodec_find_decoder(codecId);
267  AVDictionary *opts = NULL;
268  int retry_decode_open = 2;
269  // If hw accel is selected but hardware cannot handle repeat with software decoding
270  do {
271  pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec);
272 #if USE_HW_ACCEL
273  if (hw_de_on && (retry_decode_open==2)) {
274  // Up to here no decision is made if hardware or software decode
275  hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id);
276  }
277 #endif
278  retry_decode_open = 0;
279 
280  // Set number of threads equal to number of processors (not to exceed 16)
281  pCodecCtx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
282 
283  if (pCodec == NULL) {
284  throw InvalidCodec("A valid video codec could not be found for this file.", path);
285  }
286 
287  // Init options
288  av_dict_set(&opts, "strict", "experimental", 0);
289 #if USE_HW_ACCEL
290  if (hw_de_on && hw_de_supported) {
291  // Open Hardware Acceleration
292  int i_decoder_hw = 0;
293  char adapter[256];
294  char *adapter_ptr = NULL;
295  int adapter_num;
297  fprintf(stderr, "Hardware decoding device number: %d\n", adapter_num);
298 
299  // Set hardware pix format (callback)
300  pCodecCtx->get_format = get_hw_dec_format;
301 
302  if (adapter_num < 3 && adapter_num >=0) {
303 #if defined(__linux__)
304  snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128);
305  adapter_ptr = adapter;
307  switch (i_decoder_hw) {
308  case 1:
309  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
310  break;
311  case 2:
312  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
313  break;
314  case 6:
315  hw_de_av_device_type = AV_HWDEVICE_TYPE_VDPAU;
316  break;
317  case 7:
318  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
319  break;
320  default:
321  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
322  break;
323  }
324 
325 #elif defined(_WIN32)
326  adapter_ptr = NULL;
328  switch (i_decoder_hw) {
329  case 2:
330  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
331  break;
332  case 3:
333  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
334  break;
335  case 4:
336  hw_de_av_device_type = AV_HWDEVICE_TYPE_D3D11VA;
337  break;
338  case 7:
339  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
340  break;
341  default:
342  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
343  break;
344  }
345 #elif defined(__APPLE__)
346  adapter_ptr = NULL;
348  switch (i_decoder_hw) {
349  case 5:
350  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
351  break;
352  case 7:
353  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
354  break;
355  default:
356  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
357  break;
358  }
359 #endif
360 
361  } else {
362  adapter_ptr = NULL; // Just to be sure
363  }
364 
365  // Check if it is there and writable
366 #if defined(__linux__)
367  if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) {
368 #elif defined(_WIN32)
369  if( adapter_ptr != NULL ) {
370 #elif defined(__APPLE__)
371  if( adapter_ptr != NULL ) {
372 #endif
373  ZmqLogger::Instance()->AppendDebugMethod("Decode Device present using device");
374  }
375  else {
376  adapter_ptr = NULL; // use default
377  ZmqLogger::Instance()->AppendDebugMethod("Decode Device not present using default");
378  }
379 
380  hw_device_ctx = NULL;
381  // Here the first hardware initialisations are made
382  if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) {
383  if (!(pCodecCtx->hw_device_ctx = av_buffer_ref(hw_device_ctx))) {
384  throw InvalidCodec("Hardware device reference create failed.", path);
385  }
386 
387  /*
388  av_buffer_unref(&ist->hw_frames_ctx);
389  ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx);
390  if (!ist->hw_frames_ctx) {
391  av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA frames context\n");
392  return AVERROR(ENOMEM);
393  }
394 
395  frames_ctx = (AVHWFramesContext*)ist->hw_frames_ctx->data;
396 
397  frames_ctx->format = AV_PIX_FMT_CUDA;
398  frames_ctx->sw_format = avctx->sw_pix_fmt;
399  frames_ctx->width = avctx->width;
400  frames_ctx->height = avctx->height;
401 
402  av_log(avctx, AV_LOG_DEBUG, "Initializing CUDA frames context: sw_format = %s, width = %d, height = %d\n",
403  av_get_pix_fmt_name(frames_ctx->sw_format), frames_ctx->width, frames_ctx->height);
404 
405 
406  ret = av_hwframe_ctx_init(pCodecCtx->hw_device_ctx);
407  ret = av_hwframe_ctx_init(ist->hw_frames_ctx);
408  if (ret < 0) {
409  av_log(avctx, AV_LOG_ERROR, "Error initializing a CUDA frame pool\n");
410  return ret;
411  }
412  */
413  }
414  else {
415  throw InvalidCodec("Hardware device create failed.", path);
416  }
417  }
418 #endif // USE_HW_ACCEL
419 
420  // Disable per-frame threading for album arts
421  // Using FF_THREAD_FRAME adds one frame decoding delay per thread,
422  // but there's only one frame in this case.
423  if (HasAlbumArt())
424  {
425  pCodecCtx->thread_type &= ~FF_THREAD_FRAME;
426  }
427 
428  // Open video codec
429  int avcodec_return = avcodec_open2(pCodecCtx, pCodec, &opts);
430  if (avcodec_return < 0) {
431  std::stringstream avcodec_error_msg;
432  avcodec_error_msg << "A video codec was found, but could not be opened. Error: " << av_err2string(avcodec_return);
433  throw InvalidCodec(avcodec_error_msg.str(), path);
434  }
435 
436 #if USE_HW_ACCEL
437  if (hw_de_on && hw_de_supported) {
438  AVHWFramesConstraints *constraints = NULL;
439  void *hwconfig = NULL;
440  hwconfig = av_hwdevice_hwconfig_alloc(hw_device_ctx);
441 
442 // TODO: needs va_config!
443 #if ENABLE_VAAPI
444  ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config;
445  constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig);
446 #endif // ENABLE_VAAPI
447  if (constraints) {
448  if (pCodecCtx->coded_width < constraints->min_width ||
449  pCodecCtx->coded_height < constraints->min_height ||
450  pCodecCtx->coded_width > constraints->max_width ||
451  pCodecCtx->coded_height > constraints->max_height) {
452  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n");
453  hw_de_supported = 0;
454  retry_decode_open = 1;
455  AV_FREE_CONTEXT(pCodecCtx);
456  if (hw_device_ctx) {
457  av_buffer_unref(&hw_device_ctx);
458  hw_device_ctx = NULL;
459  }
460  }
461  else {
462  // All is just peachy
463  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Min width :", constraints->min_width, "Min Height :", constraints->min_height, "MaxWidth :", constraints->max_width, "MaxHeight :", constraints->max_height, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
464  retry_decode_open = 0;
465  }
466  av_hwframe_constraints_free(&constraints);
467  if (hwconfig) {
468  av_freep(&hwconfig);
469  }
470  }
471  else {
472  int max_h, max_w;
473  //max_h = ((getenv( "LIMIT_HEIGHT_MAX" )==NULL) ? MAX_SUPPORTED_HEIGHT : atoi(getenv( "LIMIT_HEIGHT_MAX" )));
475  //max_w = ((getenv( "LIMIT_WIDTH_MAX" )==NULL) ? MAX_SUPPORTED_WIDTH : atoi(getenv( "LIMIT_WIDTH_MAX" )));
477  ZmqLogger::Instance()->AppendDebugMethod("Constraints could not be found using default limit\n");
478  //cerr << "Constraints could not be found using default limit\n";
479  if (pCodecCtx->coded_width < 0 ||
480  pCodecCtx->coded_height < 0 ||
481  pCodecCtx->coded_width > max_w ||
482  pCodecCtx->coded_height > max_h ) {
483  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
484  hw_de_supported = 0;
485  retry_decode_open = 1;
486  AV_FREE_CONTEXT(pCodecCtx);
487  if (hw_device_ctx) {
488  av_buffer_unref(&hw_device_ctx);
489  hw_device_ctx = NULL;
490  }
491  }
492  else {
493  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
494  retry_decode_open = 0;
495  }
496  }
497  } // if hw_de_on && hw_de_supported
498  else {
499  ZmqLogger::Instance()->AppendDebugMethod("\nDecode in software is used\n");
500  }
501 #else
502  retry_decode_open = 0;
503 #endif // USE_HW_ACCEL
504  } while (retry_decode_open); // retry_decode_open
505  // Free options
506  av_dict_free(&opts);
507 
508  // Update the File Info struct with video details (if a video stream is found)
509  UpdateVideoInfo();
510  }
511 
512  // Is there an audio stream?
513  if (audioStream != -1) {
514  // Set the stream index
515  info.audio_stream_index = audioStream;
516 
517  // Get a pointer to the codec context for the audio stream
518  aStream = pFormatCtx->streams[audioStream];
519 
520  // Find the codec ID from stream
521  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(aStream);
522 
523  // Get codec and codec context from stream
524  const AVCodec *aCodec = avcodec_find_decoder(codecId);
525  aCodecCtx = AV_GET_CODEC_CONTEXT(aStream, aCodec);
526 
527  // Set number of threads equal to number of processors (not to exceed 16)
528  aCodecCtx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
529 
530  if (aCodec == NULL) {
531  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
532  }
533 
534  // Init options
535  AVDictionary *opts = NULL;
536  av_dict_set(&opts, "strict", "experimental", 0);
537 
538  // Open audio codec
539  if (avcodec_open2(aCodecCtx, aCodec, &opts) < 0)
540  throw InvalidCodec("An audio codec was found, but could not be opened.", path);
541 
542  // Free options
543  av_dict_free(&opts);
544 
545  // Update the File Info struct with audio details (if an audio stream is found)
546  UpdateAudioInfo();
547  }
548 
549  // Add format metadata (if any)
550  AVDictionaryEntry *tag = NULL;
551  while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
552  QString str_key = tag->key;
553  QString str_value = tag->value;
554  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
555  }
556 
557  // Init previous audio location to zero
558  previous_packet_location.frame = -1;
559  previous_packet_location.sample_start = 0;
560 
561  // Adjust cache size based on size of frame and audio
562  working_cache.SetMaxBytesFromInfo(max_concurrent_frames * info.fps.ToDouble() * 2, info.width, info.height, info.sample_rate, info.channels);
564 
565  // Scan PTS for any offsets (i.e. non-zero starting streams). At least 1 stream must start at zero timestamp.
566  // This method allows us to shift timestamps to ensure at least 1 stream is starting at zero.
567  UpdatePTSOffset();
568 
569  // Override an invalid framerate
570  if (info.fps.ToFloat() > 240.0f || (info.fps.num <= 0 || info.fps.den <= 0) || info.video_length <= 0) {
571  // Calculate FPS, duration, video bit rate, and video length manually
572  // by scanning through all the video stream packets
573  CheckFPS();
574  }
575 
576  // Mark as "open"
577  is_open = true;
578 
579  // Seek back to beginning of file (if not already seeking)
580  if (!is_seeking) {
581  Seek(1);
582  }
583  }
584 }
585 
587  // Close all objects, if reader is 'open'
588  if (is_open) {
589  // Prevent async calls to the following code
590  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
591 
592  // Mark as "closed"
593  is_open = false;
594 
595  // Keep track of most recent packet
596  AVPacket *recent_packet = packet;
597 
598  // Drain any packets from the decoder
599  packet = NULL;
600  int attempts = 0;
601  int max_attempts = 128;
602  while (packet_status.packets_decoded() < packet_status.packets_read() && attempts < max_attempts) {
603  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close (Drain decoder loop)",
604  "packets_read", packet_status.packets_read(),
605  "packets_decoded", packet_status.packets_decoded(),
606  "attempts", attempts);
607  if (packet_status.video_decoded < packet_status.video_read) {
608  ProcessVideoPacket(info.video_length);
609  }
610  if (packet_status.audio_decoded < packet_status.audio_read) {
611  ProcessAudioPacket(info.video_length);
612  }
613  attempts++;
614  }
615 
616  // Remove packet
617  if (recent_packet) {
618  RemoveAVPacket(recent_packet);
619  }
620 
621  // Close the video codec
622  if (info.has_video) {
623  if(avcodec_is_open(pCodecCtx)) {
624  avcodec_flush_buffers(pCodecCtx);
625  }
626  AV_FREE_CONTEXT(pCodecCtx);
627 #if USE_HW_ACCEL
628  if (hw_de_on) {
629  if (hw_device_ctx) {
630  av_buffer_unref(&hw_device_ctx);
631  hw_device_ctx = NULL;
632  }
633  }
634 #endif // USE_HW_ACCEL
635  }
636 
637  // Close the audio codec
638  if (info.has_audio) {
639  if(avcodec_is_open(aCodecCtx)) {
640  avcodec_flush_buffers(aCodecCtx);
641  }
642  AV_FREE_CONTEXT(aCodecCtx);
643  }
644 
645  // Clear final cache
646  final_cache.Clear();
647  working_cache.Clear();
648 
649  // Close the video file
650  avformat_close_input(&pFormatCtx);
651  av_freep(&pFormatCtx);
652 
653  // Reset some variables
654  last_frame = 0;
655  hold_packet = false;
656  largest_frame_processed = 0;
657  seek_audio_frame_found = 0;
658  seek_video_frame_found = 0;
659  current_video_frame = 0;
660  last_video_frame.reset();
661  }
662 }
663 
664 bool FFmpegReader::HasAlbumArt() {
665  // Check if the video stream we use is an attached picture
666  // This won't return true if the file has a cover image as a secondary stream
667  // like an MKV file with an attached image file
668  return pFormatCtx && videoStream >= 0 && pFormatCtx->streams[videoStream]
669  && (pFormatCtx->streams[videoStream]->disposition & AV_DISPOSITION_ATTACHED_PIC);
670 }
671 
672 void FFmpegReader::UpdateAudioInfo() {
673  // Set default audio channel layout (if needed)
674 #if HAVE_CH_LAYOUT
675  if (!av_channel_layout_check(&(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout)))
676  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout = (AVChannelLayout) AV_CHANNEL_LAYOUT_STEREO;
677 #else
678  if (AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
679  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels);
680 #endif
681 
682  if (info.sample_rate > 0) {
683  // Skip init - if info struct already populated
684  return;
685  }
686 
687  // Set values of FileInfo struct
688  info.has_audio = true;
689  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
690  info.acodec = aCodecCtx->codec->name;
691 #if HAVE_CH_LAYOUT
692  info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout.nb_channels;
693  info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout.u.mask;
694 #else
695  info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
696  info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout;
697 #endif
698  info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate;
699  info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate;
700  if (info.audio_bit_rate <= 0) {
701  // Get bitrate from format
702  info.audio_bit_rate = pFormatCtx->bit_rate;
703  }
704 
705  // Set audio timebase
706  info.audio_timebase.num = aStream->time_base.num;
707  info.audio_timebase.den = aStream->time_base.den;
708 
709  // Get timebase of audio stream (if valid) and greater than the current duration
710  if (aStream->duration > 0 && aStream->duration > info.duration) {
711  // Get duration from audio stream
712  info.duration = aStream->duration * info.audio_timebase.ToDouble();
713  } else if (pFormatCtx->duration > 0 && info.duration <= 0.0f) {
714  // Use the format's duration
715  info.duration = float(pFormatCtx->duration) / AV_TIME_BASE;
716  }
717 
718  // Calculate duration from filesize and bitrate (if any)
719  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0) {
720  // Estimate from bitrate, total bytes, and framerate
722  }
723 
724  // Check for an invalid video length
725  if (info.has_video && info.video_length <= 0) {
726  // Calculate the video length from the audio duration
728  }
729 
730  // Set video timebase (if no video stream was found)
731  if (!info.has_video) {
732  // Set a few important default video settings (so audio can be divided into frames)
733  info.fps.num = 24;
734  info.fps.den = 1;
735  info.video_timebase.num = 1;
736  info.video_timebase.den = 24;
738  info.width = 720;
739  info.height = 480;
740 
741  // Use timeline to set correct width & height (if any)
742  Clip *parent = static_cast<Clip *>(ParentClip());
743  if (parent) {
744  if (parent->ParentTimeline()) {
745  // Set max width/height based on parent clip's timeline (if attached to a timeline)
746  info.width = parent->ParentTimeline()->preview_width;
747  info.height = parent->ParentTimeline()->preview_height;
748  }
749  }
750  }
751 
752  // Fix invalid video lengths for certain types of files (MP3 for example)
753  if (info.has_video && ((info.duration * info.fps.ToDouble()) - info.video_length > 60)) {
755  }
756 
757  // Add audio metadata (if any found)
758  AVDictionaryEntry *tag = NULL;
759  while ((tag = av_dict_get(aStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
760  QString str_key = tag->key;
761  QString str_value = tag->value;
762  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
763  }
764 }
765 
766 void FFmpegReader::UpdateVideoInfo() {
767  if (info.vcodec.length() > 0) {
768  // Skip init - if info struct already populated
769  return;
770  }
771 
772  // Set values of FileInfo struct
773  info.has_video = true;
774  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
775  info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
776  info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
777  info.vcodec = pCodecCtx->codec->name;
778  info.video_bit_rate = (pFormatCtx->bit_rate / 8);
779 
780  // Frame rate from the container and codec
781  AVRational framerate = av_guess_frame_rate(pFormatCtx, pStream, NULL);
782  if (!check_fps) {
783  info.fps.num = framerate.num;
784  info.fps.den = framerate.den;
785  }
786 
787  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo", "info.fps.num", info.fps.num, "info.fps.den", info.fps.den);
788 
789  // TODO: remove excessive debug info in the next releases
790  // The debug info below is just for comparison and troubleshooting on users side during the transition period
791  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo (pStream->avg_frame_rate)", "num", pStream->avg_frame_rate.num, "den", pStream->avg_frame_rate.den);
792 
793  if (pStream->sample_aspect_ratio.num != 0) {
794  info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
795  info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
796  } else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0) {
797  info.pixel_ratio.num = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num;
798  info.pixel_ratio.den = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.den;
799  } else {
800  info.pixel_ratio.num = 1;
801  info.pixel_ratio.den = 1;
802  }
803  info.pixel_format = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
804 
805  // Calculate the DAR (display aspect ratio)
807 
808  // Reduce size fraction
809  size.Reduce();
810 
811  // Set the ratio based on the reduced fraction
812  info.display_ratio.num = size.num;
813  info.display_ratio.den = size.den;
814 
815  // Get scan type and order from codec context/params
816  if (!check_interlace) {
817  check_interlace = true;
818  AVFieldOrder field_order = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->field_order;
819  switch(field_order) {
820  case AV_FIELD_PROGRESSIVE:
821  info.interlaced_frame = false;
822  break;
823  case AV_FIELD_TT:
824  case AV_FIELD_TB:
825  info.interlaced_frame = true;
826  info.top_field_first = true;
827  break;
828  case AV_FIELD_BT:
829  case AV_FIELD_BB:
830  info.interlaced_frame = true;
831  info.top_field_first = false;
832  break;
833  case AV_FIELD_UNKNOWN:
834  // Check again later?
835  check_interlace = false;
836  break;
837  }
838  // check_interlace will prevent these checks being repeated,
839  // unless it was cleared because we got an AV_FIELD_UNKNOWN response.
840  }
841 
842  // Set the video timebase
843  info.video_timebase.num = pStream->time_base.num;
844  info.video_timebase.den = pStream->time_base.den;
845 
846  // Set the duration in seconds, and video length (# of frames)
847  info.duration = pStream->duration * info.video_timebase.ToDouble();
848 
849  // Check for valid duration (if found)
850  if (info.duration <= 0.0f && pFormatCtx->duration >= 0) {
851  // Use the format's duration
852  info.duration = float(pFormatCtx->duration) / AV_TIME_BASE;
853  }
854 
855  // Calculate duration from filesize and bitrate (if any)
856  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0) {
857  // Estimate from bitrate, total bytes, and framerate
859  }
860 
861  // Certain "image" formats do not have a valid duration
862  if (info.duration <= 0.0f && pStream->duration == AV_NOPTS_VALUE && pFormatCtx->duration == AV_NOPTS_VALUE) {
863  // Force an "image" duration
864  info.duration = 60 * 60 * 1; // 1 hour duration
865  info.video_length = 1;
866  info.has_single_image = true;
867  }
868 
869  // Get the # of video frames (if found in stream)
870  // Only set this 1 time (this method can be called multiple times)
871  if (pStream->nb_frames > 0 && info.video_length <= 0) {
872  info.video_length = pStream->nb_frames;
873  }
874 
875  // No duration found in stream of file
876  if (info.duration <= 0.0f) {
877  // No duration is found in the video stream
878  info.duration = -1;
879  info.video_length = -1;
880  is_duration_known = false;
881  } else {
882  // Yes, a duration was found
883  is_duration_known = true;
884 
885  // Calculate number of frames (if not already found in metadata)
886  // Only set this 1 time (this method can be called multiple times)
887  if (info.video_length <= 0) {
889  }
890  }
891 
892  // Add video metadata (if any)
893  AVDictionaryEntry *tag = NULL;
894  while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
895  QString str_key = tag->key;
896  QString str_value = tag->value;
897  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
898  }
899 }
900 
902  return this->is_duration_known;
903 }
904 
905 std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame) {
906  // Check for open reader (or throw exception)
907  if (!is_open)
908  throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
909 
910  // Adjust for a requested frame that is too small or too large
911  if (requested_frame < 1)
912  requested_frame = 1;
913  if (requested_frame > info.video_length && is_duration_known)
914  requested_frame = info.video_length;
915  if (info.has_video && info.video_length == 0)
916  // Invalid duration of video file
917  throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
918 
919  // Debug output
920  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame);
921 
922  // Check the cache for this frame
923  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
924  if (frame) {
925  // Debug output
926  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame);
927 
928  // Return the cached frame
929  return frame;
930  } else {
931 
932  // Prevent async calls to the remainder of this code
933  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
934 
935  // Check the cache a 2nd time (due to the potential previous lock)
936  frame = final_cache.GetFrame(requested_frame);
937  if (frame) {
938  // Debug output
939  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame);
940 
941  } else {
942  // Frame is not in cache
943  // Reset seek count
944  seek_count = 0;
945 
946  // Are we within X frames of the requested frame?
947  int64_t diff = requested_frame - last_frame;
948  if (diff >= 1 && diff <= 20) {
949  // Continue walking the stream
950  frame = ReadStream(requested_frame);
951  } else {
952  // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
953  if (enable_seek) {
954  // Only seek if enabled
955  Seek(requested_frame);
956 
957  } else if (!enable_seek && diff < 0) {
958  // Start over, since we can't seek, and the requested frame is smaller than our position
959  // Since we are seeking to frame 1, this actually just closes/re-opens the reader
960  Seek(1);
961  }
962 
963  // Then continue walking the stream
964  frame = ReadStream(requested_frame);
965  }
966  }
967  return frame;
968  }
969 }
970 
971 // Read the stream until we find the requested Frame
972 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame) {
973  // Allocate video frame
974  bool check_seek = false;
975  int packet_error = -1;
976 
977  // Debug output
978  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame, "max_concurrent_frames", max_concurrent_frames);
979 
980  // Loop through the stream until the correct frame is found
981  while (true) {
982  // Check if working frames are 'finished'
983  if (!is_seeking) {
984  // Check for final frames
985  CheckWorkingFrames(requested_frame);
986  }
987 
988  // Check if requested 'final' frame is available (and break out of loop if found)
989  bool is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
990  if (is_cache_found) {
991  break;
992  }
993 
994  if (!hold_packet || !packet) {
995  // Get the next packet
996  packet_error = GetNextPacket();
997  if (packet_error < 0 && !packet) {
998  // No more packets to be found
999  packet_status.packets_eof = true;
1000  }
1001  }
1002 
1003  // Debug output
1004  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame,"packets_read", packet_status.packets_read(), "packets_decoded", packet_status.packets_decoded(), "is_seeking", is_seeking);
1005 
1006  // Check the status of a seek (if any)
1007  if (is_seeking) {
1008  check_seek = CheckSeek(false);
1009  } else {
1010  check_seek = false;
1011  }
1012 
1013  if (check_seek) {
1014  // Packet may become NULL on Close inside Seek if CheckSeek returns false
1015  // Jump to the next iteration of this loop
1016  continue;
1017  }
1018 
1019  // Video packet
1020  if ((info.has_video && packet && packet->stream_index == videoStream) ||
1021  (info.has_video && packet_status.video_decoded < packet_status.video_read) ||
1022  (info.has_video && !packet && !packet_status.video_eof)) {
1023  // Process Video Packet
1024  ProcessVideoPacket(requested_frame);
1025  }
1026  // Audio packet
1027  if ((info.has_audio && packet && packet->stream_index == audioStream) ||
1028  (info.has_audio && !packet && packet_status.audio_decoded < packet_status.audio_read) ||
1029  (info.has_audio && !packet && !packet_status.audio_eof)) {
1030  // Process Audio Packet
1031  ProcessAudioPacket(requested_frame);
1032  }
1033 
1034  // Remove unused packets (sometimes we purposely ignore video or audio packets,
1035  // if the has_video or has_audio properties are manually overridden)
1036  if ((!info.has_video && packet && packet->stream_index == videoStream) ||
1037  (!info.has_audio && packet && packet->stream_index == audioStream)) {
1038  // Keep track of deleted packet counts
1039  if (packet->stream_index == videoStream) {
1040  packet_status.video_decoded++;
1041  } else if (packet->stream_index == audioStream) {
1042  packet_status.audio_decoded++;
1043  }
1044 
1045  // Remove unused packets (sometimes we purposely ignore video or audio packets,
1046  // if the has_video or has_audio properties are manually overridden)
1047  RemoveAVPacket(packet);
1048  packet = NULL;
1049  }
1050 
1051  // Determine end-of-stream (waiting until final decoder threads finish)
1052  // Force end-of-stream in some situations
1053  packet_status.end_of_file = packet_status.packets_eof && packet_status.video_eof && packet_status.audio_eof;
1054  if ((packet_status.packets_eof && packet_status.packets_read() == packet_status.packets_decoded()) || packet_status.end_of_file) {
1055  // Force EOF (end of file) variables to true, if decoder does not support EOF detection.
1056  // If we have no more packets, and all known packets have been decoded
1057  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (force EOF)", "packets_read", packet_status.packets_read(), "packets_decoded", packet_status.packets_decoded(), "packets_eof", packet_status.packets_eof, "video_eof", packet_status.video_eof, "audio_eof", packet_status.audio_eof, "end_of_file", packet_status.end_of_file);
1058  if (!packet_status.video_eof) {
1059  packet_status.video_eof = true;
1060  }
1061  if (!packet_status.audio_eof) {
1062  packet_status.audio_eof = true;
1063  }
1064  packet_status.end_of_file = true;
1065  break;
1066  }
1067  } // end while
1068 
1069  // Debug output
1070  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)",
1071  "packets_read", packet_status.packets_read(),
1072  "packets_decoded", packet_status.packets_decoded(),
1073  "end_of_file", packet_status.end_of_file,
1074  "largest_frame_processed", largest_frame_processed,
1075  "Working Cache Count", working_cache.Count());
1076 
1077  // Have we reached end-of-stream (or the final frame)?
1078  if (!packet_status.end_of_file && requested_frame >= info.video_length) {
1079  // Force end-of-stream
1080  packet_status.end_of_file = true;
1081  }
1082  if (packet_status.end_of_file) {
1083  // Mark any other working frames as 'finished'
1084  CheckWorkingFrames(requested_frame);
1085  }
1086 
1087  // Return requested frame (if found)
1088  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
1089  if (frame)
1090  // Return prepared frame
1091  return frame;
1092  else {
1093 
1094  // Check if largest frame is still cached
1095  frame = final_cache.GetFrame(largest_frame_processed);
1096  int samples_in_frame = Frame::GetSamplesPerFrame(requested_frame, info.fps,
1098  if (frame) {
1099  // Copy and return the largest processed frame (assuming it was the last in the video file)
1100  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1101 
1102  // Use solid color (if no image data found)
1103  if (!frame->has_image_data) {
1104  // Use solid black frame if no image data available
1105  f->AddColor(info.width, info.height, "#000");
1106  }
1107  // Silence audio data (if any), since we are repeating the last frame
1108  frame->AddAudioSilence(samples_in_frame);
1109 
1110  return frame;
1111  } else {
1112  // The largest processed frame is no longer in cache, return a blank frame
1113  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1114  f->AddColor(info.width, info.height, "#000");
1115  f->AddAudioSilence(samples_in_frame);
1116  return f;
1117  }
1118  }
1119 
1120 }
1121 
1122 // Get the next packet (if any)
1123 int FFmpegReader::GetNextPacket() {
1124  int found_packet = 0;
1125  AVPacket *next_packet;
1126  next_packet = new AVPacket();
1127  found_packet = av_read_frame(pFormatCtx, next_packet);
1128 
1129  if (packet) {
1130  // Remove previous packet before getting next one
1131  RemoveAVPacket(packet);
1132  packet = NULL;
1133  }
1134  if (found_packet >= 0) {
1135  // Update current packet pointer
1136  packet = next_packet;
1137 
1138  // Keep track of packet stats
1139  if (packet->stream_index == videoStream) {
1140  packet_status.video_read++;
1141  } else if (packet->stream_index == audioStream) {
1142  packet_status.audio_read++;
1143  }
1144  } else {
1145  // No more packets found
1146  delete next_packet;
1147  packet = NULL;
1148  }
1149  // Return if packet was found (or error number)
1150  return found_packet;
1151 }
1152 
1153 // Get an AVFrame (if any)
1154 bool FFmpegReader::GetAVFrame() {
1155  int frameFinished = 0;
1156 
1157  // Decode video frame
1158  AVFrame *next_frame = AV_ALLOCATE_FRAME();
1159 
1160 #if IS_FFMPEG_3_2
1161  int send_packet_err = 0;
1162  int64_t send_packet_pts = 0;
1163  if ((packet && packet->stream_index == videoStream) || !packet) {
1164  send_packet_err = avcodec_send_packet(pCodecCtx, packet);
1165 
1166  if (packet && send_packet_err >= 0) {
1167  send_packet_pts = GetPacketPTS();
1168  hold_packet = false;
1169  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet succeeded)", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts);
1170  }
1171  }
1172 
1173  #if USE_HW_ACCEL
1174  // Get the format from the variables set in get_hw_dec_format
1175  hw_de_av_pix_fmt = hw_de_av_pix_fmt_global;
1176  hw_de_av_device_type = hw_de_av_device_type_global;
1177  #endif // USE_HW_ACCEL
1178  if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) {
1179  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: Not sent [" + av_err2string(send_packet_err) + "])", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts);
1180  if (send_packet_err == AVERROR(EAGAIN)) {
1181  hold_packet = true;
1182  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(EAGAIN): user must read output with avcodec_receive_frame()", "send_packet_pts", send_packet_pts);
1183  }
1184  if (send_packet_err == AVERROR(EINVAL)) {
1185  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(EINVAL): codec not opened, it is an encoder, or requires flush", "send_packet_pts", send_packet_pts);
1186  }
1187  if (send_packet_err == AVERROR(ENOMEM)) {
1188  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(ENOMEM): failed to add packet to internal queue, or legitimate decoding errors", "send_packet_pts", send_packet_pts);
1189  }
1190  }
1191 
1192  // Always try and receive a packet, if not EOF.
1193  // Even if the above avcodec_send_packet failed to send,
1194  // we might still need to receive a packet.
1195  int receive_frame_err = 0;
1196  AVFrame *next_frame2;
1197 #if USE_HW_ACCEL
1198  if (hw_de_on && hw_de_supported) {
1199  next_frame2 = AV_ALLOCATE_FRAME();
1200  }
1201  else
1202 #endif // USE_HW_ACCEL
1203  {
1204  next_frame2 = next_frame;
1205  }
1206  pFrame = AV_ALLOCATE_FRAME();
1207  while (receive_frame_err >= 0) {
1208  receive_frame_err = avcodec_receive_frame(pCodecCtx, next_frame2);
1209 
1210  if (receive_frame_err != 0) {
1211  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (receive frame: frame not ready yet from decoder [\" + av_err2string(receive_frame_err) + \"])", "receive_frame_err", receive_frame_err, "send_packet_pts", send_packet_pts);
1212 
1213  if (receive_frame_err == AVERROR_EOF) {
1215  "FFmpegReader::GetAVFrame (receive frame: AVERROR_EOF: EOF detected from decoder, flushing buffers)", "send_packet_pts", send_packet_pts);
1216  avcodec_flush_buffers(pCodecCtx);
1217  packet_status.video_eof = true;
1218  }
1219  if (receive_frame_err == AVERROR(EINVAL)) {
1221  "FFmpegReader::GetAVFrame (receive frame: AVERROR(EINVAL): invalid frame received, flushing buffers)", "send_packet_pts", send_packet_pts);
1222  avcodec_flush_buffers(pCodecCtx);
1223  }
1224  if (receive_frame_err == AVERROR(EAGAIN)) {
1226  "FFmpegReader::GetAVFrame (receive frame: AVERROR(EAGAIN): output is not available in this state - user must try to send new input)", "send_packet_pts", send_packet_pts);
1227  }
1228  if (receive_frame_err == AVERROR_INPUT_CHANGED) {
1230  "FFmpegReader::GetAVFrame (receive frame: AVERROR_INPUT_CHANGED: current decoded frame has changed parameters with respect to first decoded frame)", "send_packet_pts", send_packet_pts);
1231  }
1232 
1233  // Break out of decoding loop
1234  // Nothing ready for decoding yet
1235  break;
1236  }
1237 
1238 #if USE_HW_ACCEL
1239  if (hw_de_on && hw_de_supported) {
1240  int err;
1241  if (next_frame2->format == hw_de_av_pix_fmt) {
1242  next_frame->format = AV_PIX_FMT_YUV420P;
1243  if ((err = av_hwframe_transfer_data(next_frame,next_frame2,0)) < 0) {
1244  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to transfer data to output frame)", "hw_de_on", hw_de_on);
1245  }
1246  if ((err = av_frame_copy_props(next_frame,next_frame2)) < 0) {
1247  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to copy props to output frame)", "hw_de_on", hw_de_on);
1248  }
1249  }
1250  }
1251  else
1252 #endif // USE_HW_ACCEL
1253  { // No hardware acceleration used -> no copy from GPU memory needed
1254  next_frame = next_frame2;
1255  }
1256 
1257  // TODO also handle possible further frames
1258  // Use only the first frame like avcodec_decode_video2
1259  frameFinished = 1;
1260  packet_status.video_decoded++;
1261 
1262  av_image_alloc(pFrame->data, pFrame->linesize, info.width, info.height, (AVPixelFormat)(pStream->codecpar->format), 1);
1263  av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)next_frame->data, next_frame->linesize,
1264  (AVPixelFormat)(pStream->codecpar->format), info.width, info.height);
1265 
1266  // Get display PTS from video frame, often different than packet->pts.
1267  // Sending packets to the decoder (i.e. packet->pts) is async,
1268  // and retrieving packets from the decoder (frame->pts) is async. In most decoders
1269  // sending and retrieving are separated by multiple calls to this method.
1270  if (next_frame->pts != AV_NOPTS_VALUE) {
1271  // This is the current decoded frame (and should be the pts used) for
1272  // processing this data
1273  video_pts = next_frame->pts;
1274  } else if (next_frame->pkt_dts != AV_NOPTS_VALUE) {
1275  // Some videos only set this timestamp (fallback)
1276  video_pts = next_frame->pkt_dts;
1277  }
1278 
1280  "FFmpegReader::GetAVFrame (Successful frame received)", "video_pts", video_pts, "send_packet_pts", send_packet_pts);
1281 
1282  // break out of loop after each successful image returned
1283  break;
1284  }
1285 #if USE_HW_ACCEL
1286  if (hw_de_on && hw_de_supported) {
1287  AV_FREE_FRAME(&next_frame2);
1288  }
1289  #endif // USE_HW_ACCEL
1290 #else
1291  avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
1292 
1293  // always allocate pFrame (because we do that in the ffmpeg >= 3.2 as well); it will always be freed later
1294  pFrame = AV_ALLOCATE_FRAME();
1295 
1296  // is frame finished
1297  if (frameFinished) {
1298  // AVFrames are clobbered on the each call to avcodec_decode_video, so we
1299  // must make a copy of the image data before this method is called again.
1300  avpicture_alloc((AVPicture *) pFrame, pCodecCtx->pix_fmt, info.width, info.height);
1301  av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width,
1302  info.height);
1303  }
1304 #endif // IS_FFMPEG_3_2
1305 
1306  // deallocate the frame
1307  AV_FREE_FRAME(&next_frame);
1308 
1309  // Did we get a video frame?
1310  return frameFinished;
1311 }
1312 
1313 // Check the current seek position and determine if we need to seek again
1314 bool FFmpegReader::CheckSeek(bool is_video) {
1315  // Are we seeking for a specific frame?
1316  if (is_seeking) {
1317  // Determine if both an audio and video packet have been decoded since the seek happened.
1318  // If not, allow the ReadStream method to keep looping
1319  if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
1320  return false;
1321 
1322  // Check for both streams
1323  if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
1324  return false;
1325 
1326  // Determine max seeked frame
1327  int64_t max_seeked_frame = std::max(seek_audio_frame_found, seek_video_frame_found);
1328 
1329  // determine if we are "before" the requested frame
1330  if (max_seeked_frame >= seeking_frame) {
1331  // SEEKED TOO FAR
1332  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)",
1333  "is_video_seek", is_video_seek,
1334  "max_seeked_frame", max_seeked_frame,
1335  "seeking_frame", seeking_frame,
1336  "seeking_pts", seeking_pts,
1337  "seek_video_frame_found", seek_video_frame_found,
1338  "seek_audio_frame_found", seek_audio_frame_found);
1339 
1340  // Seek again... to the nearest Keyframe
1341  Seek(seeking_frame - (10 * seek_count * seek_count));
1342  } else {
1343  // SEEK WORKED
1344  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)",
1345  "is_video_seek", is_video_seek,
1346  "packet->pts", GetPacketPTS(),
1347  "seeking_pts", seeking_pts,
1348  "seeking_frame", seeking_frame,
1349  "seek_video_frame_found", seek_video_frame_found,
1350  "seek_audio_frame_found", seek_audio_frame_found);
1351 
1352  // Seek worked, and we are "before" the requested frame
1353  is_seeking = false;
1354  seeking_frame = 0;
1355  seeking_pts = -1;
1356  }
1357  }
1358 
1359  // return the pts to seek to (if any)
1360  return is_seeking;
1361 }
1362 
1363 // Process a video packet
1364 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
1365  // Get the AVFrame from the current packet
1366  // This sets the video_pts to the correct timestamp
1367  int frame_finished = GetAVFrame();
1368 
1369  // Check if the AVFrame is finished and set it
1370  if (!frame_finished) {
1371  // No AVFrame decoded yet, bail out
1372  if (pFrame) {
1373  RemoveAVFrame(pFrame);
1374  }
1375  return;
1376  }
1377 
1378  // Calculate current frame #
1379  int64_t current_frame = ConvertVideoPTStoFrame(video_pts);
1380 
1381  // Track 1st video packet after a successful seek
1382  if (!seek_video_frame_found && is_seeking)
1383  seek_video_frame_found = current_frame;
1384 
1385  // Create or get the existing frame object. Requested frame needs to be created
1386  // in working_cache at least once. Seek can clear the working_cache, so we must
1387  // add the requested frame back to the working_cache here. If it already exists,
1388  // it will be moved to the top of the working_cache.
1389  working_cache.Add(CreateFrame(requested_frame));
1390 
1391  // Debug output
1392  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame);
1393 
1394  // Init some things local (for OpenMP)
1395  PixelFormat pix_fmt = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
1396  int height = info.height;
1397  int width = info.width;
1398  int64_t video_length = info.video_length;
1399 
1400  // Create variables for a RGB Frame (since most videos are not in RGB, we must convert it)
1401  AVFrame *pFrameRGB = nullptr;
1402  uint8_t *buffer = nullptr;
1403 
1404  // Allocate an AVFrame structure
1405  pFrameRGB = AV_ALLOCATE_FRAME();
1406  if (pFrameRGB == nullptr)
1407  throw OutOfMemory("Failed to allocate frame buffer", path);
1408 
1409  // Determine the max size of this source image (based on the timeline's size, the scaling mode,
1410  // and the scaling keyframes). This is a performance improvement, to keep the images as small as possible,
1411  // without losing quality. NOTE: We cannot go smaller than the timeline itself, or the add_layer timeline
1412  // method will scale it back to timeline size before scaling it smaller again. This needs to be fixed in
1413  // the future.
1414  int max_width = info.width;
1415  int max_height = info.height;
1416 
1417  Clip *parent = static_cast<Clip *>(ParentClip());
1418  if (parent) {
1419  if (parent->ParentTimeline()) {
1420  // Set max width/height based on parent clip's timeline (if attached to a timeline)
1421  max_width = parent->ParentTimeline()->preview_width;
1422  max_height = parent->ParentTimeline()->preview_height;
1423  }
1424  if (parent->scale == SCALE_FIT || parent->scale == SCALE_STRETCH) {
1425  // Best fit or Stretch scaling (based on max timeline size * scaling keyframes)
1426  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1427  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1428  max_width = std::max(float(max_width), max_width * max_scale_x);
1429  max_height = std::max(float(max_height), max_height * max_scale_y);
1430 
1431  } else if (parent->scale == SCALE_CROP) {
1432  // Cropping scale mode (based on max timeline size * cropped size * scaling keyframes)
1433  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1434  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1435  QSize width_size(max_width * max_scale_x,
1436  round(max_width / (float(info.width) / float(info.height))));
1437  QSize height_size(round(max_height / (float(info.height) / float(info.width))),
1438  max_height * max_scale_y);
1439  // respect aspect ratio
1440  if (width_size.width() >= max_width && width_size.height() >= max_height) {
1441  max_width = std::max(max_width, width_size.width());
1442  max_height = std::max(max_height, width_size.height());
1443  } else {
1444  max_width = std::max(max_width, height_size.width());
1445  max_height = std::max(max_height, height_size.height());
1446  }
1447 
1448  } else {
1449  // Scale video to equivalent unscaled size
1450  // Since the preview window can change sizes, we want to always
1451  // scale against the ratio of original video size to timeline size
1452  float preview_ratio = 1.0;
1453  if (parent->ParentTimeline()) {
1454  Timeline *t = (Timeline *) parent->ParentTimeline();
1455  preview_ratio = t->preview_width / float(t->info.width);
1456  }
1457  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1458  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1459  max_width = info.width * max_scale_x * preview_ratio;
1460  max_height = info.height * max_scale_y * preview_ratio;
1461  }
1462  }
1463 
1464  // Determine if image needs to be scaled (for performance reasons)
1465  int original_height = height;
1466  if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
1467  // Override width and height (but maintain aspect ratio)
1468  float ratio = float(width) / float(height);
1469  int possible_width = round(max_height * ratio);
1470  int possible_height = round(max_width / ratio);
1471 
1472  if (possible_width <= max_width) {
1473  // use calculated width, and max_height
1474  width = possible_width;
1475  height = max_height;
1476  } else {
1477  // use max_width, and calculated height
1478  width = max_width;
1479  height = possible_height;
1480  }
1481  }
1482 
1483  // Determine required buffer size and allocate buffer
1484  const int bytes_per_pixel = 4;
1485  int buffer_size = (width * height * bytes_per_pixel) + 128;
1486  buffer = new unsigned char[buffer_size]();
1487 
1488  // Copy picture data from one AVFrame (or AVPicture) to another one.
1489  AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
1490 
1491  int scale_mode = SWS_FAST_BILINEAR;
1492  if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
1493  scale_mode = SWS_BICUBIC;
1494  }
1495  SwsContext *img_convert_ctx = sws_getContext(info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx), width,
1496  height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL);
1497 
1498  // Resize / Convert to RGB
1499  sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0,
1500  original_height, pFrameRGB->data, pFrameRGB->linesize);
1501 
1502  // Create or get the existing frame object
1503  std::shared_ptr<Frame> f = CreateFrame(current_frame);
1504 
1505  // Add Image data to frame
1506  if (!ffmpeg_has_alpha(AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx))) {
1507  // Add image with no alpha channel, Speed optimization
1508  f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888_Premultiplied, buffer);
1509  } else {
1510  // Add image with alpha channel (this will be converted to premultipled when needed, but is slower)
1511  f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888, buffer);
1512  }
1513 
1514  // Update working cache
1515  working_cache.Add(f);
1516 
1517  // Keep track of last last_video_frame
1518  last_video_frame = f;
1519 
1520  // Free the RGB image
1521  AV_FREE_FRAME(&pFrameRGB);
1522 
1523  // Remove frame and packet
1524  RemoveAVFrame(pFrame);
1525  sws_freeContext(img_convert_ctx);
1526 
1527  // Get video PTS in seconds
1528  video_pts_seconds = (double(video_pts) * info.video_timebase.ToDouble()) + pts_offset_seconds;
1529 
1530  // Debug output
1531  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number, "video_pts_seconds", video_pts_seconds);
1532 }
1533 
1534 // Process an audio packet
1535 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame) {
1536  AudioLocation location;
1537  // Calculate location of current audio packet
1538  if (packet && packet->pts != AV_NOPTS_VALUE) {
1539  // Determine related video frame and starting sample # from audio PTS
1540  location = GetAudioPTSLocation(packet->pts);
1541 
1542  // Track 1st audio packet after a successful seek
1543  if (!seek_audio_frame_found && is_seeking)
1544  seek_audio_frame_found = location.frame;
1545  }
1546 
1547  // Create or get the existing frame object. Requested frame needs to be created
1548  // in working_cache at least once. Seek can clear the working_cache, so we must
1549  // add the requested frame back to the working_cache here. If it already exists,
1550  // it will be moved to the top of the working_cache.
1551  working_cache.Add(CreateFrame(requested_frame));
1552 
1553  // Debug output
1554  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)",
1555  "requested_frame", requested_frame,
1556  "target_frame", location.frame,
1557  "starting_sample", location.sample_start);
1558 
1559  // Init an AVFrame to hold the decoded audio samples
1560  int frame_finished = 0;
1561  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
1562  AV_RESET_FRAME(audio_frame);
1563 
1564  int packet_samples = 0;
1565  int data_size = 0;
1566 
1567 #if IS_FFMPEG_3_2
1568  int send_packet_err = avcodec_send_packet(aCodecCtx, packet);
1569  if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) {
1570  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Packet not sent)");
1571  }
1572  else {
1573  int receive_frame_err = avcodec_receive_frame(aCodecCtx, audio_frame);
1574  if (receive_frame_err >= 0) {
1575  frame_finished = 1;
1576  }
1577  if (receive_frame_err == AVERROR_EOF) {
1578  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (EOF detected from decoder)");
1579  packet_status.audio_eof = true;
1580  }
1581  if (receive_frame_err == AVERROR(EINVAL) || receive_frame_err == AVERROR_EOF) {
1582  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (invalid frame received or EOF from decoder)");
1583  avcodec_flush_buffers(aCodecCtx);
1584  }
1585  if (receive_frame_err != 0) {
1586  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (frame not ready yet from decoder)");
1587  }
1588  }
1589 #else
1590  int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
1591 #endif
1592 
1593  if (frame_finished) {
1594  packet_status.audio_decoded++;
1595 
1596  // This can be different than the current packet, so we need to look
1597  // at the current AVFrame from the audio decoder. This timestamp should
1598  // be used for the remainder of this function
1599  audio_pts = audio_frame->pts;
1600 
1601  // Determine related video frame and starting sample # from audio PTS
1602  location = GetAudioPTSLocation(audio_pts);
1603 
1604  // determine how many samples were decoded
1605  int plane_size = -1;
1606 #if HAVE_CH_LAYOUT
1607  int nb_channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout.nb_channels;
1608 #else
1609  int nb_channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
1610 #endif
1611  data_size = av_samples_get_buffer_size(&plane_size, nb_channels,
1612  audio_frame->nb_samples, (AVSampleFormat) (AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1);
1613 
1614  // Calculate total number of samples
1615  packet_samples = audio_frame->nb_samples * nb_channels;
1616  } else {
1617  if (audio_frame) {
1618  // Free audio frame
1619  AV_FREE_FRAME(&audio_frame);
1620  }
1621  }
1622 
1623  // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
1624  int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
1625 
1626  // Bail if no samples found
1627  if (pts_remaining_samples == 0) {
1628  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (No samples, bailing)",
1629  "packet_samples", packet_samples,
1630  "info.channels", info.channels,
1631  "pts_remaining_samples", pts_remaining_samples);
1632  return;
1633  }
1634 
1635  while (pts_remaining_samples) {
1636  // Get Samples per frame (for this frame number)
1637  int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
1638 
1639  // Calculate # of samples to add to this frame
1640  int samples = samples_per_frame - previous_packet_location.sample_start;
1641  if (samples > pts_remaining_samples)
1642  samples = pts_remaining_samples;
1643 
1644  // Decrement remaining samples
1645  pts_remaining_samples -= samples;
1646 
1647  if (pts_remaining_samples > 0) {
1648  // next frame
1649  previous_packet_location.frame++;
1650  previous_packet_location.sample_start = 0;
1651  } else {
1652  // Increment sample start
1653  previous_packet_location.sample_start += samples;
1654  }
1655  }
1656 
1657  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)",
1658  "packet_samples", packet_samples,
1659  "info.channels", info.channels,
1660  "info.sample_rate", info.sample_rate,
1661  "aCodecCtx->sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx));
1662 
1663  // Create output frame
1664  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1665  AV_RESET_FRAME(audio_converted);
1666  audio_converted->nb_samples = audio_frame->nb_samples;
1667  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_FLTP, 0);
1668 
1669  SWRCONTEXT *avr = NULL;
1670 
1671  // setup resample context
1672  avr = SWR_ALLOC();
1673 #if HAVE_CH_LAYOUT
1674  av_opt_set_chlayout(avr, "in_chlayout", &AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout, 0);
1675  av_opt_set_chlayout(avr, "out_chlayout", &AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout, 0);
1676 #else
1677  av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1678  av_opt_set_int(avr, "out_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1679  av_opt_set_int(avr, "in_channels", info.channels, 0);
1680  av_opt_set_int(avr, "out_channels", info.channels, 0);
1681 #endif
1682  av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0);
1683  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
1684  av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
1685  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1686  SWR_INIT(avr);
1687 
1688  // Convert audio samples
1689  int nb_samples = SWR_CONVERT(avr, // audio resample context
1690  audio_converted->data, // output data pointers
1691  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1692  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1693  audio_frame->data, // input data pointers
1694  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1695  audio_frame->nb_samples); // number of input samples to convert
1696 
1697  // Deallocate resample buffer
1698  SWR_CLOSE(avr);
1699  SWR_FREE(&avr);
1700  avr = NULL;
1701 
1702  int64_t starting_frame_number = -1;
1703  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++) {
1704  // Array of floats (to hold samples for each channel)
1705  starting_frame_number = location.frame;
1706  int channel_buffer_size = nb_samples;
1707  auto *channel_buffer = (float *) (audio_converted->data[channel_filter]);
1708 
1709  // Loop through samples, and add them to the correct frames
1710  int start = location.sample_start;
1711  int remaining_samples = channel_buffer_size;
1712  while (remaining_samples > 0) {
1713  // Get Samples per frame (for this frame number)
1714  int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
1715 
1716  // Calculate # of samples to add to this frame
1717  int samples = std::fmin(samples_per_frame - start, remaining_samples);
1718 
1719  // Create or get the existing frame object
1720  std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1721 
1722  // Add samples for current channel to the frame.
1723  f->AddAudio(true, channel_filter, start, channel_buffer, samples, 1.0f);
1724 
1725  // Debug output
1726  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)",
1727  "frame", starting_frame_number,
1728  "start", start,
1729  "samples", samples,
1730  "channel", channel_filter,
1731  "samples_per_frame", samples_per_frame);
1732 
1733  // Add or update cache
1734  working_cache.Add(f);
1735 
1736  // Decrement remaining samples
1737  remaining_samples -= samples;
1738 
1739  // Increment buffer (to next set of samples)
1740  if (remaining_samples > 0)
1741  channel_buffer += samples;
1742 
1743  // Increment frame number
1744  starting_frame_number++;
1745 
1746  // Reset starting sample #
1747  start = 0;
1748  }
1749  }
1750 
1751  // Free AVFrames
1752  av_free(audio_converted->data[0]);
1753  AV_FREE_FRAME(&audio_converted);
1754  AV_FREE_FRAME(&audio_frame);
1755 
1756  // Get audio PTS in seconds
1757  audio_pts_seconds = (double(audio_pts) * info.audio_timebase.ToDouble()) + pts_offset_seconds;
1758 
1759  // Debug output
1760  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)",
1761  "requested_frame", requested_frame,
1762  "starting_frame", location.frame,
1763  "end_frame", starting_frame_number - 1,
1764  "audio_pts_seconds", audio_pts_seconds);
1765 
1766 }
1767 
1768 
1769 // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
1770 void FFmpegReader::Seek(int64_t requested_frame) {
1771  // Adjust for a requested frame that is too small or too large
1772  if (requested_frame < 1)
1773  requested_frame = 1;
1774  if (requested_frame > info.video_length)
1775  requested_frame = info.video_length;
1776  if (requested_frame > largest_frame_processed && packet_status.end_of_file) {
1777  // Not possible to search past largest_frame once EOF is reached (no more packets)
1778  return;
1779  }
1780 
1781  // Debug output
1782  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek",
1783  "requested_frame", requested_frame,
1784  "seek_count", seek_count,
1785  "last_frame", last_frame);
1786 
1787  // Clear working cache (since we are seeking to another location in the file)
1788  working_cache.Clear();
1789 
1790  // Reset the last frame variable
1791  video_pts = 0.0;
1792  video_pts_seconds = NO_PTS_OFFSET;
1793  audio_pts = 0.0;
1794  audio_pts_seconds = NO_PTS_OFFSET;
1795  hold_packet = false;
1796  last_frame = 0;
1797  current_video_frame = 0;
1798  largest_frame_processed = 0;
1799  bool has_audio_override = info.has_audio;
1800  bool has_video_override = info.has_video;
1801 
1802  // Init end-of-file detection variables
1803  packet_status.reset(false);
1804 
1805  // Increment seek count
1806  seek_count++;
1807 
1808  // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
1809  int buffer_amount = std::max(max_concurrent_frames, 8);
1810  if (requested_frame - buffer_amount < 20) {
1811  // prevent Open() from seeking again
1812  is_seeking = true;
1813 
1814  // Close and re-open file (basically seeking to frame 1)
1815  Close();
1816  Open();
1817 
1818  // Update overrides (since closing and re-opening might update these)
1819  info.has_audio = has_audio_override;
1820  info.has_video = has_video_override;
1821 
1822  // Not actually seeking, so clear these flags
1823  is_seeking = false;
1824  if (seek_count == 1) {
1825  // Don't redefine this on multiple seek attempts for a specific frame
1826  seeking_frame = 1;
1827  seeking_pts = ConvertFrameToVideoPTS(1);
1828  }
1829  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1830  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1831 
1832  } else {
1833  // Seek to nearest key-frame (aka, i-frame)
1834  bool seek_worked = false;
1835  int64_t seek_target = 0;
1836 
1837  // Seek video stream (if any), except album arts
1838  if (!seek_worked && info.has_video && !HasAlbumArt()) {
1839  seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1840  if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1841  fprintf(stderr, "%s: error while seeking video stream\n", pFormatCtx->AV_FILENAME);
1842  } else {
1843  // VIDEO SEEK
1844  is_video_seek = true;
1845  seek_worked = true;
1846  }
1847  }
1848 
1849  // Seek audio stream (if not already seeked... and if an audio stream is found)
1850  if (!seek_worked && info.has_audio) {
1851  seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1852  if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1853  fprintf(stderr, "%s: error while seeking audio stream\n", pFormatCtx->AV_FILENAME);
1854  } else {
1855  // AUDIO SEEK
1856  is_video_seek = false;
1857  seek_worked = true;
1858  }
1859  }
1860 
1861  // Was the seek successful?
1862  if (seek_worked) {
1863  // Flush audio buffer
1864  if (info.has_audio)
1865  avcodec_flush_buffers(aCodecCtx);
1866 
1867  // Flush video buffer
1868  if (info.has_video)
1869  avcodec_flush_buffers(pCodecCtx);
1870 
1871  // Reset previous audio location to zero
1872  previous_packet_location.frame = -1;
1873  previous_packet_location.sample_start = 0;
1874 
1875  // init seek flags
1876  is_seeking = true;
1877  if (seek_count == 1) {
1878  // Don't redefine this on multiple seek attempts for a specific frame
1879  seeking_pts = seek_target;
1880  seeking_frame = requested_frame;
1881  }
1882  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1883  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1884 
1885  } else {
1886  // seek failed
1887  seeking_pts = 0;
1888  seeking_frame = 0;
1889 
1890  // prevent Open() from seeking again
1891  is_seeking = true;
1892 
1893  // Close and re-open file (basically seeking to frame 1)
1894  Close();
1895  Open();
1896 
1897  // Not actually seeking, so clear these flags
1898  is_seeking = false;
1899 
1900  // disable seeking for this reader (since it failed)
1901  enable_seek = false;
1902 
1903  // Update overrides (since closing and re-opening might update these)
1904  info.has_audio = has_audio_override;
1905  info.has_video = has_video_override;
1906  }
1907  }
1908 }
1909 
1910 // Get the PTS for the current video packet
1911 int64_t FFmpegReader::GetPacketPTS() {
1912  if (packet) {
1913  int64_t current_pts = packet->pts;
1914  if (current_pts == AV_NOPTS_VALUE && packet->dts != AV_NOPTS_VALUE)
1915  current_pts = packet->dts;
1916 
1917  // Return adjusted PTS
1918  return current_pts;
1919  } else {
1920  // No packet, return NO PTS
1921  return AV_NOPTS_VALUE;
1922  }
1923 }
1924 
1925 // Update PTS Offset (if any)
1926 void FFmpegReader::UpdatePTSOffset() {
1927  if (pts_offset_seconds != NO_PTS_OFFSET) {
1928  // Skip this method if we have already set PTS offset
1929  return;
1930  }
1931  pts_offset_seconds = 0.0;
1932  double video_pts_offset_seconds = 0.0;
1933  double audio_pts_offset_seconds = 0.0;
1934 
1935  bool has_video_pts = false;
1936  if (!info.has_video) {
1937  // Mark as checked
1938  has_video_pts = true;
1939  }
1940  bool has_audio_pts = false;
1941  if (!info.has_audio) {
1942  // Mark as checked
1943  has_audio_pts = true;
1944  }
1945 
1946  // Loop through the stream (until a packet from all streams is found)
1947  while (!has_video_pts || !has_audio_pts) {
1948  // Get the next packet (if any)
1949  if (GetNextPacket() < 0)
1950  // Break loop when no more packets found
1951  break;
1952 
1953  // Get PTS of this packet
1954  int64_t pts = GetPacketPTS();
1955 
1956  // Video packet
1957  if (!has_video_pts && packet->stream_index == videoStream) {
1958  // Get the video packet start time (in seconds)
1959  video_pts_offset_seconds = 0.0 - (video_pts * info.video_timebase.ToDouble());
1960 
1961  // Is timestamp close to zero (within X seconds)
1962  // Ignore wildly invalid timestamps (i.e. -234923423423)
1963  if (std::abs(video_pts_offset_seconds) <= 10.0) {
1964  has_video_pts = true;
1965  }
1966  }
1967  else if (!has_audio_pts && packet->stream_index == audioStream) {
1968  // Get the audio packet start time (in seconds)
1969  audio_pts_offset_seconds = 0.0 - (pts * info.audio_timebase.ToDouble());
1970 
1971  // Is timestamp close to zero (within X seconds)
1972  // Ignore wildly invalid timestamps (i.e. -234923423423)
1973  if (std::abs(audio_pts_offset_seconds) <= 10.0) {
1974  has_audio_pts = true;
1975  }
1976  }
1977  }
1978 
1979  // Do we have all valid timestamps to determine PTS offset?
1980  if (has_video_pts && has_audio_pts) {
1981  // Set PTS Offset to the smallest offset
1982  // [ video timestamp ]
1983  // [ audio timestamp ]
1984  //
1985  // ** SHIFT TIMESTAMPS TO ZERO **
1986  //
1987  //[ video timestamp ]
1988  // [ audio timestamp ]
1989  //
1990  // Since all offsets are negative at this point, we want the max value, which
1991  // represents the closest to zero
1992  pts_offset_seconds = std::max(video_pts_offset_seconds, audio_pts_offset_seconds);
1993  }
1994 }
1995 
1996 // Convert PTS into Frame Number
1997 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) {
1998  // Apply PTS offset
1999  int64_t previous_video_frame = current_video_frame;
2000 
2001  // Get the video packet start time (in seconds)
2002  double video_seconds = (double(pts) * info.video_timebase.ToDouble()) + pts_offset_seconds;
2003 
2004  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
2005  int64_t frame = round(video_seconds * info.fps.ToDouble()) + 1;
2006 
2007  // Keep track of the expected video frame #
2008  if (current_video_frame == 0)
2009  current_video_frame = frame;
2010  else {
2011 
2012  // Sometimes frames are duplicated due to identical (or similar) timestamps
2013  if (frame == previous_video_frame) {
2014  // return -1 frame number
2015  frame = -1;
2016  } else {
2017  // Increment expected frame
2018  current_video_frame++;
2019  }
2020  }
2021 
2022  // Return frame #
2023  return frame;
2024 }
2025 
2026 // Convert Frame Number into Video PTS
2027 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number) {
2028  // Get timestamp of this frame (in seconds)
2029  double seconds = (double(frame_number - 1) / info.fps.ToDouble()) + pts_offset_seconds;
2030 
2031  // Calculate the # of video packets in this timestamp
2032  int64_t video_pts = round(seconds / info.video_timebase.ToDouble());
2033 
2034  // Apply PTS offset (opposite)
2035  return video_pts;
2036 }
2037 
2038 // Convert Frame Number into Video PTS
2039 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number) {
2040  // Get timestamp of this frame (in seconds)
2041  double seconds = (double(frame_number - 1) / info.fps.ToDouble()) + pts_offset_seconds;
2042 
2043  // Calculate the # of audio packets in this timestamp
2044  int64_t audio_pts = round(seconds / info.audio_timebase.ToDouble());
2045 
2046  // Apply PTS offset (opposite)
2047  return audio_pts;
2048 }
2049 
2050 // Calculate Starting video frame and sample # for an audio PTS
2051 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) {
2052  // Get the audio packet start time (in seconds)
2053  double audio_seconds = (double(pts) * info.audio_timebase.ToDouble()) + pts_offset_seconds;
2054 
2055  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
2056  double frame = (audio_seconds * info.fps.ToDouble()) + 1;
2057 
2058  // Frame # as a whole number (no more decimals)
2059  int64_t whole_frame = int64_t(frame);
2060 
2061  // Remove the whole number, and only get the decimal of the frame
2062  double sample_start_percentage = frame - double(whole_frame);
2063 
2064  // Get Samples per frame
2065  int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
2066 
2067  // Calculate the sample # to start on
2068  int sample_start = round(double(samples_per_frame) * sample_start_percentage);
2069 
2070  // Protect against broken (i.e. negative) timestamps
2071  if (whole_frame < 1)
2072  whole_frame = 1;
2073  if (sample_start < 0)
2074  sample_start = 0;
2075 
2076  // Prepare final audio packet location
2077  AudioLocation location = {whole_frame, sample_start};
2078 
2079  // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
2080  if (previous_packet_location.frame != -1) {
2081  if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame)) {
2082  int64_t orig_frame = location.frame;
2083  int orig_start = location.sample_start;
2084 
2085  // Update sample start, to prevent gaps in audio
2086  location.sample_start = previous_packet_location.sample_start;
2087  location.frame = previous_packet_location.frame;
2088 
2089  // Debug output
2090  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2091 
2092  } else {
2093  // Debug output
2094  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2095  }
2096  }
2097 
2098  // Set previous location
2099  previous_packet_location = location;
2100 
2101  // Return the associated video frame and starting sample #
2102  return location;
2103 }
2104 
2105 // Create a new Frame (or return an existing one) and add it to the working queue.
2106 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame) {
2107  // Check working cache
2108  std::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
2109 
2110  if (!output) {
2111  // (re-)Check working cache
2112  output = working_cache.GetFrame(requested_frame);
2113  if(output) return output;
2114 
2115  // Create a new frame on the working cache
2116  output = std::make_shared<Frame>(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels);
2117  output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
2118  output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
2119  output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
2120 
2121  working_cache.Add(output);
2122 
2123  // Set the largest processed frame (if this is larger)
2124  if (requested_frame > largest_frame_processed)
2125  largest_frame_processed = requested_frame;
2126  }
2127  // Return frame
2128  return output;
2129 }
2130 
2131 // Determine if frame is partial due to seek
2132 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
2133 
2134  // Sometimes a seek gets partial frames, and we need to remove them
2135  bool seek_trash = false;
2136  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
2137  if (seek_video_frame_found > max_seeked_frame) {
2138  max_seeked_frame = seek_video_frame_found;
2139  }
2140  if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
2141  (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame)) {
2142  seek_trash = true;
2143  }
2144 
2145  return seek_trash;
2146 }
2147 
2148 // Check the working queue, and move finished frames to the finished queue
2149 void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) {
2150 
2151  // Prevent async calls to the following code
2152  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
2153 
2154  // Get a list of current working queue frames in the cache (in-progress frames)
2155  std::vector<std::shared_ptr<openshot::Frame>> working_frames = working_cache.GetFrames();
2156  std::vector<std::shared_ptr<openshot::Frame>>::iterator working_itr;
2157 
2158  // Loop through all working queue frames (sorted by frame #)
2159  for(working_itr = working_frames.begin(); working_itr != working_frames.end(); ++working_itr)
2160  {
2161  // Get working frame
2162  std::shared_ptr<Frame> f = *working_itr;
2163 
2164  // Was a frame found? Is frame requested yet?
2165  if (!f || f->number > requested_frame) {
2166  // If not, skip to next one
2167  continue;
2168  }
2169 
2170  // Calculate PTS in seconds (of working frame), and the most recent processed pts value
2171  double frame_pts_seconds = (double(f->number - 1) / info.fps.ToDouble()) + pts_offset_seconds;
2172  double recent_pts_seconds = std::max(video_pts_seconds, audio_pts_seconds);
2173 
2174  // Determine if video and audio are ready (based on timestamps)
2175  bool is_video_ready = false;
2176  bool is_audio_ready = false;
2177  double recent_pts_diff = recent_pts_seconds - frame_pts_seconds;
2178  if ((frame_pts_seconds <= video_pts_seconds)
2179  || (recent_pts_diff > 1.5)
2180  || packet_status.video_eof || packet_status.end_of_file) {
2181  // Video stream is past this frame (so it must be done)
2182  // OR video stream is too far behind, missing, or end-of-file
2183  is_video_ready = true;
2184  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (video ready)",
2185  "frame_number", f->number,
2186  "frame_pts_seconds", frame_pts_seconds,
2187  "video_pts_seconds", video_pts_seconds,
2188  "recent_pts_diff", recent_pts_diff);
2189  if (info.has_video && !f->has_image_data) {
2190  // Frame has no image data (copy from previous frame)
2191  // Loop backwards through final frames (looking for the nearest, previous frame image)
2192  for (int64_t previous_frame = requested_frame - 1; previous_frame > 0; previous_frame--) {
2193  std::shared_ptr<Frame> previous_frame_instance = final_cache.GetFrame(previous_frame);
2194  if (previous_frame_instance && previous_frame_instance->has_image_data) {
2195  // Copy image from last decoded frame
2196  f->AddImage(std::make_shared<QImage>(previous_frame_instance->GetImage()->copy()));
2197  break;
2198  }
2199  }
2200 
2201  if (last_video_frame && !f->has_image_data) {
2202  // Copy image from last decoded frame
2203  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
2204  } else if (!f->has_image_data) {
2205  f->AddColor("#000000");
2206  }
2207  }
2208  }
2209 
2210  double audio_pts_diff = audio_pts_seconds - frame_pts_seconds;
2211  if ((frame_pts_seconds < audio_pts_seconds && audio_pts_diff > 1.0)
2212  || (recent_pts_diff > 1.5)
2213  || packet_status.audio_eof || packet_status.end_of_file) {
2214  // Audio stream is past this frame (so it must be done)
2215  // OR audio stream is too far behind, missing, or end-of-file
2216  // Adding a bit of margin here, to allow for partial audio packets
2217  is_audio_ready = true;
2218  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (audio ready)",
2219  "frame_number", f->number,
2220  "frame_pts_seconds", frame_pts_seconds,
2221  "audio_pts_seconds", audio_pts_seconds,
2222  "audio_pts_diff", audio_pts_diff,
2223  "recent_pts_diff", recent_pts_diff);
2224  }
2225  bool is_seek_trash = IsPartialFrame(f->number);
2226 
2227  // Adjust for available streams
2228  if (!info.has_video) is_video_ready = true;
2229  if (!info.has_audio) is_audio_ready = true;
2230 
2231  // Debug output
2232  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames",
2233  "frame_number", f->number,
2234  "is_video_ready", is_video_ready,
2235  "is_audio_ready", is_audio_ready,
2236  "video_eof", packet_status.video_eof,
2237  "audio_eof", packet_status.audio_eof,
2238  "end_of_file", packet_status.end_of_file);
2239 
2240  // Check if working frame is final
2241  if ((!packet_status.end_of_file && is_video_ready && is_audio_ready) || packet_status.end_of_file || is_seek_trash) {
2242  // Debug output
2243  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)",
2244  "requested_frame", requested_frame,
2245  "f->number", f->number,
2246  "is_seek_trash", is_seek_trash,
2247  "Working Cache Count", working_cache.Count(),
2248  "Final Cache Count", final_cache.Count(),
2249  "end_of_file", packet_status.end_of_file);
2250 
2251  if (!is_seek_trash) {
2252  // Move frame to final cache
2253  final_cache.Add(f);
2254 
2255  // Remove frame from working cache
2256  working_cache.Remove(f->number);
2257 
2258  // Update last frame processed
2259  last_frame = f->number;
2260  } else {
2261  // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
2262  working_cache.Remove(f->number);
2263  }
2264 
2265  }
2266  }
2267 
2268  // Clear vector of frames
2269  working_frames.clear();
2270  working_frames.shrink_to_fit();
2271 }
2272 
2273 // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
2274 void FFmpegReader::CheckFPS() {
2275  if (check_fps) {
2276  // Do not check FPS more than 1 time
2277  return;
2278  } else {
2279  check_fps = true;
2280  }
2281 
2282  int frames_per_second[3] = {0,0,0};
2283  int max_fps_index = sizeof(frames_per_second) / sizeof(frames_per_second[0]);
2284  int fps_index = 0;
2285 
2286  int all_frames_detected = 0;
2287  int starting_frames_detected = 0;
2288 
2289  // Loop through the stream
2290  while (true) {
2291  // Get the next packet (if any)
2292  if (GetNextPacket() < 0)
2293  // Break loop when no more packets found
2294  break;
2295 
2296  // Video packet
2297  if (packet->stream_index == videoStream) {
2298  // Get the video packet start time (in seconds)
2299  double video_seconds = (double(GetPacketPTS()) * info.video_timebase.ToDouble()) + pts_offset_seconds;
2300  fps_index = int(video_seconds); // truncate float timestamp to int (second 1, second 2, second 3)
2301 
2302  // Is this video packet from the first few seconds?
2303  if (fps_index >= 0 && fps_index < max_fps_index) {
2304  // Yes, keep track of how many frames per second (over the first few seconds)
2305  starting_frames_detected++;
2306  frames_per_second[fps_index]++;
2307  }
2308 
2309  // Track all video packets detected
2310  all_frames_detected++;
2311  }
2312  }
2313 
2314  // Calculate FPS (based on the first few seconds of video packets)
2315  float avg_fps = 30.0;
2316  if (starting_frames_detected > 0 && fps_index > 0) {
2317  avg_fps = float(starting_frames_detected) / std::min(fps_index, max_fps_index);
2318  }
2319 
2320  // Verify average FPS is a reasonable value
2321  if (avg_fps < 8.0) {
2322  // Invalid FPS assumed, so switching to a sane default FPS instead
2323  avg_fps = 30.0;
2324  }
2325 
2326  // Update FPS (truncate average FPS to Integer)
2327  info.fps = Fraction(int(avg_fps), 1);
2328 
2329  // Update Duration and Length
2330  if (all_frames_detected > 0) {
2331  // Use all video frames detected to calculate # of frames
2332  info.video_length = all_frames_detected;
2333  info.duration = all_frames_detected / avg_fps;
2334  } else {
2335  // Use previous duration to calculate # of frames
2336  info.video_length = info.duration * avg_fps;
2337  }
2338 
2339  // Update video bit rate
2341 }
2342 
2343 // Remove AVFrame from cache (and deallocate its memory)
2344 void FFmpegReader::RemoveAVFrame(AVFrame *remove_frame) {
2345  // Remove pFrame (if exists)
2346  if (remove_frame) {
2347  // Free memory
2348  av_freep(&remove_frame->data[0]);
2349 #ifndef WIN32
2350  AV_FREE_FRAME(&remove_frame);
2351 #endif
2352  }
2353 }
2354 
2355 // Remove AVPacket from cache (and deallocate its memory)
2356 void FFmpegReader::RemoveAVPacket(AVPacket *remove_packet) {
2357  // deallocate memory for packet
2358  AV_FREE_PACKET(remove_packet);
2359 
2360  // Delete the object
2361  delete remove_packet;
2362 }
2363 
2364 // Generate JSON string of this object
2365 std::string FFmpegReader::Json() const {
2366 
2367  // Return formatted string
2368  return JsonValue().toStyledString();
2369 }
2370 
2371 // Generate Json::Value for this object
2372 Json::Value FFmpegReader::JsonValue() const {
2373 
2374  // Create root json object
2375  Json::Value root = ReaderBase::JsonValue(); // get parent properties
2376  root["type"] = "FFmpegReader";
2377  root["path"] = path;
2378 
2379  // return JsonValue
2380  return root;
2381 }
2382 
2383 // Load JSON string into this object
2384 void FFmpegReader::SetJson(const std::string value) {
2385 
2386  // Parse JSON string into JSON objects
2387  try {
2388  const Json::Value root = openshot::stringToJson(value);
2389  // Set all values that match
2390  SetJsonValue(root);
2391  }
2392  catch (const std::exception& e) {
2393  // Error parsing JSON (or missing keys)
2394  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
2395  }
2396 }
2397 
2398 // Load Json::Value into this object
2399 void FFmpegReader::SetJsonValue(const Json::Value root) {
2400 
2401  // Set parent data
2403 
2404  // Set data from Json (if key is found)
2405  if (!root["path"].isNull())
2406  path = root["path"].asString();
2407 
2408  // Re-Open path, and re-init everything (if needed)
2409  if (is_open) {
2410  Close();
2411  Open();
2412  }
2413 }
openshot::stringToJson
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
openshot::CacheMemory::Clear
void Clear()
Clear the cache of all frames.
Definition: CacheMemory.cpp:221
AV_FIND_DECODER_CODEC_ID
#define AV_FIND_DECODER_CODEC_ID(av_stream)
Definition: FFmpegUtilities.h:206
openshot::ReaderInfo::sample_rate
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:60
openshot::FFmpegReader::FFmpegReader
FFmpegReader(const std::string &path, bool inspect_reader=true)
Constructor for FFmpegReader.
Definition: FFmpegReader.cpp:71
openshot::Fraction::ToFloat
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:35
openshot::Settings::HARDWARE_DECODER
int HARDWARE_DECODER
Use video codec for faster video decoding (if supported)
Definition: Settings.h:62
openshot::Coordinate::Y
double Y
The Y value of the coordinate (usually representing the value of the property being animated)
Definition: Coordinate.h:41
openshot::CacheMemory::Count
int64_t Count()
Count the frames in the queue.
Definition: CacheMemory.cpp:235
FFmpegUtilities.h
Header file for FFmpegUtilities.
openshot::ReaderBase::JsonValue
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
Definition: ReaderBase.cpp:107
openshot::InvalidCodec
Exception when no valid codec is found for a file.
Definition: Exceptions.h:172
openshot::TimelineBase::preview_width
int preview_width
Optional preview width of timeline image. If your preview window is smaller than the timeline,...
Definition: TimelineBase.h:44
openshot::PacketStatus::reset
void reset(bool eof)
Definition: FFmpegReader.h:68
openshot::CacheMemory::GetFrame
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheMemory.cpp:80
openshot::FFmpegReader::GetFrame
std::shared_ptr< openshot::Frame > GetFrame(int64_t requested_frame) override
Definition: FFmpegReader.cpp:905
AV_COPY_PICTURE_DATA
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
Definition: FFmpegUtilities.h:218
openshot::CacheMemory::Add
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
Definition: CacheMemory.cpp:46
PixelFormat
#define PixelFormat
Definition: FFmpegUtilities.h:102
AV_ALLOCATE_FRAME
#define AV_ALLOCATE_FRAME()
Definition: FFmpegUtilities.h:198
openshot::ReaderBase::SetJsonValue
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: ReaderBase.cpp:162
SWR_CONVERT
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Definition: FFmpegUtilities.h:144
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:28
openshot::Point::co
Coordinate co
This is the primary coordinate.
Definition: Point.h:66
FF_NUM_PROCESSORS
#define FF_NUM_PROCESSORS
Definition: OpenMPUtilities.h:24
openshot::Clip::scale_y
openshot::Keyframe scale_y
Curve representing the vertical scaling in percent (0 to 1)
Definition: Clip.h:307
openshot::AudioLocation
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: AudioLocation.h:25
openshot::AudioLocation::frame
int64_t frame
Definition: AudioLocation.h:26
openshot::Clip
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:89
openshot::Fraction
This class represents a fraction.
Definition: Fraction.h:30
openshot::AudioLocation::sample_start
int sample_start
Definition: AudioLocation.h:27
AV_FREE_FRAME
#define AV_FREE_FRAME(av_frame)
Definition: FFmpegUtilities.h:202
openshot::Keyframe::GetMaxPoint
Point GetMaxPoint() const
Get max point (by Y coordinate)
Definition: KeyFrame.cpp:245
openshot::ReaderBase::info
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:88
openshot::ReaderInfo::interlaced_frame
bool interlaced_frame
Definition: ReaderBase.h:56
Timeline.h
Header file for Timeline class.
openshot::Clip::ParentTimeline
void ParentTimeline(openshot::TimelineBase *new_timeline) override
Set associated Timeline pointer.
Definition: Clip.cpp:383
openshot::FFmpegReader::~FFmpegReader
virtual ~FFmpegReader()
Destructor.
Definition: FFmpegReader.cpp:100
openshot::ReaderInfo::audio_bit_rate
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:59
openshot::CacheMemory::Remove
void Remove(int64_t frame_number)
Remove a specific frame.
Definition: CacheMemory.cpp:154
AV_FREE_PACKET
#define AV_FREE_PACKET(av_packet)
Definition: FFmpegUtilities.h:203
openshot::ReaderInfo::duration
float duration
Length of time (in seconds)
Definition: ReaderBase.h:43
openshot::ReaderInfo::has_video
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:40
openshot::FFmpegReader::JsonValue
Json::Value JsonValue() const override
Generate Json::Value for this object.
Definition: FFmpegReader.cpp:2372
openshot::PacketStatus::audio_read
int64_t audio_read
Definition: FFmpegReader.h:49
openshot::ReaderInfo::width
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:46
openshot::FFmpegReader::SetJson
void SetJson(const std::string value) override
Load JSON string into this object.
Definition: FFmpegReader.cpp:2384
openshot::PacketStatus::packets_eof
bool packets_eof
Definition: FFmpegReader.h:55
hw_de_av_pix_fmt_global
AVPixelFormat hw_de_av_pix_fmt_global
Definition: FFmpegReader.cpp:67
openshot::PacketStatus::audio_decoded
int64_t audio_decoded
Definition: FFmpegReader.h:50
openshot::Fraction::ToDouble
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:40
openshot::PacketStatus::video_read
int64_t video_read
Definition: FFmpegReader.h:47
hw_de_on
int hw_de_on
Definition: FFmpegReader.cpp:65
openshot::CacheBase::SetMaxBytesFromInfo
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:30
openshot::Clip::scale_x
openshot::Keyframe scale_x
Curve representing the horizontal scaling in percent (0 to 1)
Definition: Clip.h:306
AV_GET_CODEC_ATTRIBUTES
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
Definition: FFmpegUtilities.h:213
openshot::ReaderInfo::video_length
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:53
hw_de_av_device_type_global
AVHWDeviceType hw_de_av_device_type_global
Definition: FFmpegReader.cpp:68
openshot::ReaderInfo::height
int height
The height of the video (in pixels)
Definition: ReaderBase.h:45
openshot::PacketStatus::video_eof
bool video_eof
Definition: FFmpegReader.h:53
openshot::Fraction::num
int num
Numerator for the fraction.
Definition: Fraction.h:32
if
if(!codec) codec
ZmqLogger.h
Header file for ZeroMQ-based Logger class.
openshot::Fraction::den
int den
Denominator for the fraction.
Definition: Fraction.h:33
OPEN_MP_NUM_PROCESSORS
#define OPEN_MP_NUM_PROCESSORS
Definition: OpenMPUtilities.h:23
AV_RESET_FRAME
#define AV_RESET_FRAME(av_frame)
Definition: FFmpegUtilities.h:201
openshot::AudioLocation::is_near
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
Definition: FFmpegReader.cpp:107
SWR_CLOSE
#define SWR_CLOSE(ctx)
Definition: FFmpegUtilities.h:147
openshot::ReaderInfo::has_audio
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:41
openshot::Settings::DE_LIMIT_HEIGHT_MAX
int DE_LIMIT_HEIGHT_MAX
Maximum rows that hardware decode can handle.
Definition: Settings.h:74
openshot::InvalidJSON
Exception for invalid JSON.
Definition: Exceptions.h:217
openshot::FFmpegReader::enable_seek
bool enable_seek
Definition: FFmpegReader.h:232
openshot::ReaderInfo::file_size
int64_t file_size
Size of file (in bytes)
Definition: ReaderBase.h:44
openshot::Timeline
This class represents a timeline.
Definition: Timeline.h:148
openshot::FFmpegReader::Open
void Open() override
Open File - which is called by the constructor automatically.
Definition: FFmpegReader.cpp:207
openshot::OutOfMemory
Exception when memory could not be allocated.
Definition: Exceptions.h:348
openshot::SCALE_CROP
@ SCALE_CROP
Scale the clip until both height and width fill the canvas (cropping the overlap)
Definition: Enums.h:37
SWR_INIT
#define SWR_INIT(ctx)
Definition: FFmpegUtilities.h:149
SWRCONTEXT
#define SWRCONTEXT
Definition: FFmpegUtilities.h:150
openshot::PacketStatus::audio_eof
bool audio_eof
Definition: FFmpegReader.h:54
openshot::ReaderInfo::has_single_image
bool has_single_image
Determines if this file only contains a single image.
Definition: ReaderBase.h:42
openshot::FFmpegReader::final_cache
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:228
openshot::ReaderInfo::video_timebase
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:55
openshot::Settings::Instance
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: Settings.cpp:23
openshot::ReaderInfo::metadata
std::map< std::string, std::string > metadata
An optional map/dictionary of metadata for this reader.
Definition: ReaderBase.h:65
path
path
Definition: FFmpegWriter.cpp:1479
openshot::Frame::GetSamplesPerFrame
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:484
openshot::InvalidFile
Exception for files that can not be found or opened.
Definition: Exceptions.h:187
openshot::ReaderInfo::audio_stream_index
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:63
openshot::ZmqLogger::Instance
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: ZmqLogger.cpp:35
openshot::ReaderInfo::audio_timebase
openshot::Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:64
openshot::FFmpegReader::Close
void Close() override
Close File.
Definition: FFmpegReader.cpp:586
openshot::SCALE_FIT
@ SCALE_FIT
Scale the clip until either height or width fills the canvas (with no cropping)
Definition: Enums.h:38
openshot::PacketStatus::packets_read
int64_t packets_read()
Definition: FFmpegReader.h:58
openshot::ReaderInfo::pixel_format
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:47
openshot::ZmqLogger::AppendDebugMethod
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:178
openshot::ReaderInfo::vcodec
std::string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:52
openshot::PacketStatus::packets_decoded
int64_t packets_decoded()
Definition: FFmpegReader.h:63
AV_GET_CODEC_TYPE
#define AV_GET_CODEC_TYPE(av_stream)
Definition: FFmpegUtilities.h:205
openshot::ReaderClosed
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:363
openshot::ReaderInfo::channel_layout
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:62
AV_FREE_CONTEXT
#define AV_FREE_CONTEXT(av_context)
Definition: FFmpegUtilities.h:204
PIX_FMT_RGBA
#define PIX_FMT_RGBA
Definition: FFmpegUtilities.h:105
AV_GET_CODEC_PIXEL_FORMAT
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
Definition: FFmpegUtilities.h:214
AVCODEC_REGISTER_ALL
#define AVCODEC_REGISTER_ALL
Definition: FFmpegUtilities.h:194
SWR_FREE
#define SWR_FREE(ctx)
Definition: FFmpegUtilities.h:148
openshot::Settings::DE_LIMIT_WIDTH_MAX
int DE_LIMIT_WIDTH_MAX
Maximum columns that hardware decode can handle.
Definition: Settings.h:77
openshot::ReaderInfo::fps
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:48
AV_GET_SAMPLE_FORMAT
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
Definition: FFmpegUtilities.h:216
openshot::ReaderInfo::video_bit_rate
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:49
openshot::PacketStatus::end_of_file
bool end_of_file
Definition: FFmpegReader.h:56
openshot::Clip::scale
openshot::ScaleType scale
The scale determines how a clip should be resized to fit its parent.
Definition: Clip.h:168
openshot::ReaderInfo::top_field_first
bool top_field_first
Definition: ReaderBase.h:57
openshot::ChannelLayout
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
Definition: ChannelLayouts.h:28
SWR_ALLOC
#define SWR_ALLOC()
Definition: FFmpegUtilities.h:146
openshot::ReaderInfo::pixel_ratio
openshot::Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square)
Definition: ReaderBase.h:50
AV_REGISTER_ALL
#define AV_REGISTER_ALL
Definition: FFmpegUtilities.h:193
openshot::CacheMemory::GetFrames
std::vector< std::shared_ptr< openshot::Frame > > GetFrames()
Get an array of all Frames.
Definition: CacheMemory.cpp:96
AV_GET_CODEC_CONTEXT
#define AV_GET_CODEC_CONTEXT(av_stream, av_codec)
Definition: FFmpegUtilities.h:207
openshot::ReaderInfo::video_stream_index
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:54
openshot::FFmpegReader::SetJsonValue
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
Definition: FFmpegReader.cpp:2399
openshot::SCALE_STRETCH
@ SCALE_STRETCH
Scale the clip until both height and width fill the canvas (distort to fit)
Definition: Enums.h:39
openshot::ReaderInfo::acodec
std::string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:58
openshot::NoStreamsFound
Exception when no streams are found in the file.
Definition: Exceptions.h:285
openshot::ReaderInfo::display_ratio
openshot::Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3)
Definition: ReaderBase.h:51
openshot::ReaderInfo::channels
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:61
openshot::FFmpegReader::Json
std::string Json() const override
Generate JSON string of this object.
Definition: FFmpegReader.cpp:2365
openshot::FFmpegReader::GetIsDurationKnown
bool GetIsDurationKnown()
Return true if frame can be read with GetFrame()
Definition: FFmpegReader.cpp:901
openshot::PacketStatus::video_decoded
int64_t video_decoded
Definition: FFmpegReader.h:48
opts
AVDictionary * opts
Definition: FFmpegWriter.cpp:1486
Exceptions.h
Header file for all Exception classes.
openshot::Settings::HW_DE_DEVICE_SET
int HW_DE_DEVICE_SET
Which GPU to use to decode (0 is the first)
Definition: Settings.h:80
FFmpegReader.h
Header file for FFmpegReader class.
openshot::ReaderBase::getFrameMutex
std::recursive_mutex getFrameMutex
Mutex for multiple threads.
Definition: ReaderBase.h:79
openshot::ReaderBase::ParentClip
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
Definition: ReaderBase.cpp:245