OpenShot Library | libopenshot  0.2.0
FFmpegReader.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for FFmpegReader class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2013 OpenShot Studios, LLC, Fabrice Bellard
9  * (http://www.openshotstudios.com). This file is part of
10  * OpenShot Library (http://www.openshot.org), an open-source project
11  * dedicated to delivering high quality video editing and animation solutions
12  * to the world.
13  *
14  * This file is originally based on the Libavformat API example, and then modified
15  * by the libopenshot project.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/FFmpegReader.h"
32 
33 using namespace openshot;
34 
36  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
37  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
38  check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
39  prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
40  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0), packet(NULL) {
41 
42  // Initialize FFMpeg, and register all formats and codecs
43  av_register_all();
44  avcodec_register_all();
45 
46  // Init cache
50 
51  // Open and Close the reader, to populate it's attributes (such as height, width, etc...)
52  Open();
53  Close();
54 }
55 
56 FFmpegReader::FFmpegReader(string path, bool inspect_reader)
57  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
58  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
59  check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
60  prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
61  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0), packet(NULL) {
62 
63  // Initialize FFMpeg, and register all formats and codecs
64  av_register_all();
65  avcodec_register_all();
66 
67  // Init cache
71 
72  // Open and Close the reader, to populate it's attributes (such as height, width, etc...)
73  if (inspect_reader) {
74  Open();
75  Close();
76  }
77 }
78 
80  if (is_open)
81  // Auto close reader if not already done
82  Close();
83 }
84 
85 // This struct holds the associated video frame and starting sample # for an audio packet.
86 bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount)
87 {
88  // Is frame even close to this one?
89  if (abs(location.frame - frame) >= 2)
90  // This is too far away to be considered
91  return false;
92 
93  // Note that samples_per_frame can vary slightly frame to frame when the
94  // audio sampling rate is not an integer multiple of the video fps.
95  int64_t diff = samples_per_frame * (location.frame - frame) + location.sample_start - sample_start;
96  if (abs(diff) <= amount)
97  // close
98  return true;
99 
100  // not close
101  return false;
102 }
103 
105 {
106  // Open reader if not already open
107  if (!is_open)
108  {
109  // Initialize format context
110  pFormatCtx = NULL;
111 
112  // Open video file
113  if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
114  throw InvalidFile("File could not be opened.", path);
115 
116  // Retrieve stream information
117  if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
118  throw NoStreamsFound("No streams found in file.", path);
119 
120  videoStream = -1;
121  audioStream = -1;
122  // Loop through each stream, and identify the video and audio stream index
123  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++)
124  {
125  // Is this a video stream?
126  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
127  videoStream = i;
128  }
129  // Is this an audio stream?
130  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
131  audioStream = i;
132  }
133  }
134  if (videoStream == -1 && audioStream == -1)
135  throw NoStreamsFound("No video or audio streams found in this file.", path);
136 
137  // Is there a video stream?
138  if (videoStream != -1)
139  {
140  // Set the stream index
141  info.video_stream_index = videoStream;
142 
143  // Set the codec and codec context pointers
144  pStream = pFormatCtx->streams[videoStream];
145 
146  // Find the codec ID from stream
147  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(pStream);
148 
149  // Get codec and codec context from stream
150  AVCodec *pCodec = avcodec_find_decoder(codecId);
151  pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec);
152 
153  // Set number of threads equal to number of processors (not to exceed 16)
154  pCodecCtx->thread_count = min(OPEN_MP_NUM_PROCESSORS, 16);
155 
156  if (pCodec == NULL) {
157  throw InvalidCodec("A valid video codec could not be found for this file.", path);
158  }
159 
160  // Init options
161  AVDictionary *opts = NULL;
162  av_dict_set(&opts, "strict", "experimental", 0);
163 
164  // Open video codec
165  if (avcodec_open2(pCodecCtx, pCodec, &opts) < 0)
166  throw InvalidCodec("A video codec was found, but could not be opened.", path);
167 
168  // Free options
169  av_dict_free(&opts);
170 
171  // Update the File Info struct with video details (if a video stream is found)
172  UpdateVideoInfo();
173  }
174 
175  // Is there an audio stream?
176  if (audioStream != -1)
177  {
178  // Set the stream index
179  info.audio_stream_index = audioStream;
180 
181  // Get a pointer to the codec context for the audio stream
182  aStream = pFormatCtx->streams[audioStream];
183 
184  // Find the codec ID from stream
185  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(aStream);
186 
187  // Get codec and codec context from stream
188  AVCodec *aCodec = avcodec_find_decoder(codecId);
189  aCodecCtx = AV_GET_CODEC_CONTEXT(aStream, aCodec);
190 
191  // Set number of threads equal to number of processors (not to exceed 16)
192  aCodecCtx->thread_count = min(OPEN_MP_NUM_PROCESSORS, 16);
193 
194  if (aCodec == NULL) {
195  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
196  }
197 
198  // Init options
199  AVDictionary *opts = NULL;
200  av_dict_set(&opts, "strict", "experimental", 0);
201 
202  // Open audio codec
203  if (avcodec_open2(aCodecCtx, aCodec, &opts) < 0)
204  throw InvalidCodec("An audio codec was found, but could not be opened.", path);
205 
206  // Free options
207  av_dict_free(&opts);
208 
209  // Update the File Info struct with audio details (if an audio stream is found)
210  UpdateAudioInfo();
211  }
212 
213  // Add format metadata (if any)
214  AVDictionaryEntry *tag = NULL;
215  while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
216  QString str_key = tag->key;
217  QString str_value = tag->value;
218  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
219  }
220 
221  // Init previous audio location to zero
222  previous_packet_location.frame = -1;
223  previous_packet_location.sample_start = 0;
224 
225  // Adjust cache size based on size of frame and audio
229 
230  // Mark as "open"
231  is_open = true;
232  }
233 }
234 
236 {
237  // Close all objects, if reader is 'open'
238  if (is_open)
239  {
240  // Mark as "closed"
241  is_open = false;
242 
243  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
244 
245  // Close the codec
246  if (info.has_video)
247  {
248  avcodec_flush_buffers(pCodecCtx);
249  AV_FREE_CONTEXT(pCodecCtx);
250  }
251  if (info.has_audio)
252  {
253  avcodec_flush_buffers(aCodecCtx);
254  AV_FREE_CONTEXT(aCodecCtx);
255  }
256 
257  // Clear final cache
258  final_cache.Clear();
259  working_cache.Clear();
260  missing_frames.Clear();
261 
262  // Clear processed lists
263  {
264  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
265  processed_video_frames.clear();
266  processed_audio_frames.clear();
267  processing_video_frames.clear();
268  processing_audio_frames.clear();
269  missing_audio_frames.clear();
270  missing_video_frames.clear();
271  missing_audio_frames_source.clear();
272  missing_video_frames_source.clear();
273  checked_frames.clear();
274  }
275 
276  // Close the video file
277  avformat_close_input(&pFormatCtx);
278  av_freep(&pFormatCtx);
279 
280  // Reset some variables
281  last_frame = 0;
282  largest_frame_processed = 0;
283  seek_audio_frame_found = 0;
284  seek_video_frame_found = 0;
285  current_video_frame = 0;
286  has_missing_frames = false;
287  }
288 }
289 
290 void FFmpegReader::UpdateAudioInfo()
291 {
292  // Set values of FileInfo struct
293  info.has_audio = true;
294  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
295  info.acodec = aCodecCtx->codec->name;
296  info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
297  if (AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
298  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout( AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels );
299  info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout;
300  info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate;
301  info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate;
302 
303  // Set audio timebase
304  info.audio_timebase.num = aStream->time_base.num;
305  info.audio_timebase.den = aStream->time_base.den;
306 
307  // Get timebase of audio stream (if valid) and greater than the current duration
308  if (aStream->duration > 0.0f && aStream->duration > info.duration)
309  info.duration = aStream->duration * info.audio_timebase.ToDouble();
310 
311  // Check for an invalid video length
312  if (info.has_video && info.video_length <= 0)
313  {
314  // Calculate the video length from the audio duration
316  }
317 
318  // Set video timebase (if no video stream was found)
319  if (!info.has_video)
320  {
321  // Set a few important default video settings (so audio can be divided into frames)
322  info.fps.num = 24;
323  info.fps.den = 1;
324  info.video_timebase.num = 1;
325  info.video_timebase.den = 24;
327  info.width = 720;
328  info.height = 480;
329  }
330 
331  // Add audio metadata (if any found)
332  AVDictionaryEntry *tag = NULL;
333  while ((tag = av_dict_get(aStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
334  QString str_key = tag->key;
335  QString str_value = tag->value;
336  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
337  }
338 }
339 
340 void FFmpegReader::UpdateVideoInfo()
341 {
342  // Set values of FileInfo struct
343  info.has_video = true;
344  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
345  info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
346  info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
347  info.vcodec = pCodecCtx->codec->name;
348  info.video_bit_rate = pFormatCtx->bit_rate;
349  if (!check_fps)
350  {
351  // set frames per second (fps)
352  info.fps.num = pStream->avg_frame_rate.num;
353  info.fps.den = pStream->avg_frame_rate.den;
354  }
355 
356  if (pStream->sample_aspect_ratio.num != 0)
357  {
358  info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
359  info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
360  }
361  else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0)
362  {
363  info.pixel_ratio.num = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num;
364  info.pixel_ratio.den = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.den;
365  }
366  else
367  {
368  info.pixel_ratio.num = 1;
369  info.pixel_ratio.den = 1;
370  }
371  info.pixel_format = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
372 
373  // Calculate the DAR (display aspect ratio)
375 
376  // Reduce size fraction
377  size.Reduce();
378 
379  // Set the ratio based on the reduced fraction
380  info.display_ratio.num = size.num;
381  info.display_ratio.den = size.den;
382 
383  // Set the video timebase
384  info.video_timebase.num = pStream->time_base.num;
385  info.video_timebase.den = pStream->time_base.den;
386 
387  // Set the duration in seconds, and video length (# of frames)
388  info.duration = pStream->duration * info.video_timebase.ToDouble();
389 
390  // Check for valid duration (if found)
391  if (info.duration <= 0.0f && pFormatCtx->duration >= 0)
392  // Use the format's duration
393  info.duration = pFormatCtx->duration / AV_TIME_BASE;
394 
395  // Calculate duration from filesize and bitrate (if any)
396  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0)
397  // Estimate from bitrate, total bytes, and framerate
399 
400  // No duration found in stream of file
401  if (info.duration <= 0.0f)
402  {
403  // No duration is found in the video stream
404  info.duration = -1;
405  info.video_length = -1;
406  is_duration_known = false;
407  }
408  else
409  {
410  // Yes, a duration was found
411  is_duration_known = true;
412 
413  // Calculate number of frames
415  }
416 
417  // Override an invalid framerate
418  if (info.fps.ToFloat() > 240.0f || (info.fps.num == 0 || info.fps.den == 0))
419  {
420  // Set a few important default video settings (so audio can be divided into frames)
421  info.fps.num = 24;
422  info.fps.den = 1;
423  info.video_timebase.num = 1;
424  info.video_timebase.den = 24;
425 
426  // Calculate number of frames
428  }
429 
430  // Add video metadata (if any)
431  AVDictionaryEntry *tag = NULL;
432  while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
433  QString str_key = tag->key;
434  QString str_value = tag->value;
435  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
436  }
437 }
438 
439 
440 std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame)
441 {
442  // Check for open reader (or throw exception)
443  if (!is_open)
444  throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
445 
446  // Adjust for a requested frame that is too small or too large
447  if (requested_frame < 1)
448  requested_frame = 1;
449  if (requested_frame > info.video_length && is_duration_known)
450  requested_frame = info.video_length;
451  if (info.has_video && info.video_length == 0)
452  // Invalid duration of video file
453  throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
454 
455  // Debug output
456  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame, "", -1, "", -1, "", -1, "", -1);
457 
458  // Check the cache for this frame
459  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
460  if (frame) {
461  // Debug output
462  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame, "", -1, "", -1, "", -1, "", -1, "", -1);
463 
464  // Return the cached frame
465  return frame;
466  }
467  else
468  {
469  #pragma omp critical (ReadStream)
470  {
471  // Check the cache a 2nd time (due to a potential previous lock)
472  if (has_missing_frames)
473  CheckMissingFrame(requested_frame);
474  frame = final_cache.GetFrame(requested_frame);
475  if (frame) {
476  // Debug output
477  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame, "", -1, "", -1, "", -1, "", -1, "", -1);
478 
479  // Return the cached frame
480  }
481  else {
482  // Frame is not in cache
483  // Reset seek count
484  seek_count = 0;
485 
486  // Check for first frame (always need to get frame 1 before other frames, to correctly calculate offsets)
487  if (last_frame == 0 && requested_frame != 1)
488  // Get first frame
489  ReadStream(1);
490 
491  // Are we within X frames of the requested frame?
492  int64_t diff = requested_frame - last_frame;
493  if (diff >= 1 && diff <= 20)
494  {
495  // Continue walking the stream
496  frame = ReadStream(requested_frame);
497  }
498  else
499  {
500  // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
501  if (enable_seek)
502  // Only seek if enabled
503  Seek(requested_frame);
504 
505  else if (!enable_seek && diff < 0)
506  {
507  // Start over, since we can't seek, and the requested frame is smaller than our position
508  Close();
509  Open();
510  }
511 
512  // Then continue walking the stream
513  frame = ReadStream(requested_frame);
514  }
515  }
516  } //omp critical
517  return frame;
518  }
519 }
520 
521 // Read the stream until we find the requested Frame
522 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame)
523 {
524  // Allocate video frame
525  bool end_of_stream = false;
526  bool check_seek = false;
527  bool frame_finished = false;
528  int packet_error = -1;
529 
530  // Minimum number of packets to process (for performance reasons)
531  int packets_processed = 0;
532  int minimum_packets = OPEN_MP_NUM_PROCESSORS;
533  int max_packets = 4096;
534 
535  // Set the number of threads in OpenMP
536  omp_set_num_threads(OPEN_MP_NUM_PROCESSORS);
537  // Allow nested OpenMP sections
538  omp_set_nested(true);
539 
540  // Debug output
541  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame, "OPEN_MP_NUM_PROCESSORS", OPEN_MP_NUM_PROCESSORS, "", -1, "", -1, "", -1, "", -1);
542 
543  #pragma omp parallel
544  {
545  #pragma omp single
546  {
547  // Loop through the stream until the correct frame is found
548  while (true)
549  {
550  // Get the next packet into a local variable called packet
551  packet_error = GetNextPacket();
552 
553  int processing_video_frames_size = 0;
554  int processing_audio_frames_size = 0;
555  {
556  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
557  processing_video_frames_size = processing_video_frames.size();
558  processing_audio_frames_size = processing_audio_frames.size();
559  }
560 
561  // Wait if too many frames are being processed
562  while (processing_video_frames_size + processing_audio_frames_size >= minimum_packets) {
563  usleep(2500);
564  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
565  processing_video_frames_size = processing_video_frames.size();
566  processing_audio_frames_size = processing_audio_frames.size();
567  }
568 
569  // Get the next packet (if any)
570  if (packet_error < 0)
571  {
572  // Break loop when no more packets found
573  end_of_stream = true;
574  break;
575  }
576 
577  // Debug output
578  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "minimum_packets", minimum_packets, "packets_processed", packets_processed, "is_seeking", is_seeking);
579 
580  // Video packet
581  if (info.has_video && packet->stream_index == videoStream)
582  {
583  // Reset this counter, since we have a video packet
584  num_packets_since_video_frame = 0;
585 
586  // Check the status of a seek (if any)
587  if (is_seeking)
588  #pragma omp critical (openshot_seek)
589  check_seek = CheckSeek(true);
590  else
591  check_seek = false;
592 
593  if (check_seek) {
594  // Jump to the next iteration of this loop
595  continue;
596  }
597 
598  // Get the AVFrame from the current packet
599  frame_finished = GetAVFrame();
600 
601  // Check if the AVFrame is finished and set it
602  if (frame_finished)
603  {
604  // Update PTS / Frame Offset (if any)
605  UpdatePTSOffset(true);
606 
607  // Process Video Packet
608  ProcessVideoPacket(requested_frame);
609  }
610 
611  }
612  // Audio packet
613  else if (info.has_audio && packet->stream_index == audioStream)
614  {
615  // Increment this (to track # of packets since the last video packet)
616  num_packets_since_video_frame++;
617 
618  // Check the status of a seek (if any)
619  if (is_seeking)
620  #pragma omp critical (openshot_seek)
621  check_seek = CheckSeek(false);
622  else
623  check_seek = false;
624 
625  if (check_seek) {
626  // Jump to the next iteration of this loop
627  continue;
628  }
629 
630  // Update PTS / Frame Offset (if any)
631  UpdatePTSOffset(false);
632 
633  // Determine related video frame and starting sample # from audio PTS
634  AudioLocation location = GetAudioPTSLocation(packet->pts);
635 
636  // Process Audio Packet
637  ProcessAudioPacket(requested_frame, location.frame, location.sample_start);
638  }
639 
640  // Check if working frames are 'finished'
641  bool is_cache_found = false;
642  if (!is_seeking) {
643  // Check for any missing frames
644  CheckMissingFrame(requested_frame);
645 
646  // Check for final frames
647  CheckWorkingFrames(false, requested_frame);
648  }
649 
650  // Check if requested 'final' frame is available
651  is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
652 
653  // Increment frames processed
654  packets_processed++;
655 
656  // Break once the frame is found
657  if ((is_cache_found && packets_processed >= minimum_packets) || packets_processed > max_packets)
658  break;
659 
660  } // end while
661 
662  } // end omp single
663  } // end omp parallel
664 
665  // Debug output
666  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)", "packets_processed", packets_processed, "end_of_stream", end_of_stream, "largest_frame_processed", largest_frame_processed, "Working Cache Count", working_cache.Count(), "", -1, "", -1);
667 
668  // End of stream?
669  if (end_of_stream)
670  // Mark the any other working frames as 'finished'
671  CheckWorkingFrames(end_of_stream, requested_frame);
672 
673  // Return requested frame (if found)
674  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
675  if (frame)
676  // Return prepared frame
677  return frame;
678  else {
679 
680  // Check if largest frame is still cached
681  frame = final_cache.GetFrame(largest_frame_processed);
682  if (frame) {
683  // return the largest processed frame (assuming it was the last in the video file)
684  return frame;
685  }
686  else {
687  // The largest processed frame is no longer in cache, return a blank frame
688  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
689  f->AddColor(info.width, info.height, "#000");
690  return f;
691  }
692  }
693 
694 }
695 
696 // Get the next packet (if any)
697 int FFmpegReader::GetNextPacket()
698 {
699  int found_packet = 0;
700  AVPacket *next_packet = new AVPacket();
701  found_packet = av_read_frame(pFormatCtx, next_packet);
702 
703  if (packet) {
704  // Remove previous packet before getting next one
705  RemoveAVPacket(packet);
706  packet = NULL;
707  }
708 
709  if (found_packet >= 0)
710  {
711  // Update current packet pointer
712  packet = next_packet;
713  }
714 
715  // Return if packet was found (or error number)
716  return found_packet;
717 }
718 
719 // Get an AVFrame (if any)
720 bool FFmpegReader::GetAVFrame()
721 {
722  int frameFinished = -1;
723  int ret = 0;
724 
725  // Decode video frame
726  AVFrame *next_frame = AV_ALLOCATE_FRAME();
727  #pragma omp critical (packet_cache)
728  {
729  #if IS_FFMPEG_3_2
730  frameFinished = 0;
731  ret = avcodec_send_packet(pCodecCtx, packet);
732  if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
733  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Packet not sent)", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
734  }
735  else {
736  pFrame = new AVFrame();
737  while (ret >= 0) {
738  ret = avcodec_receive_frame(pCodecCtx, next_frame);
739  if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
740  break;
741  }
742  // TODO also handle possible further frames
743  // Use only the first frame like avcodec_decode_video2
744  if (frameFinished == 0 ) {
745  frameFinished = 1;
746  av_image_alloc(pFrame->data, pFrame->linesize, info.width, info.height, (AVPixelFormat)(pStream->codecpar->format), 1);
747  av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)next_frame->data, next_frame->linesize,
748  (AVPixelFormat)(pStream->codecpar->format), info.width, info.height);
749  if (!check_interlace) {
750  check_interlace = true;
751  info.interlaced_frame = next_frame->interlaced_frame;
752  info.top_field_first = next_frame->top_field_first;
753  }
754  }
755  }
756  }
757  #else
758  avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
759 
760  // is frame finished
761  if (frameFinished) {
762  // AVFrames are clobbered on the each call to avcodec_decode_video, so we
763  // must make a copy of the image data before this method is called again.
764  pFrame = AV_ALLOCATE_FRAME();
765  avpicture_alloc((AVPicture *) pFrame, pCodecCtx->pix_fmt, info.width, info.height);
766  av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width,
767  info.height);
768 
769  // Detect interlaced frame (only once)
770  if (!check_interlace) {
771  check_interlace = true;
772  info.interlaced_frame = next_frame->interlaced_frame;
773  info.top_field_first = next_frame->top_field_first;
774  }
775  }
776  #endif
777  }
778 
779  // deallocate the frame
780  AV_FREE_FRAME(&next_frame);
781 
782  // Did we get a video frame?
783  return frameFinished;
784 }
785 
786 // Check the current seek position and determine if we need to seek again
787 bool FFmpegReader::CheckSeek(bool is_video)
788 {
789  // Are we seeking for a specific frame?
790  if (is_seeking)
791  {
792  // Determine if both an audio and video packet have been decoded since the seek happened.
793  // If not, allow the ReadStream method to keep looping
794  if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
795  return false;
796 
797  // Check for both streams
798  if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
799  return false;
800 
801  // Determine max seeked frame
802  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
803  if (seek_video_frame_found > max_seeked_frame)
804  max_seeked_frame = seek_video_frame_found;
805 
806  // determine if we are "before" the requested frame
807  if (max_seeked_frame >= seeking_frame)
808  {
809  // SEEKED TOO FAR
810  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)", "is_video_seek", is_video_seek, "max_seeked_frame", max_seeked_frame, "seeking_frame", seeking_frame, "seeking_pts", seeking_pts, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
811 
812  // Seek again... to the nearest Keyframe
813  Seek(seeking_frame - (10 * seek_count * seek_count));
814  }
815  else
816  {
817  // SEEK WORKED
818  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)", "is_video_seek", is_video_seek, "current_pts", packet->pts, "seeking_pts", seeking_pts, "seeking_frame", seeking_frame, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
819 
820  // Seek worked, and we are "before" the requested frame
821  is_seeking = false;
822  seeking_frame = 0;
823  seeking_pts = -1;
824  }
825  }
826 
827  // return the pts to seek to (if any)
828  return is_seeking;
829 }
830 
831 // Process a video packet
832 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame)
833 {
834  // Calculate current frame #
835  int64_t current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
836 
837  // Track 1st video packet after a successful seek
838  if (!seek_video_frame_found && is_seeking)
839  seek_video_frame_found = current_frame;
840 
841  // Are we close enough to decode the frame? and is this frame # valid?
842  if ((current_frame < (requested_frame - 20)) or (current_frame == -1))
843  {
844  // Remove frame and packet
845  RemoveAVFrame(pFrame);
846 
847  // Debug output
848  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Skipped)", "requested_frame", requested_frame, "current_frame", current_frame, "", -1, "", -1, "", -1, "", -1);
849 
850  // Skip to next frame without decoding or caching
851  return;
852  }
853 
854  // Debug output
855  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame, "", -1, "", -1, "", -1, "", -1);
856 
857  // Init some things local (for OpenMP)
858  PixelFormat pix_fmt = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
859  int height = info.height;
860  int width = info.width;
861  int64_t video_length = info.video_length;
862  AVFrame *my_frame = pFrame;
863 
864  // Add video frame to list of processing video frames
865  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
866  processing_video_frames[current_frame] = current_frame;
867 
868  #pragma omp task firstprivate(current_frame, my_frame, height, width, video_length, pix_fmt)
869  {
870  // Create variables for a RGB Frame (since most videos are not in RGB, we must convert it)
871  AVFrame *pFrameRGB = NULL;
872  int numBytes;
873  uint8_t *buffer = NULL;
874 
875  // Allocate an AVFrame structure
876  pFrameRGB = AV_ALLOCATE_FRAME();
877  if (pFrameRGB == NULL)
878  throw OutOfBoundsFrame("Convert Image Broke!", current_frame, video_length);
879 
880  // Determine if video needs to be scaled down (for performance reasons)
881  // Timelines pass their size to the clips, which pass their size to the readers (as max size)
882  // If a clip is being scaled larger, it will set max_width and max_height = 0 (which means don't down scale)
883  int original_height = height;
884  if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
885  // Override width and height (but maintain aspect ratio)
886  float ratio = float(width) / float(height);
887  int possible_width = round(max_height * ratio);
888  int possible_height = round(max_width / ratio);
889 
890  if (possible_width <= max_width) {
891  // use calculated width, and max_height
892  width = possible_width;
893  height = max_height;
894  } else {
895  // use max_width, and calculated height
896  width = max_width;
897  height = possible_height;
898  }
899  }
900 
901  // Determine required buffer size and allocate buffer
902  numBytes = AV_GET_IMAGE_SIZE(PIX_FMT_RGBA, width, height);
903 
904  #pragma omp critical (video_buffer)
905  buffer = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t));
906 
907  // Copy picture data from one AVFrame (or AVPicture) to another one.
908  AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
909 
910  SwsContext *img_convert_ctx = sws_getContext(info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx), width,
911  height, PIX_FMT_RGBA, SWS_BILINEAR, NULL, NULL, NULL);
912 
913  // Resize / Convert to RGB
914  sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
915  original_height, pFrameRGB->data, pFrameRGB->linesize);
916 
917  // Create or get the existing frame object
918  std::shared_ptr<Frame> f = CreateFrame(current_frame);
919 
920  // Add Image data to frame
921  f->AddImage(width, height, 4, QImage::Format_RGBA8888, buffer);
922 
923  // Update working cache
924  working_cache.Add(f);
925 
926  // Keep track of last last_video_frame
927  #pragma omp critical (video_buffer)
928  last_video_frame = f;
929 
930  // Free the RGB image
931  av_free(buffer);
932  AV_FREE_FRAME(&pFrameRGB);
933 
934  // Remove frame and packet
935  RemoveAVFrame(my_frame);
936  sws_freeContext(img_convert_ctx);
937 
938  // Remove video frame from list of processing video frames
939  {
940  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
941  processing_video_frames.erase(current_frame);
942  processed_video_frames[current_frame] = current_frame;
943  }
944 
945  // Debug output
946  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number, "", -1, "", -1, "", -1);
947 
948  } // end omp task
949 
950 }
951 
952 // Process an audio packet
953 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample)
954 {
955  // Track 1st audio packet after a successful seek
956  if (!seek_audio_frame_found && is_seeking)
957  seek_audio_frame_found = target_frame;
958 
959  // Are we close enough to decode the frame's audio?
960  if (target_frame < (requested_frame - 20))
961  {
962  // Debug output
963  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Skipped)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample, "", -1, "", -1, "", -1);
964 
965  // Skip to next frame without decoding or caching
966  return;
967  }
968 
969  // Debug output
970  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample, "", -1, "", -1, "", -1);
971 
972  // Init an AVFrame to hold the decoded audio samples
973  int frame_finished = 0;
974  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
975  AV_RESET_FRAME(audio_frame);
976 
977  int packet_samples = 0;
978  int data_size = 0;
979 
980  // re-initialize buffer size (it gets changed in the avcodec_decode_audio2 method call)
981  int buf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE;
982  #pragma omp critical (ProcessAudioPacket)
983  {
984  #if IS_FFMPEG_3_2
985  int ret = 0;
986  frame_finished = 1;
987  while((packet->size > 0 || (!packet->data && frame_finished)) && ret >= 0) {
988  frame_finished = 0;
989  ret = avcodec_send_packet(aCodecCtx, packet);
990  if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
991  avcodec_send_packet(aCodecCtx, NULL);
992  break;
993  }
994  if (ret >= 0)
995  packet->size = 0;
996  ret = avcodec_receive_frame(aCodecCtx, audio_frame);
997  if (ret >= 0)
998  frame_finished = 1;
999  if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1000  avcodec_flush_buffers(aCodecCtx);
1001  ret = 0;
1002  }
1003  if (ret >= 0) {
1004  ret = frame_finished;
1005  }
1006  }
1007  if (!packet->data && !frame_finished)
1008  {
1009  ret = -1;
1010  }
1011  #else
1012  int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
1013 #endif
1014  }
1015 
1016  if (frame_finished) {
1017 
1018  // determine how many samples were decoded
1019  int planar = av_sample_fmt_is_planar((AVSampleFormat)AV_GET_CODEC_PIXEL_FORMAT(aStream, aCodecCtx));
1020  int plane_size = -1;
1021  data_size = av_samples_get_buffer_size(&plane_size,
1022  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels,
1023  audio_frame->nb_samples,
1024  (AVSampleFormat)(AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1);
1025 
1026  // Calculate total number of samples
1027  packet_samples = audio_frame->nb_samples * AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
1028  }
1029 
1030  // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
1031  int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
1032 
1033  // DEBUG (FOR AUDIO ISSUES) - Get the audio packet start time (in seconds)
1034  int64_t adjusted_pts = packet->pts + audio_pts_offset;
1035  double audio_seconds = double(adjusted_pts) * info.audio_timebase.ToDouble();
1036  double sample_seconds = double(pts_total) / info.sample_rate;
1037 
1038  // Debug output
1039  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info A)", "pts_counter", pts_counter, "PTS", adjusted_pts, "Offset", audio_pts_offset, "PTS Diff", adjusted_pts - prev_pts, "Samples", pts_remaining_samples, "Sample PTS ratio", float(adjusted_pts - prev_pts) / pts_remaining_samples);
1040  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info B)", "Sample Diff", pts_remaining_samples - prev_samples - prev_pts, "Total", pts_total, "PTS Seconds", audio_seconds, "Sample Seconds", sample_seconds, "Seconds Diff", audio_seconds - sample_seconds, "raw samples", packet_samples);
1041 
1042  // DEBUG (FOR AUDIO ISSUES)
1043  prev_pts = adjusted_pts;
1044  pts_total += pts_remaining_samples;
1045  pts_counter++;
1046  prev_samples = pts_remaining_samples;
1047 
1048  // Add audio frame to list of processing audio frames
1049  {
1050  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1051  processing_audio_frames.insert(pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
1052  }
1053 
1054  while (pts_remaining_samples)
1055  {
1056  // Get Samples per frame (for this frame number)
1057  int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
1058 
1059  // Calculate # of samples to add to this frame
1060  int samples = samples_per_frame - previous_packet_location.sample_start;
1061  if (samples > pts_remaining_samples)
1062  samples = pts_remaining_samples;
1063 
1064  // Decrement remaining samples
1065  pts_remaining_samples -= samples;
1066 
1067  if (pts_remaining_samples > 0) {
1068  // next frame
1069  previous_packet_location.frame++;
1070  previous_packet_location.sample_start = 0;
1071 
1072  // Add audio frame to list of processing audio frames
1073  {
1074  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1075  processing_audio_frames.insert(pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
1076  }
1077 
1078  } else {
1079  // Increment sample start
1080  previous_packet_location.sample_start += samples;
1081  }
1082  }
1083 
1084 
1085  // Allocate audio buffer
1086  int16_t *audio_buf = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE];
1087 
1088  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)", "packet_samples", packet_samples, "info.channels", info.channels, "info.sample_rate", info.sample_rate, "aCodecCtx->sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), "AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16, "", -1);
1089 
1090  // Create output frame
1091  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1092  AV_RESET_FRAME(audio_converted);
1093  audio_converted->nb_samples = audio_frame->nb_samples;
1094  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1095 
1096  AVAudioResampleContext *avr = NULL;
1097  int nb_samples = 0;
1098 
1099  // setup resample context
1100  avr = avresample_alloc_context();
1101  av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1102  av_opt_set_int(avr, "out_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1103  av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0);
1104  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1105  av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
1106  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1107  av_opt_set_int(avr, "in_channels", info.channels, 0);
1108  av_opt_set_int(avr, "out_channels", info.channels, 0);
1109  int r = avresample_open(avr);
1110 
1111  // Convert audio samples
1112  nb_samples = avresample_convert(avr, // audio resample context
1113  audio_converted->data, // output data pointers
1114  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1115  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1116  audio_frame->data, // input data pointers
1117  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1118  audio_frame->nb_samples); // number of input samples to convert
1119 
1120  // Copy audio samples over original samples
1121  memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels);
1122 
1123  // Deallocate resample buffer
1124  avresample_close(avr);
1125  avresample_free(&avr);
1126  avr = NULL;
1127 
1128  // Free AVFrames
1129  av_free(audio_converted->data[0]);
1130  AV_FREE_FRAME(&audio_converted);
1131 
1132  int64_t starting_frame_number = -1;
1133  bool partial_frame = true;
1134  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
1135  {
1136  // Array of floats (to hold samples for each channel)
1137  starting_frame_number = target_frame;
1138  int channel_buffer_size = packet_samples / info.channels;
1139  float *channel_buffer = new float[channel_buffer_size];
1140 
1141  // Init buffer array
1142  for (int z = 0; z < channel_buffer_size; z++)
1143  channel_buffer[z] = 0.0f;
1144 
1145  // Loop through all samples and add them to our Frame based on channel.
1146  // Toggle through each channel number, since channel data is stored like (left right left right)
1147  int channel = 0;
1148  int position = 0;
1149  for (int sample = 0; sample < packet_samples; sample++)
1150  {
1151  // Only add samples for current channel
1152  if (channel_filter == channel)
1153  {
1154  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
1155  channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1156 
1157  // Increment audio position
1158  position++;
1159  }
1160 
1161  // increment channel (if needed)
1162  if ((channel + 1) < info.channels)
1163  // move to next channel
1164  channel ++;
1165  else
1166  // reset channel
1167  channel = 0;
1168  }
1169 
1170  // Loop through samples, and add them to the correct frames
1171  int start = starting_sample;
1172  int remaining_samples = channel_buffer_size;
1173  float *iterate_channel_buffer = channel_buffer; // pointer to channel buffer
1174  while (remaining_samples > 0)
1175  {
1176  // Get Samples per frame (for this frame number)
1177  int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
1178 
1179  // Calculate # of samples to add to this frame
1180  int samples = samples_per_frame - start;
1181  if (samples > remaining_samples)
1182  samples = remaining_samples;
1183 
1184  // Create or get the existing frame object
1185  std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1186 
1187  // Determine if this frame was "partially" filled in
1188  if (samples_per_frame == start + samples)
1189  partial_frame = false;
1190  else
1191  partial_frame = true;
1192 
1193  // Add samples for current channel to the frame. Reduce the volume to 98%, to prevent
1194  // some louder samples from maxing out at 1.0 (not sure why this happens)
1195  f->AddAudio(true, channel_filter, start, iterate_channel_buffer, samples, 0.98f);
1196 
1197  // Debug output
1198  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)", "frame", starting_frame_number, "start", start, "samples", samples, "channel", channel_filter, "partial_frame", partial_frame, "samples_per_frame", samples_per_frame);
1199 
1200  // Add or update cache
1201  working_cache.Add(f);
1202 
1203  // Decrement remaining samples
1204  remaining_samples -= samples;
1205 
1206  // Increment buffer (to next set of samples)
1207  if (remaining_samples > 0)
1208  iterate_channel_buffer += samples;
1209 
1210  // Increment frame number
1211  starting_frame_number++;
1212 
1213  // Reset starting sample #
1214  start = 0;
1215  }
1216 
1217  // clear channel buffer
1218  delete[] channel_buffer;
1219  channel_buffer = NULL;
1220  iterate_channel_buffer = NULL;
1221  }
1222 
1223  // Clean up some arrays
1224  delete[] audio_buf;
1225  audio_buf = NULL;
1226 
1227  // Remove audio frame from list of processing audio frames
1228  {
1229  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1230  // Update all frames as completed
1231  for (int64_t f = target_frame; f < starting_frame_number; f++) {
1232  // Remove the frame # from the processing list. NOTE: If more than one thread is
1233  // processing this frame, the frame # will be in this list multiple times. We are only
1234  // removing a single instance of it here.
1235  processing_audio_frames.erase(processing_audio_frames.find(f));
1236 
1237  // Check and see if this frame is also being processed by another thread
1238  if (processing_audio_frames.count(f) == 0)
1239  // No other thread is processing it. Mark the audio as processed (final)
1240  processed_audio_frames[f] = f;
1241  }
1242 
1243  if (target_frame == starting_frame_number) {
1244  // This typically never happens, but just in case, remove the currently processing number
1245  processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1246  }
1247  }
1248 
1249  // Free audio frame
1250  AV_FREE_FRAME(&audio_frame);
1251 
1252  // Debug output
1253  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)", "requested_frame", requested_frame, "starting_frame", target_frame, "end_frame", starting_frame_number - 1, "", -1, "", -1, "", -1);
1254 
1255 }
1256 
1257 
1258 
1259 // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
1260 void FFmpegReader::Seek(int64_t requested_frame)
1261 {
1262  // Adjust for a requested frame that is too small or too large
1263  if (requested_frame < 1)
1264  requested_frame = 1;
1265  if (requested_frame > info.video_length)
1266  requested_frame = info.video_length;
1267 
1268  int processing_video_frames_size = 0;
1269  int processing_audio_frames_size = 0;
1270  {
1271  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1272  processing_video_frames_size = processing_video_frames.size();
1273  processing_audio_frames_size = processing_audio_frames.size();
1274  }
1275 
1276  // Debug output
1277  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek", "requested_frame", requested_frame, "seek_count", seek_count, "last_frame", last_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "video_pts_offset", video_pts_offset);
1278 
1279  // Wait for any processing frames to complete
1280  while (processing_video_frames_size + processing_audio_frames_size > 0) {
1281  usleep(2500);
1282  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1283  processing_video_frames_size = processing_video_frames.size();
1284  processing_audio_frames_size = processing_audio_frames.size();
1285  }
1286 
1287  // Clear working cache (since we are seeking to another location in the file)
1288  working_cache.Clear();
1289  missing_frames.Clear();
1290 
1291  // Clear processed lists
1292  {
1293  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1294  processing_audio_frames.clear();
1295  processing_video_frames.clear();
1296  processed_video_frames.clear();
1297  processed_audio_frames.clear();
1298  missing_audio_frames.clear();
1299  missing_video_frames.clear();
1300  missing_audio_frames_source.clear();
1301  missing_video_frames_source.clear();
1302  checked_frames.clear();
1303  }
1304 
1305  // Reset the last frame variable
1306  last_frame = 0;
1307  current_video_frame = 0;
1308  largest_frame_processed = 0;
1309  num_checks_since_final = 0;
1310  num_packets_since_video_frame = 0;
1311  has_missing_frames = false;
1312  bool has_audio_override = info.has_audio;
1313  bool has_video_override = info.has_video;
1314 
1315  // Increment seek count
1316  seek_count++;
1317 
1318  // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
1319  int buffer_amount = max(OPEN_MP_NUM_PROCESSORS, 8);
1320  if (requested_frame - buffer_amount < 20)
1321  {
1322  // Close and re-open file (basically seeking to frame 1)
1323  Close();
1324  Open();
1325 
1326  // Update overrides (since closing and re-opening might update these)
1327  info.has_audio = has_audio_override;
1328  info.has_video = has_video_override;
1329 
1330  // Not actually seeking, so clear these flags
1331  is_seeking = false;
1332  if (seek_count == 1) {
1333  // Don't redefine this on multiple seek attempts for a specific frame
1334  seeking_frame = 1;
1335  seeking_pts = ConvertFrameToVideoPTS(1);
1336  }
1337  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1338  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1339  }
1340  else
1341  {
1342  // Seek to nearest key-frame (aka, i-frame)
1343  bool seek_worked = false;
1344  int64_t seek_target = 0;
1345 
1346  // Seek video stream (if any)
1347  if (!seek_worked && info.has_video)
1348  {
1349  seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1350  if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1351  fprintf(stderr, "%s: error while seeking video stream\n", pFormatCtx->filename);
1352  } else
1353  {
1354  // VIDEO SEEK
1355  is_video_seek = true;
1356  seek_worked = true;
1357  }
1358  }
1359 
1360  // Seek audio stream (if not already seeked... and if an audio stream is found)
1361  if (!seek_worked && info.has_audio)
1362  {
1363  seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1364  if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1365  fprintf(stderr, "%s: error while seeking audio stream\n", pFormatCtx->filename);
1366  } else
1367  {
1368  // AUDIO SEEK
1369  is_video_seek = false;
1370  seek_worked = true;
1371  }
1372  }
1373 
1374  // Was the seek successful?
1375  if (seek_worked)
1376  {
1377  // Flush audio buffer
1378  if (info.has_audio)
1379  avcodec_flush_buffers(aCodecCtx);
1380 
1381  // Flush video buffer
1382  if (info.has_video)
1383  avcodec_flush_buffers(pCodecCtx);
1384 
1385  // Reset previous audio location to zero
1386  previous_packet_location.frame = -1;
1387  previous_packet_location.sample_start = 0;
1388 
1389  // init seek flags
1390  is_seeking = true;
1391  if (seek_count == 1) {
1392  // Don't redefine this on multiple seek attempts for a specific frame
1393  seeking_pts = seek_target;
1394  seeking_frame = requested_frame;
1395  }
1396  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1397  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1398 
1399  }
1400  else
1401  {
1402  // seek failed
1403  is_seeking = false;
1404  seeking_pts = 0;
1405  seeking_frame = 0;
1406 
1407  // dislable seeking for this reader (since it failed)
1408  // TODO: Find a safer way to do this... not sure how common it is for a seek to fail.
1409  enable_seek = false;
1410 
1411  // Close and re-open file (basically seeking to frame 1)
1412  Close();
1413  Open();
1414 
1415  // Update overrides (since closing and re-opening might update these)
1416  info.has_audio = has_audio_override;
1417  info.has_video = has_video_override;
1418  }
1419  }
1420 }
1421 
1422 // Get the PTS for the current video packet
1423 int64_t FFmpegReader::GetVideoPTS()
1424 {
1425  int64_t current_pts = 0;
1426  if(packet->dts != AV_NOPTS_VALUE)
1427  current_pts = packet->dts;
1428 
1429  // Return adjusted PTS
1430  return current_pts;
1431 }
1432 
1433 // Update PTS Offset (if any)
1434 void FFmpegReader::UpdatePTSOffset(bool is_video)
1435 {
1436  // Determine the offset between the PTS and Frame number (only for 1st frame)
1437  if (is_video)
1438  {
1439  // VIDEO PACKET
1440  if (video_pts_offset == 99999) // Has the offset been set yet?
1441  {
1442  // Find the difference between PTS and frame number (no more than 10 timebase units allowed)
1443  video_pts_offset = 0 - max(GetVideoPTS(), (int64_t) info.video_timebase.ToInt() * 10);
1444 
1445  // debug output
1446  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Video)", "video_pts_offset", video_pts_offset, "is_video", is_video, "", -1, "", -1, "", -1, "", -1);
1447  }
1448  }
1449  else
1450  {
1451  // AUDIO PACKET
1452  if (audio_pts_offset == 99999) // Has the offset been set yet?
1453  {
1454  // Find the difference between PTS and frame number (no more than 10 timebase units allowed)
1455  audio_pts_offset = 0 - max(packet->pts, (int64_t) info.audio_timebase.ToInt() * 10);
1456 
1457  // debug output
1458  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Audio)", "audio_pts_offset", audio_pts_offset, "is_video", is_video, "", -1, "", -1, "", -1, "", -1);
1459  }
1460  }
1461 }
1462 
1463 // Convert PTS into Frame Number
1464 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts)
1465 {
1466  // Apply PTS offset
1467  pts = pts + video_pts_offset;
1468  int64_t previous_video_frame = current_video_frame;
1469 
1470  // Get the video packet start time (in seconds)
1471  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1472 
1473  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1474  int64_t frame = round(video_seconds * info.fps.ToDouble()) + 1;
1475 
1476  // Keep track of the expected video frame #
1477  if (current_video_frame == 0)
1478  current_video_frame = frame;
1479  else {
1480 
1481  // Sometimes frames are duplicated due to identical (or similar) timestamps
1482  if (frame == previous_video_frame) {
1483  // return -1 frame number
1484  frame = -1;
1485  }
1486  else
1487  // Increment expected frame
1488  current_video_frame++;
1489 
1490  if (current_video_frame < frame)
1491  // has missing frames
1492  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)", "calculated frame", frame, "previous_video_frame", previous_video_frame, "current_video_frame", current_video_frame, "", -1, "", -1, "", -1);
1493 
1494  // Sometimes frames are missing due to varying timestamps, or they were dropped. Determine
1495  // if we are missing a video frame.
1496  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1497  while (current_video_frame < frame) {
1498  if (!missing_video_frames.count(current_video_frame)) {
1499  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)", "current_video_frame", current_video_frame, "previous_video_frame", previous_video_frame, "", -1, "", -1, "", -1, "", -1);
1500  missing_video_frames.insert(pair<int64_t, int64_t>(current_video_frame, previous_video_frame));
1501  missing_video_frames_source.insert(pair<int64_t, int64_t>(previous_video_frame, current_video_frame));
1502  }
1503 
1504  // Mark this reader as containing missing frames
1505  has_missing_frames = true;
1506 
1507  // Increment current frame
1508  current_video_frame++;
1509  }
1510  }
1511 
1512  // Return frame #
1513  return frame;
1514 }
1515 
1516 // Convert Frame Number into Video PTS
1517 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number)
1518 {
1519  // Get timestamp of this frame (in seconds)
1520  double seconds = double(frame_number) / info.fps.ToDouble();
1521 
1522  // Calculate the # of video packets in this timestamp
1523  int64_t video_pts = round(seconds / info.video_timebase.ToDouble());
1524 
1525  // Apply PTS offset (opposite)
1526  return video_pts - video_pts_offset;
1527 }
1528 
1529 // Convert Frame Number into Video PTS
1530 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number)
1531 {
1532  // Get timestamp of this frame (in seconds)
1533  double seconds = double(frame_number) / info.fps.ToDouble();
1534 
1535  // Calculate the # of audio packets in this timestamp
1536  int64_t audio_pts = round(seconds / info.audio_timebase.ToDouble());
1537 
1538  // Apply PTS offset (opposite)
1539  return audio_pts - audio_pts_offset;
1540 }
1541 
1542 // Calculate Starting video frame and sample # for an audio PTS
1543 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts)
1544 {
1545  // Apply PTS offset
1546  pts = pts + audio_pts_offset;
1547 
1548  // Get the audio packet start time (in seconds)
1549  double audio_seconds = double(pts) * info.audio_timebase.ToDouble();
1550 
1551  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1552  double frame = (audio_seconds * info.fps.ToDouble()) + 1;
1553 
1554  // Frame # as a whole number (no more decimals)
1555  int64_t whole_frame = int64_t(frame);
1556 
1557  // Remove the whole number, and only get the decimal of the frame
1558  double sample_start_percentage = frame - double(whole_frame);
1559 
1560  // Get Samples per frame
1561  int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
1562 
1563  // Calculate the sample # to start on
1564  int sample_start = round(double(samples_per_frame) * sample_start_percentage);
1565 
1566  // Protect against broken (i.e. negative) timestamps
1567  if (whole_frame < 1)
1568  whole_frame = 1;
1569  if (sample_start < 0)
1570  sample_start = 0;
1571 
1572  // Prepare final audio packet location
1573  AudioLocation location = {whole_frame, sample_start};
1574 
1575  // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
1576  if (previous_packet_location.frame != -1) {
1577  if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame))
1578  {
1579  int64_t orig_frame = location.frame;
1580  int orig_start = location.sample_start;
1581 
1582  // Update sample start, to prevent gaps in audio
1583  location.sample_start = previous_packet_location.sample_start;
1584  location.frame = previous_packet_location.frame;
1585 
1586  // Debug output
1587  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts, "", -1);
1588 
1589  } else {
1590  // Debug output
1591  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts, "", -1, "", -1);
1592 
1593  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1594  for (int64_t audio_frame = previous_packet_location.frame; audio_frame < location.frame; audio_frame++) {
1595  if (!missing_audio_frames.count(audio_frame)) {
1596  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (tracking missing frame)", "missing_audio_frame", audio_frame, "previous_audio_frame", previous_packet_location.frame, "new location frame", location.frame, "", -1, "", -1, "", -1);
1597  missing_audio_frames.insert(pair<int64_t, int64_t>(previous_packet_location.frame - 1, audio_frame));
1598  }
1599  }
1600  }
1601  }
1602 
1603  // Set previous location
1604  previous_packet_location = location;
1605 
1606  // Return the associated video frame and starting sample #
1607  return location;
1608 }
1609 
1610 // Create a new Frame (or return an existing one) and add it to the working queue.
1611 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame)
1612 {
1613  // Check working cache
1614  std::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
1615  if (!output)
1616  {
1617  // Create a new frame on the working cache
1618  output = std::make_shared<Frame>(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels);
1619  output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
1620  output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
1621  output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
1622 
1623  working_cache.Add(output);
1624 
1625  // Set the largest processed frame (if this is larger)
1626  if (requested_frame > largest_frame_processed)
1627  largest_frame_processed = requested_frame;
1628  }
1629 
1630  // Return new frame
1631  return output;
1632 }
1633 
1634 // Determine if frame is partial due to seek
1635 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
1636 
1637  // Sometimes a seek gets partial frames, and we need to remove them
1638  bool seek_trash = false;
1639  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
1640  if (seek_video_frame_found > max_seeked_frame)
1641  max_seeked_frame = seek_video_frame_found;
1642  if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
1643  (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame))
1644  seek_trash = true;
1645 
1646  return seek_trash;
1647 }
1648 
1649 // Check if a frame is missing and attempt to replace it's frame image (and
1650 bool FFmpegReader::CheckMissingFrame(int64_t requested_frame)
1651 {
1652  // Lock
1653  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1654 
1655  // Init # of times this frame has been checked so far
1656  int checked_count = 0;
1657 
1658  // Increment check count for this frame (or init to 1)
1659  if (checked_frames.count(requested_frame) == 0)
1660  checked_frames[requested_frame] = 1;
1661  else
1662  checked_frames[requested_frame]++;
1663  checked_count = checked_frames[requested_frame];
1664 
1665  // Debug output
1666  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame", "requested_frame", requested_frame, "has_missing_frames", has_missing_frames, "missing_video_frames.size()", missing_video_frames.size(), "checked_count", checked_count, "", -1, "", -1);
1667 
1668  // Missing frames (sometimes frame #'s are skipped due to invalid or missing timestamps)
1669  map<int64_t, int64_t>::iterator itr;
1670  bool found_missing_frame = false;
1671 
1672  // Check if requested frame is a missing frame
1673  if (missing_video_frames.count(requested_frame) || missing_audio_frames.count(requested_frame)) {
1674  int64_t missing_source_frame = -1;
1675  if (missing_video_frames.count(requested_frame))
1676  missing_source_frame = missing_video_frames.find(requested_frame)->second;
1677  else if (missing_audio_frames.count(requested_frame))
1678  missing_source_frame = missing_audio_frames.find(requested_frame)->second;
1679 
1680  // Increment missing source frame check count (or init to 1)
1681  if (checked_frames.count(missing_source_frame) == 0)
1682  checked_frames[missing_source_frame] = 1;
1683  else
1684  checked_frames[missing_source_frame]++;
1685 
1686  // Get the previous frame of this missing frame (if it's available in missing cache)
1687  std::shared_ptr<Frame> parent_frame = missing_frames.GetFrame(missing_source_frame);
1688  if (parent_frame == NULL) {
1689  parent_frame = final_cache.GetFrame(missing_source_frame);
1690  if (parent_frame != NULL) {
1691  // Add missing final frame to missing cache
1692  missing_frames.Add(parent_frame);
1693  }
1694  }
1695 
1696  // Create blank missing frame
1697  std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
1698 
1699  // Debug output
1700  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame, "", -1, "", -1, "", -1);
1701 
1702  // If previous frame found, copy image from previous to missing frame (else we'll just wait a bit and try again later)
1703  if (parent_frame != NULL) {
1704  // Debug output
1705  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame, "", -1, "", -1, "", -1);
1706 
1707  // Add this frame to the processed map (since it's already done)
1708  std::shared_ptr<QImage> parent_image = parent_frame->GetImage();
1709  if (parent_image) {
1710  missing_frame->AddImage(std::shared_ptr<QImage>(new QImage(*parent_image)));
1711 
1712  processed_video_frames[missing_frame->number] = missing_frame->number;
1713  processed_audio_frames[missing_frame->number] = missing_frame->number;
1714 
1715  // Move frame to final cache
1716  final_cache.Add(missing_frame);
1717 
1718  // Remove frame from working cache
1719  working_cache.Remove(missing_frame->number);
1720 
1721  // Update last_frame processed
1722  last_frame = missing_frame->number;
1723  }
1724  }
1725 
1726  }
1727 
1728  return found_missing_frame;
1729 }
1730 
1731 // Check the working queue, and move finished frames to the finished queue
1732 void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_frame)
1733 {
1734  // Loop through all working queue frames
1735  bool checked_count_tripped = false;
1736  int max_checked_count = 80;
1737 
1738  while (true)
1739  {
1740  // Get the front frame of working cache
1741  std::shared_ptr<Frame> f(working_cache.GetSmallestFrame());
1742 
1743  // Was a frame found?
1744  if (!f)
1745  // No frames found
1746  break;
1747 
1748  // Remove frames which are too old
1749  if (f && f->number < (requested_frame - (OPEN_MP_NUM_PROCESSORS * 2))) {
1750  working_cache.Remove(f->number);
1751  }
1752 
1753  // Check if this frame is 'missing'
1754  CheckMissingFrame(f->number);
1755 
1756  // Init # of times this frame has been checked so far
1757  int checked_count = 0;
1758  int checked_frames_size = 0;
1759 
1760  bool is_video_ready = false;
1761  bool is_audio_ready = false;
1762  { // limit scope of next few lines
1763  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1764  is_video_ready = processed_video_frames.count(f->number);
1765  is_audio_ready = processed_audio_frames.count(f->number);
1766 
1767  // Get check count for this frame
1768  checked_frames_size = checked_frames.size();
1769  if (!checked_count_tripped || f->number >= requested_frame)
1770  checked_count = checked_frames[f->number];
1771  else
1772  // Force checked count over the limit
1773  checked_count = max_checked_count;
1774  }
1775 
1776  if (previous_packet_location.frame == f->number && !end_of_stream)
1777  is_audio_ready = false; // don't finalize the last processed audio frame
1778  bool is_seek_trash = IsPartialFrame(f->number);
1779 
1780  // Adjust for available streams
1781  if (!info.has_video) is_video_ready = true;
1782  if (!info.has_audio) is_audio_ready = true;
1783 
1784  // Make final any frames that get stuck (for whatever reason)
1785  if (checked_count >= max_checked_count && (!is_video_ready || !is_audio_ready)) {
1786  // Debug output
1787  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (exceeded checked_count)", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
1788 
1789  // Trigger checked count tripped mode (clear out all frames before requested frame)
1790  checked_count_tripped = true;
1791 
1792  if (info.has_video && !is_video_ready && last_video_frame) {
1793  // Copy image from last frame
1794  f->AddImage(std::shared_ptr<QImage>(new QImage(*last_video_frame->GetImage())));
1795  is_video_ready = true;
1796  }
1797 
1798  if (info.has_audio && !is_audio_ready) {
1799  // Mark audio as processed, and indicate the frame has audio data
1800  is_audio_ready = true;
1801  }
1802  }
1803 
1804  // Debug output
1805  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
1806 
1807  // Check if working frame is final
1808  if ((!end_of_stream && is_video_ready && is_audio_ready) || end_of_stream || is_seek_trash)
1809  {
1810  // Debug output
1811  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)", "requested_frame", requested_frame, "f->number", f->number, "is_seek_trash", is_seek_trash, "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "end_of_stream", end_of_stream);
1812 
1813  if (!is_seek_trash)
1814  {
1815  // Add missing image (if needed - sometimes end_of_stream causes frames with only audio)
1816  if (info.has_video && !is_video_ready && last_video_frame)
1817  // Copy image from last frame
1818  f->AddImage(std::shared_ptr<QImage>(new QImage(*last_video_frame->GetImage())));
1819 
1820  // Reset counter since last 'final' frame
1821  num_checks_since_final = 0;
1822 
1823  // Move frame to final cache
1824  final_cache.Add(f);
1825 
1826  // Add to missing cache (if another frame depends on it)
1827  {
1828  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1829  if (missing_video_frames_source.count(f->number)) {
1830  // Debug output
1831  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (add frame to missing cache)", "f->number", f->number, "is_seek_trash", is_seek_trash, "Missing Cache Count", missing_frames.Count(), "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "", -1);
1832  missing_frames.Add(f);
1833  }
1834 
1835  // Remove from 'checked' count
1836  checked_frames.erase(f->number);
1837  }
1838 
1839  // Remove frame from working cache
1840  working_cache.Remove(f->number);
1841 
1842  // Update last frame processed
1843  last_frame = f->number;
1844 
1845  } else {
1846  // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
1847  working_cache.Remove(f->number);
1848  }
1849  }
1850  else
1851  // Stop looping
1852  break;
1853  }
1854 }
1855 
1856 // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
1857 void FFmpegReader::CheckFPS()
1858 {
1859  check_fps = true;
1860  AV_ALLOCATE_IMAGE(pFrame, AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx), info.width, info.height);
1861 
1862  int first_second_counter = 0;
1863  int second_second_counter = 0;
1864  int third_second_counter = 0;
1865  int forth_second_counter = 0;
1866  int fifth_second_counter = 0;
1867 
1868  int iterations = 0;
1869  int threshold = 500;
1870 
1871  // Loop through the stream
1872  while (true)
1873  {
1874  // Get the next packet (if any)
1875  if (GetNextPacket() < 0)
1876  // Break loop when no more packets found
1877  break;
1878 
1879  // Video packet
1880  if (packet->stream_index == videoStream)
1881  {
1882  // Check if the AVFrame is finished and set it
1883  if (GetAVFrame())
1884  {
1885  // Update PTS / Frame Offset (if any)
1886  UpdatePTSOffset(true);
1887 
1888  // Get PTS of this packet
1889  int64_t pts = GetVideoPTS();
1890 
1891  // Remove pFrame
1892  RemoveAVFrame(pFrame);
1893 
1894  // Apply PTS offset
1895  pts += video_pts_offset;
1896 
1897  // Get the video packet start time (in seconds)
1898  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1899 
1900  // Increment the correct counter
1901  if (video_seconds <= 1.0)
1902  first_second_counter++;
1903  else if (video_seconds > 1.0 && video_seconds <= 2.0)
1904  second_second_counter++;
1905  else if (video_seconds > 2.0 && video_seconds <= 3.0)
1906  third_second_counter++;
1907  else if (video_seconds > 3.0 && video_seconds <= 4.0)
1908  forth_second_counter++;
1909  else if (video_seconds > 4.0 && video_seconds <= 5.0)
1910  fifth_second_counter++;
1911  else
1912  // Too far
1913  break;
1914  }
1915  }
1916 
1917  // Increment counters
1918  iterations++;
1919 
1920  // Give up (if threshold exceeded)
1921  if (iterations > threshold)
1922  break;
1923  }
1924 
1925  // Double check that all counters have greater than zero (or give up)
1926  if (second_second_counter == 0 || third_second_counter == 0 || forth_second_counter == 0 || fifth_second_counter == 0)
1927  {
1928  // Seek to frame 1
1929  Seek(1);
1930 
1931  // exit with no changes to FPS (not enough data to calculate)
1932  return;
1933  }
1934 
1935  int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
1936  int avg_fps = round(sum_fps / 4.0f);
1937 
1938  // Sometimes the FPS is incorrectly detected by FFmpeg. If the 1st and 2nd seconds counters
1939  // agree with each other, we are going to adjust the FPS of this reader instance. Otherwise, print
1940  // a warning message.
1941 
1942  // Get diff from actual frame rate
1943  double fps = info.fps.ToDouble();
1944  double diff = fps - double(avg_fps);
1945 
1946  // Is difference bigger than 1 frame?
1947  if (diff <= -1 || diff >= 1)
1948  {
1949  // Compare to half the frame rate (the most common type of issue)
1950  double half_fps = Fraction(info.fps.num / 2, info.fps.den).ToDouble();
1951  diff = half_fps - double(avg_fps);
1952 
1953  // Is difference bigger than 1 frame?
1954  if (diff <= -1 || diff >= 1)
1955  {
1956  // Update FPS for this reader instance
1957  info.fps = Fraction(avg_fps, 1);
1958  }
1959  else
1960  {
1961  // Update FPS for this reader instance (to 1/2 the original framerate)
1962  info.fps = Fraction(info.fps.num / 2, info.fps.den);
1963  }
1964  }
1965 
1966  // Seek to frame 1
1967  Seek(1);
1968 }
1969 
1970 // Remove AVFrame from cache (and deallocate it's memory)
1971 void FFmpegReader::RemoveAVFrame(AVFrame* remove_frame)
1972 {
1973  // Remove pFrame (if exists)
1974  if (remove_frame)
1975  {
1976  // Free memory
1977  av_freep(&remove_frame->data[0]);
1978  }
1979 }
1980 
1981 // Remove AVPacket from cache (and deallocate it's memory)
1982 void FFmpegReader::RemoveAVPacket(AVPacket* remove_packet)
1983 {
1984  // deallocate memory for packet
1985  AV_FREE_PACKET(remove_packet);
1986 
1987  // Delete the object
1988  delete remove_packet;
1989 }
1990 
1991 /// Get the smallest video frame that is still being processed
1992 int64_t FFmpegReader::GetSmallestVideoFrame()
1993 {
1994  // Loop through frame numbers
1995  map<int64_t, int64_t>::iterator itr;
1996  int64_t smallest_frame = -1;
1997  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1998  for(itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr)
1999  {
2000  if (itr->first < smallest_frame || smallest_frame == -1)
2001  smallest_frame = itr->first;
2002  }
2003 
2004  // Return frame number
2005  return smallest_frame;
2006 }
2007 
2008 /// Get the smallest audio frame that is still being processed
2009 int64_t FFmpegReader::GetSmallestAudioFrame()
2010 {
2011  // Loop through frame numbers
2012  map<int64_t, int64_t>::iterator itr;
2013  int64_t smallest_frame = -1;
2014  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
2015  for(itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr)
2016  {
2017  if (itr->first < smallest_frame || smallest_frame == -1)
2018  smallest_frame = itr->first;
2019  }
2020 
2021  // Return frame number
2022  return smallest_frame;
2023 }
2024 
2025 // Generate JSON string of this object
2027 
2028  // Return formatted string
2029  return JsonValue().toStyledString();
2030 }
2031 
2032 // Generate Json::JsonValue for this object
2034 
2035  // Create root json object
2036  Json::Value root = ReaderBase::JsonValue(); // get parent properties
2037  root["type"] = "FFmpegReader";
2038  root["path"] = path;
2039 
2040  // return JsonValue
2041  return root;
2042 }
2043 
2044 // Load JSON string into this object
2045 void FFmpegReader::SetJson(string value) {
2046 
2047  // Parse JSON string into JSON objects
2048  Json::Value root;
2049  Json::Reader reader;
2050  bool success = reader.parse( value, root );
2051  if (!success)
2052  // Raise exception
2053  throw InvalidJSON("JSON could not be parsed (or is invalid)", "");
2054 
2055  try
2056  {
2057  // Set all values that match
2058  SetJsonValue(root);
2059  }
2060  catch (exception e)
2061  {
2062  // Error parsing JSON (or missing keys)
2063  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)", "");
2064  }
2065 }
2066 
2067 // Load Json::JsonValue into this object
2068 void FFmpegReader::SetJsonValue(Json::Value root) {
2069 
2070  // Set parent data
2072 
2073  // Set data from Json (if key is found)
2074  if (!root["path"].isNull())
2075  path = root["path"].asString();
2076 
2077  // Re-Open path, and re-init everything (if needed)
2078  if (is_open)
2079  {
2080  Close();
2081  Open();
2082  }
2083 }
#define AV_RESET_FRAME(av_frame)
int max_height
The maximium image height needed by this clip (used for optimizations)
Definition: ReaderBase.h:104
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
Definition: Fraction.h:44
#define AV_FIND_DECODER_CODEC_ID(av_stream)
CriticalSection processingCriticalSection
Definition: ReaderBase.h:101
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:83
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:67
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
std::shared_ptr< Frame > GetFrame(int64_t requested_frame)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:41
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
float duration
Length of time (in seconds)
Definition: ReaderBase.h:64
string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:79
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:71
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:46
void Add(std::shared_ptr< Frame > frame)
Add a Frame to the cache.
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define OPEN_MP_NUM_PROCESSORS
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
string Json()
Get and Set JSON methods.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:234
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:61
void Close()
Close File.
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
Definition: ReaderBase.h:72
int64_t file_size
Size of file (in bytes)
Definition: ReaderBase.h:65
FFmpegReader(string path)
std::shared_ptr< Frame > GetSmallestFrame()
Get the smallest frame number.
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:80
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:62
#define AV_FREE_CONTEXT(av_context)
Exception when no valid codec is found for a file.
Definition: Exceptions.h:122
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:84
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:74
#define AV_FREE_PACKET(av_packet)
Exception when no streams are found in the file.
Definition: Exceptions.h:192
int height
The height of the video (in pixels)
Definition: ReaderBase.h:66
void SetJson(string value)
Load JSON string into this object.
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
Definition: Exceptions.h:132
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
Definition: ZmqLogger.cpp:162
This class represents a fraction.
Definition: Fraction.h:42
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
Definition: ReaderBase.cpp:113
auto AV_GET_CODEC_CONTEXT
#define PixelFormat
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
Definition: ReaderBase.cpp:168
ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:112
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Definition: Fraction.cpp:51
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:69
#define AV_GET_IMAGE_SIZE(pix_fmt, width, height)
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:76
Exception for frames that are out of bounds.
Definition: Exceptions.h:202
std::map< string, string > metadata
An optional map/dictionary of metadata for this reader.
Definition: ReaderBase.h:86
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:38
int64_t Count()
Count the frames in the queue.
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
Definition: ReaderBase.h:71
This namespace is the default namespace for all code in the openshot library.
Exception for invalid JSON.
Definition: Exceptions.h:152
#define AV_GET_CODEC_TYPE(av_stream)
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:68
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: FFmpegReader.h:59
void Open()
Open File - which is called by the constructor automatically.
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:70
#define PIX_FMT_RGBA
Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:85
string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:73
void Remove(int64_t frame_number)
Remove a specific frame.
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:219
~FFmpegReader()
Destructor.
int den
Denominator for the fraction.
Definition: Fraction.h:45
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:82
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:75
#define AV_ALLOCATE_IMAGE(av_frame, pix_fmt, width, height)
int max_width
The maximum image width needed by this clip (used for optimizations)
Definition: ReaderBase.h:103
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:519
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:46
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:81