31 #include "../include/FFmpegReader.h" 36 : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
37 audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
38 check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
39 prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
40 current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0), packet(NULL) {
44 avcodec_register_all();
57 : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
58 audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
59 check_fps(false),
enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
60 prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
61 current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0), packet(NULL) {
65 avcodec_register_all();
89 if (abs(location.
frame - frame) >= 2)
95 int64_t diff = samples_per_frame * (location.
frame - frame) + location.
sample_start - sample_start;
96 if (abs(diff) <= amount)
113 if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
114 throw InvalidFile(
"File could not be opened.", path);
117 if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
123 for (
unsigned int i = 0; i < pFormatCtx->nb_streams; i++)
126 if (
AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
130 if (
AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
134 if (videoStream == -1 && audioStream == -1)
135 throw NoStreamsFound(
"No video or audio streams found in this file.", path);
138 if (videoStream != -1)
144 pStream = pFormatCtx->streams[videoStream];
150 AVCodec *pCodec = avcodec_find_decoder(codecId);
156 if (pCodec == NULL) {
157 throw InvalidCodec(
"A valid video codec could not be found for this file.", path);
161 AVDictionary *opts = NULL;
162 av_dict_set(&opts,
"strict",
"experimental", 0);
165 if (avcodec_open2(pCodecCtx, pCodec, &opts) < 0)
166 throw InvalidCodec(
"A video codec was found, but could not be opened.", path);
176 if (audioStream != -1)
182 aStream = pFormatCtx->streams[audioStream];
188 AVCodec *aCodec = avcodec_find_decoder(codecId);
194 if (aCodec == NULL) {
195 throw InvalidCodec(
"A valid audio codec could not be found for this file.", path);
199 AVDictionary *opts = NULL;
200 av_dict_set(&opts,
"strict",
"experimental", 0);
203 if (avcodec_open2(aCodecCtx, aCodec, &opts) < 0)
204 throw InvalidCodec(
"An audio codec was found, but could not be opened.", path);
214 AVDictionaryEntry *tag = NULL;
215 while ((tag = av_dict_get(pFormatCtx->metadata,
"", tag, AV_DICT_IGNORE_SUFFIX))) {
216 QString str_key = tag->key;
217 QString str_value = tag->value;
218 info.
metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
222 previous_packet_location.
frame = -1;
243 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::Close",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
248 avcodec_flush_buffers(pCodecCtx);
253 avcodec_flush_buffers(aCodecCtx);
259 working_cache.
Clear();
260 missing_frames.
Clear();
265 processed_video_frames.clear();
266 processed_audio_frames.clear();
267 processing_video_frames.clear();
268 processing_audio_frames.clear();
269 missing_audio_frames.clear();
270 missing_video_frames.clear();
271 missing_audio_frames_source.clear();
272 missing_video_frames_source.clear();
273 checked_frames.clear();
277 avformat_close_input(&pFormatCtx);
278 av_freep(&pFormatCtx);
282 largest_frame_processed = 0;
283 seek_audio_frame_found = 0;
284 seek_video_frame_found = 0;
285 current_video_frame = 0;
286 has_missing_frames =
false;
290 void FFmpegReader::UpdateAudioInfo()
294 info.
file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
308 if (aStream->duration > 0.0f && aStream->duration >
info.
duration)
332 AVDictionaryEntry *tag = NULL;
333 while ((tag = av_dict_get(aStream->metadata,
"", tag, AV_DICT_IGNORE_SUFFIX))) {
334 QString str_key = tag->key;
335 QString str_value = tag->value;
336 info.
metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
340 void FFmpegReader::UpdateVideoInfo()
344 info.
file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
356 if (pStream->sample_aspect_ratio.num != 0)
406 is_duration_known =
false;
411 is_duration_known =
true;
431 AVDictionaryEntry *tag = NULL;
432 while ((tag = av_dict_get(pStream->metadata,
"", tag, AV_DICT_IGNORE_SUFFIX))) {
433 QString str_key = tag->key;
434 QString str_value = tag->value;
435 info.
metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
444 throw ReaderClosed(
"The FFmpegReader is closed. Call Open() before calling this method.", path);
447 if (requested_frame < 1)
453 throw InvalidFile(
"Could not detect the duration of the video or audio stream.", path);
456 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"requested_frame", requested_frame,
"last_frame", last_frame,
"", -1,
"", -1,
"", -1,
"", -1);
462 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"returned cached frame", requested_frame,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
469 #pragma omp critical (ReadStream) 472 if (has_missing_frames)
473 CheckMissingFrame(requested_frame);
477 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"returned cached frame on 2nd look", requested_frame,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
487 if (last_frame == 0 && requested_frame != 1)
492 int64_t diff = requested_frame - last_frame;
493 if (diff >= 1 && diff <= 20)
496 frame = ReadStream(requested_frame);
503 Seek(requested_frame);
513 frame = ReadStream(requested_frame);
522 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame)
525 bool end_of_stream =
false;
526 bool check_seek =
false;
527 bool frame_finished =
false;
528 int packet_error = -1;
531 int packets_processed = 0;
533 int max_packets = 4096;
538 omp_set_nested(
true);
541 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream",
"requested_frame", requested_frame,
"OPEN_MP_NUM_PROCESSORS",
OPEN_MP_NUM_PROCESSORS,
"", -1,
"", -1,
"", -1,
"", -1);
551 packet_error = GetNextPacket();
553 int processing_video_frames_size = 0;
554 int processing_audio_frames_size = 0;
557 processing_video_frames_size = processing_video_frames.size();
558 processing_audio_frames_size = processing_audio_frames.size();
562 while (processing_video_frames_size + processing_audio_frames_size >= minimum_packets) {
565 processing_video_frames_size = processing_video_frames.size();
566 processing_audio_frames_size = processing_audio_frames.size();
570 if (packet_error < 0)
573 end_of_stream =
true;
578 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream (GetNextPacket)",
"requested_frame", requested_frame,
"processing_video_frames_size", processing_video_frames_size,
"processing_audio_frames_size", processing_audio_frames_size,
"minimum_packets", minimum_packets,
"packets_processed", packets_processed,
"is_seeking", is_seeking);
584 num_packets_since_video_frame = 0;
588 #pragma omp critical (openshot_seek) 589 check_seek = CheckSeek(
true);
599 frame_finished = GetAVFrame();
605 UpdatePTSOffset(
true);
608 ProcessVideoPacket(requested_frame);
613 else if (
info.
has_audio && packet->stream_index == audioStream)
616 num_packets_since_video_frame++;
620 #pragma omp critical (openshot_seek) 621 check_seek = CheckSeek(
false);
631 UpdatePTSOffset(
false);
641 bool is_cache_found =
false;
644 CheckMissingFrame(requested_frame);
647 CheckWorkingFrames(
false, requested_frame);
657 if ((is_cache_found && packets_processed >= minimum_packets) || packets_processed > max_packets)
666 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream (Completed)",
"packets_processed", packets_processed,
"end_of_stream", end_of_stream,
"largest_frame_processed", largest_frame_processed,
"Working Cache Count", working_cache.
Count(),
"", -1,
"", -1);
671 CheckWorkingFrames(end_of_stream, requested_frame);
688 std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
697 int FFmpegReader::GetNextPacket()
699 int found_packet = 0;
700 AVPacket *next_packet =
new AVPacket();
701 found_packet = av_read_frame(pFormatCtx, next_packet);
705 RemoveAVPacket(packet);
709 if (found_packet >= 0)
712 packet = next_packet;
720 bool FFmpegReader::GetAVFrame()
722 int frameFinished = -1;
727 #pragma omp critical (packet_cache) 731 ret = avcodec_send_packet(pCodecCtx, packet);
732 if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
733 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAVFrame (Packet not sent)",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
736 pFrame =
new AVFrame();
738 ret = avcodec_receive_frame(pCodecCtx, next_frame);
739 if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
744 if (frameFinished == 0 ) {
746 av_image_alloc(pFrame->data, pFrame->linesize,
info.
width,
info.
height, (AVPixelFormat)(pStream->codecpar->format), 1);
747 av_image_copy(pFrame->data, pFrame->linesize, (
const uint8_t**)next_frame->data, next_frame->linesize,
749 if (!check_interlace) {
750 check_interlace =
true;
758 avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
766 av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt,
info.
width,
770 if (!check_interlace) {
771 check_interlace =
true;
783 return frameFinished;
787 bool FFmpegReader::CheckSeek(
bool is_video)
794 if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
802 int64_t max_seeked_frame = seek_audio_frame_found;
803 if (seek_video_frame_found > max_seeked_frame)
804 max_seeked_frame = seek_video_frame_found;
807 if (max_seeked_frame >= seeking_frame)
810 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckSeek (Too far, seek again)",
"is_video_seek", is_video_seek,
"max_seeked_frame", max_seeked_frame,
"seeking_frame", seeking_frame,
"seeking_pts", seeking_pts,
"seek_video_frame_found", seek_video_frame_found,
"seek_audio_frame_found", seek_audio_frame_found);
813 Seek(seeking_frame - (10 * seek_count * seek_count));
818 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckSeek (Successful)",
"is_video_seek", is_video_seek,
"current_pts", packet->pts,
"seeking_pts", seeking_pts,
"seeking_frame", seeking_frame,
"seek_video_frame_found", seek_video_frame_found,
"seek_audio_frame_found", seek_audio_frame_found);
832 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame)
835 int64_t current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
838 if (!seek_video_frame_found && is_seeking)
839 seek_video_frame_found = current_frame;
842 if ((current_frame < (requested_frame - 20)) or (current_frame == -1))
845 RemoveAVFrame(pFrame);
848 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (Skipped)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"", -1,
"", -1,
"", -1,
"", -1);
855 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (Before)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"", -1,
"", -1,
"", -1,
"", -1);
862 AVFrame *my_frame = pFrame;
866 processing_video_frames[current_frame] = current_frame;
868 #pragma omp task firstprivate(current_frame, my_frame, height, width, video_length, pix_fmt) 871 AVFrame *pFrameRGB = NULL;
873 uint8_t *buffer = NULL;
877 if (pFrameRGB == NULL)
878 throw OutOfBoundsFrame(
"Convert Image Broke!", current_frame, video_length);
883 int original_height = height;
886 float ratio = float(width) / float(height);
887 int possible_width = round(
max_height * ratio);
888 int possible_height = round(
max_width / ratio);
892 width = possible_width;
897 height = possible_height;
904 #pragma omp critical (video_buffer) 905 buffer = (uint8_t *) av_malloc(numBytes *
sizeof(uint8_t));
914 sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
915 original_height, pFrameRGB->data, pFrameRGB->linesize);
918 std::shared_ptr<Frame> f = CreateFrame(current_frame);
921 f->AddImage(width, height, 4, QImage::Format_RGBA8888, buffer);
924 working_cache.
Add(f);
927 #pragma omp critical (video_buffer) 928 last_video_frame = f;
935 RemoveAVFrame(my_frame);
936 sws_freeContext(img_convert_ctx);
941 processing_video_frames.erase(current_frame);
942 processed_video_frames[current_frame] = current_frame;
946 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (After)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"f->number", f->number,
"", -1,
"", -1,
"", -1);
953 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_frame,
int starting_sample)
956 if (!seek_audio_frame_found && is_seeking)
957 seek_audio_frame_found = target_frame;
960 if (target_frame < (requested_frame - 20))
963 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Skipped)",
"requested_frame", requested_frame,
"target_frame", target_frame,
"starting_sample", starting_sample,
"", -1,
"", -1,
"", -1);
970 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Before)",
"requested_frame", requested_frame,
"target_frame", target_frame,
"starting_sample", starting_sample,
"", -1,
"", -1,
"", -1);
973 int frame_finished = 0;
977 int packet_samples = 0;
982 #pragma omp critical (ProcessAudioPacket) 987 while((packet->size > 0 || (!packet->data && frame_finished)) && ret >= 0) {
989 ret = avcodec_send_packet(aCodecCtx, packet);
990 if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
991 avcodec_send_packet(aCodecCtx, NULL);
996 ret = avcodec_receive_frame(aCodecCtx, audio_frame);
999 if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1000 avcodec_flush_buffers(aCodecCtx);
1004 ret = frame_finished;
1007 if (!packet->data && !frame_finished)
1012 int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
1016 if (frame_finished) {
1020 int plane_size = -1;
1021 data_size = av_samples_get_buffer_size(&plane_size,
1023 audio_frame->nb_samples,
1031 int pts_remaining_samples = packet_samples /
info.
channels;
1034 int64_t adjusted_pts = packet->pts + audio_pts_offset;
1039 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Decode Info A)",
"pts_counter", pts_counter,
"PTS", adjusted_pts,
"Offset", audio_pts_offset,
"PTS Diff", adjusted_pts - prev_pts,
"Samples", pts_remaining_samples,
"Sample PTS ratio",
float(adjusted_pts - prev_pts) / pts_remaining_samples);
1040 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Decode Info B)",
"Sample Diff", pts_remaining_samples - prev_samples - prev_pts,
"Total", pts_total,
"PTS Seconds", audio_seconds,
"Sample Seconds", sample_seconds,
"Seconds Diff", audio_seconds - sample_seconds,
"raw samples", packet_samples);
1043 prev_pts = adjusted_pts;
1044 pts_total += pts_remaining_samples;
1046 prev_samples = pts_remaining_samples;
1051 processing_audio_frames.insert(pair<int, int>(previous_packet_location.
frame, previous_packet_location.
frame));
1054 while (pts_remaining_samples)
1060 int samples = samples_per_frame - previous_packet_location.
sample_start;
1061 if (samples > pts_remaining_samples)
1062 samples = pts_remaining_samples;
1065 pts_remaining_samples -= samples;
1067 if (pts_remaining_samples > 0) {
1069 previous_packet_location.
frame++;
1075 processing_audio_frames.insert(pair<int, int>(previous_packet_location.
frame, previous_packet_location.
frame));
1088 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (ReSample)",
"packet_samples", packet_samples,
"info.channels",
info.
channels,
"info.sample_rate",
info.
sample_rate,
"aCodecCtx->sample_fmt",
AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx),
"AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16,
"", -1);
1093 audio_converted->nb_samples = audio_frame->nb_samples;
1094 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1096 AVAudioResampleContext *avr = NULL;
1100 avr = avresample_alloc_context();
1104 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1109 int r = avresample_open(avr);
1112 nb_samples = avresample_convert(avr,
1113 audio_converted->data,
1114 audio_converted->linesize[0],
1115 audio_converted->nb_samples,
1117 audio_frame->linesize[0],
1118 audio_frame->nb_samples);
1121 memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels);
1124 avresample_close(avr);
1125 avresample_free(&avr);
1129 av_free(audio_converted->data[0]);
1132 int64_t starting_frame_number = -1;
1133 bool partial_frame =
true;
1134 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
1137 starting_frame_number = target_frame;
1138 int channel_buffer_size = packet_samples /
info.
channels;
1139 float *channel_buffer =
new float[channel_buffer_size];
1142 for (
int z = 0; z < channel_buffer_size; z++)
1143 channel_buffer[z] = 0.0f;
1149 for (
int sample = 0; sample < packet_samples; sample++)
1152 if (channel_filter == channel)
1155 channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1171 int start = starting_sample;
1172 int remaining_samples = channel_buffer_size;
1173 float *iterate_channel_buffer = channel_buffer;
1174 while (remaining_samples > 0)
1180 int samples = samples_per_frame - start;
1181 if (samples > remaining_samples)
1182 samples = remaining_samples;
1185 std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1188 if (samples_per_frame == start + samples)
1189 partial_frame =
false;
1191 partial_frame =
true;
1195 f->AddAudio(
true, channel_filter, start, iterate_channel_buffer, samples, 0.98f);
1198 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (f->AddAudio)",
"frame", starting_frame_number,
"start", start,
"samples", samples,
"channel", channel_filter,
"partial_frame", partial_frame,
"samples_per_frame", samples_per_frame);
1201 working_cache.
Add(f);
1204 remaining_samples -= samples;
1207 if (remaining_samples > 0)
1208 iterate_channel_buffer += samples;
1211 starting_frame_number++;
1218 delete[] channel_buffer;
1219 channel_buffer = NULL;
1220 iterate_channel_buffer = NULL;
1231 for (int64_t f = target_frame; f < starting_frame_number; f++) {
1235 processing_audio_frames.erase(processing_audio_frames.find(f));
1238 if (processing_audio_frames.count(f) == 0)
1240 processed_audio_frames[f] = f;
1243 if (target_frame == starting_frame_number) {
1245 processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1253 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (After)",
"requested_frame", requested_frame,
"starting_frame", target_frame,
"end_frame", starting_frame_number - 1,
"", -1,
"", -1,
"", -1);
1260 void FFmpegReader::Seek(int64_t requested_frame)
1263 if (requested_frame < 1)
1264 requested_frame = 1;
1268 int processing_video_frames_size = 0;
1269 int processing_audio_frames_size = 0;
1272 processing_video_frames_size = processing_video_frames.size();
1273 processing_audio_frames_size = processing_audio_frames.size();
1277 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::Seek",
"requested_frame", requested_frame,
"seek_count", seek_count,
"last_frame", last_frame,
"processing_video_frames_size", processing_video_frames_size,
"processing_audio_frames_size", processing_audio_frames_size,
"video_pts_offset", video_pts_offset);
1280 while (processing_video_frames_size + processing_audio_frames_size > 0) {
1283 processing_video_frames_size = processing_video_frames.size();
1284 processing_audio_frames_size = processing_audio_frames.size();
1288 working_cache.
Clear();
1289 missing_frames.
Clear();
1294 processing_audio_frames.clear();
1295 processing_video_frames.clear();
1296 processed_video_frames.clear();
1297 processed_audio_frames.clear();
1298 missing_audio_frames.clear();
1299 missing_video_frames.clear();
1300 missing_audio_frames_source.clear();
1301 missing_video_frames_source.clear();
1302 checked_frames.clear();
1307 current_video_frame = 0;
1308 largest_frame_processed = 0;
1309 num_checks_since_final = 0;
1310 num_packets_since_video_frame = 0;
1311 has_missing_frames =
false;
1320 if (requested_frame - buffer_amount < 20)
1332 if (seek_count == 1) {
1335 seeking_pts = ConvertFrameToVideoPTS(1);
1337 seek_audio_frame_found = 0;
1338 seek_video_frame_found = 0;
1343 bool seek_worked =
false;
1344 int64_t seek_target = 0;
1349 seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1351 fprintf(stderr,
"%s: error while seeking video stream\n", pFormatCtx->filename);
1355 is_video_seek =
true;
1363 seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1365 fprintf(stderr,
"%s: error while seeking audio stream\n", pFormatCtx->filename);
1369 is_video_seek =
false;
1379 avcodec_flush_buffers(aCodecCtx);
1383 avcodec_flush_buffers(pCodecCtx);
1386 previous_packet_location.
frame = -1;
1391 if (seek_count == 1) {
1393 seeking_pts = seek_target;
1394 seeking_frame = requested_frame;
1396 seek_audio_frame_found = 0;
1397 seek_video_frame_found = 0;
1423 int64_t FFmpegReader::GetVideoPTS()
1425 int64_t current_pts = 0;
1426 if(packet->dts != AV_NOPTS_VALUE)
1427 current_pts = packet->dts;
1434 void FFmpegReader::UpdatePTSOffset(
bool is_video)
1440 if (video_pts_offset == 99999)
1446 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::UpdatePTSOffset (Video)",
"video_pts_offset", video_pts_offset,
"is_video", is_video,
"", -1,
"", -1,
"", -1,
"", -1);
1452 if (audio_pts_offset == 99999)
1458 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::UpdatePTSOffset (Audio)",
"audio_pts_offset", audio_pts_offset,
"is_video", is_video,
"", -1,
"", -1,
"", -1,
"", -1);
1464 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts)
1467 pts = pts + video_pts_offset;
1468 int64_t previous_video_frame = current_video_frame;
1477 if (current_video_frame == 0)
1478 current_video_frame = frame;
1482 if (frame == previous_video_frame) {
1488 current_video_frame++;
1490 if (current_video_frame < frame)
1492 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)",
"calculated frame", frame,
"previous_video_frame", previous_video_frame,
"current_video_frame", current_video_frame,
"", -1,
"", -1,
"", -1);
1497 while (current_video_frame < frame) {
1498 if (!missing_video_frames.count(current_video_frame)) {
1499 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)",
"current_video_frame", current_video_frame,
"previous_video_frame", previous_video_frame,
"", -1,
"", -1,
"", -1,
"", -1);
1500 missing_video_frames.insert(pair<int64_t, int64_t>(current_video_frame, previous_video_frame));
1501 missing_video_frames_source.insert(pair<int64_t, int64_t>(previous_video_frame, current_video_frame));
1505 has_missing_frames =
true;
1508 current_video_frame++;
1517 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number)
1526 return video_pts - video_pts_offset;
1530 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number)
1539 return audio_pts - audio_pts_offset;
1543 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts)
1546 pts = pts + audio_pts_offset;
1555 int64_t whole_frame = int64_t(frame);
1558 double sample_start_percentage = frame - double(whole_frame);
1564 int sample_start = round(
double(samples_per_frame) * sample_start_percentage);
1567 if (whole_frame < 1)
1569 if (sample_start < 0)
1576 if (previous_packet_location.
frame != -1) {
1577 if (location.
is_near(previous_packet_location, samples_per_frame, samples_per_frame))
1579 int64_t orig_frame = location.
frame;
1584 location.
frame = previous_packet_location.
frame;
1587 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)",
"Source Frame", orig_frame,
"Source Audio Sample", orig_start,
"Target Frame", location.
frame,
"Target Audio Sample", location.
sample_start,
"pts", pts,
"", -1);
1591 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)",
"Previous location frame", previous_packet_location.
frame,
"Target Frame", location.
frame,
"Target Audio Sample", location.
sample_start,
"pts", pts,
"", -1,
"", -1);
1594 for (int64_t audio_frame = previous_packet_location.
frame; audio_frame < location.
frame; audio_frame++) {
1595 if (!missing_audio_frames.count(audio_frame)) {
1596 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (tracking missing frame)",
"missing_audio_frame", audio_frame,
"previous_audio_frame", previous_packet_location.
frame,
"new location frame", location.
frame,
"", -1,
"", -1,
"", -1);
1597 missing_audio_frames.insert(pair<int64_t, int64_t>(previous_packet_location.
frame - 1, audio_frame));
1604 previous_packet_location = location;
1611 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame)
1614 std::shared_ptr<Frame> output = working_cache.
GetFrame(requested_frame);
1623 working_cache.
Add(output);
1626 if (requested_frame > largest_frame_processed)
1627 largest_frame_processed = requested_frame;
1635 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
1638 bool seek_trash =
false;
1639 int64_t max_seeked_frame = seek_audio_frame_found;
1640 if (seek_video_frame_found > max_seeked_frame)
1641 max_seeked_frame = seek_video_frame_found;
1642 if ((
info.
has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
1643 (
info.
has_video && seek_video_frame_found && max_seeked_frame >= requested_frame))
1650 bool FFmpegReader::CheckMissingFrame(int64_t requested_frame)
1656 int checked_count = 0;
1659 if (checked_frames.count(requested_frame) == 0)
1660 checked_frames[requested_frame] = 1;
1662 checked_frames[requested_frame]++;
1663 checked_count = checked_frames[requested_frame];
1666 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame",
"requested_frame", requested_frame,
"has_missing_frames", has_missing_frames,
"missing_video_frames.size()", missing_video_frames.size(),
"checked_count", checked_count,
"", -1,
"", -1);
1669 map<int64_t, int64_t>::iterator itr;
1670 bool found_missing_frame =
false;
1673 if (missing_video_frames.count(requested_frame) || missing_audio_frames.count(requested_frame)) {
1674 int64_t missing_source_frame = -1;
1675 if (missing_video_frames.count(requested_frame))
1676 missing_source_frame = missing_video_frames.find(requested_frame)->second;
1677 else if (missing_audio_frames.count(requested_frame))
1678 missing_source_frame = missing_audio_frames.find(requested_frame)->second;
1681 if (checked_frames.count(missing_source_frame) == 0)
1682 checked_frames[missing_source_frame] = 1;
1684 checked_frames[missing_source_frame]++;
1687 std::shared_ptr<Frame> parent_frame = missing_frames.
GetFrame(missing_source_frame);
1688 if (parent_frame == NULL) {
1690 if (parent_frame != NULL) {
1692 missing_frames.
Add(parent_frame);
1697 std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
1700 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"missing_source_frame", missing_source_frame,
"", -1,
"", -1,
"", -1);
1703 if (parent_frame != NULL) {
1705 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"missing_source_frame", missing_source_frame,
"", -1,
"", -1,
"", -1);
1708 std::shared_ptr<QImage> parent_image = parent_frame->GetImage();
1710 missing_frame->AddImage(std::shared_ptr<QImage>(
new QImage(*parent_image)));
1712 processed_video_frames[missing_frame->number] = missing_frame->number;
1713 processed_audio_frames[missing_frame->number] = missing_frame->number;
1719 working_cache.
Remove(missing_frame->number);
1722 last_frame = missing_frame->number;
1728 return found_missing_frame;
1732 void FFmpegReader::CheckWorkingFrames(
bool end_of_stream, int64_t requested_frame)
1735 bool checked_count_tripped =
false;
1736 int max_checked_count = 80;
1750 working_cache.
Remove(f->number);
1754 CheckMissingFrame(f->number);
1757 int checked_count = 0;
1758 int checked_frames_size = 0;
1760 bool is_video_ready =
false;
1761 bool is_audio_ready =
false;
1764 is_video_ready = processed_video_frames.count(f->number);
1765 is_audio_ready = processed_audio_frames.count(f->number);
1768 checked_frames_size = checked_frames.size();
1769 if (!checked_count_tripped || f->number >= requested_frame)
1770 checked_count = checked_frames[f->number];
1773 checked_count = max_checked_count;
1776 if (previous_packet_location.
frame == f->number && !end_of_stream)
1777 is_audio_ready =
false;
1778 bool is_seek_trash = IsPartialFrame(f->number);
1785 if (checked_count >= max_checked_count && (!is_video_ready || !is_audio_ready)) {
1787 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (exceeded checked_count)",
"requested_frame", requested_frame,
"frame_number", f->number,
"is_video_ready", is_video_ready,
"is_audio_ready", is_audio_ready,
"checked_count", checked_count,
"checked_frames_size", checked_frames_size);
1790 checked_count_tripped =
true;
1792 if (
info.
has_video && !is_video_ready && last_video_frame) {
1794 f->AddImage(std::shared_ptr<QImage>(
new QImage(*last_video_frame->GetImage())));
1795 is_video_ready =
true;
1800 is_audio_ready =
true;
1805 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames",
"requested_frame", requested_frame,
"frame_number", f->number,
"is_video_ready", is_video_ready,
"is_audio_ready", is_audio_ready,
"checked_count", checked_count,
"checked_frames_size", checked_frames_size);
1808 if ((!end_of_stream && is_video_ready && is_audio_ready) || end_of_stream || is_seek_trash)
1811 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (mark frame as final)",
"requested_frame", requested_frame,
"f->number", f->number,
"is_seek_trash", is_seek_trash,
"Working Cache Count", working_cache.
Count(),
"Final Cache Count",
final_cache.
Count(),
"end_of_stream", end_of_stream);
1818 f->AddImage(std::shared_ptr<QImage>(
new QImage(*last_video_frame->GetImage())));
1821 num_checks_since_final = 0;
1829 if (missing_video_frames_source.count(f->number)) {
1831 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (add frame to missing cache)",
"f->number", f->number,
"is_seek_trash", is_seek_trash,
"Missing Cache Count", missing_frames.
Count(),
"Working Cache Count", working_cache.
Count(),
"Final Cache Count",
final_cache.
Count(),
"", -1);
1832 missing_frames.
Add(f);
1836 checked_frames.erase(f->number);
1840 working_cache.
Remove(f->number);
1843 last_frame = f->number;
1847 working_cache.
Remove(f->number);
1857 void FFmpegReader::CheckFPS()
1862 int first_second_counter = 0;
1863 int second_second_counter = 0;
1864 int third_second_counter = 0;
1865 int forth_second_counter = 0;
1866 int fifth_second_counter = 0;
1869 int threshold = 500;
1875 if (GetNextPacket() < 0)
1880 if (packet->stream_index == videoStream)
1886 UpdatePTSOffset(
true);
1889 int64_t pts = GetVideoPTS();
1892 RemoveAVFrame(pFrame);
1895 pts += video_pts_offset;
1901 if (video_seconds <= 1.0)
1902 first_second_counter++;
1903 else if (video_seconds > 1.0 && video_seconds <= 2.0)
1904 second_second_counter++;
1905 else if (video_seconds > 2.0 && video_seconds <= 3.0)
1906 third_second_counter++;
1907 else if (video_seconds > 3.0 && video_seconds <= 4.0)
1908 forth_second_counter++;
1909 else if (video_seconds > 4.0 && video_seconds <= 5.0)
1910 fifth_second_counter++;
1921 if (iterations > threshold)
1926 if (second_second_counter == 0 || third_second_counter == 0 || forth_second_counter == 0 || fifth_second_counter == 0)
1935 int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
1936 int avg_fps = round(sum_fps / 4.0f);
1944 double diff = fps - double(avg_fps);
1947 if (diff <= -1 || diff >= 1)
1951 diff = half_fps - double(avg_fps);
1954 if (diff <= -1 || diff >= 1)
1971 void FFmpegReader::RemoveAVFrame(AVFrame* remove_frame)
1977 av_freep(&remove_frame->data[0]);
1982 void FFmpegReader::RemoveAVPacket(AVPacket* remove_packet)
1988 delete remove_packet;
1992 int64_t FFmpegReader::GetSmallestVideoFrame()
1995 map<int64_t, int64_t>::iterator itr;
1996 int64_t smallest_frame = -1;
1998 for(itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr)
2000 if (itr->first < smallest_frame || smallest_frame == -1)
2001 smallest_frame = itr->first;
2005 return smallest_frame;
2009 int64_t FFmpegReader::GetSmallestAudioFrame()
2012 map<int64_t, int64_t>::iterator itr;
2013 int64_t smallest_frame = -1;
2015 for(itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr)
2017 if (itr->first < smallest_frame || smallest_frame == -1)
2018 smallest_frame = itr->first;
2022 return smallest_frame;
2037 root[
"type"] =
"FFmpegReader";
2038 root[
"path"] = path;
2049 Json::Reader reader;
2050 bool success = reader.parse( value, root );
2053 throw InvalidJSON(
"JSON could not be parsed (or is invalid)",
"");
2063 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)",
"");
2074 if (!root[
"path"].isNull())
2075 path = root[
"path"].asString();
#define AV_RESET_FRAME(av_frame)
int max_height
The maximium image height needed by this clip (used for optimizations)
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
#define AV_FIND_DECODER_CODEC_ID(av_stream)
CriticalSection processingCriticalSection
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
int width
The width of the video (in pixesl)
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
std::shared_ptr< Frame > GetFrame(int64_t requested_frame)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
float duration
Length of time (in seconds)
string acodec
The name of the audio codec used to encode / decode the video stream.
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
void Add(std::shared_ptr< Frame > frame)
Add a Frame to the cache.
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define OPEN_MP_NUM_PROCESSORS
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
string Json()
Get and Set JSON methods.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Exception when a reader is closed, and a frame is requested.
bool has_video
Determines if this file has a video stream.
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
int64_t file_size
Size of file (in bytes)
FFmpegReader(string path)
std::shared_ptr< Frame > GetSmallestFrame()
Get the smallest frame number.
int audio_bit_rate
The bit rate of the audio stream (in bytes)
bool has_audio
Determines if this file has an audio stream.
#define AV_FREE_CONTEXT(av_context)
Exception when no valid codec is found for a file.
int audio_stream_index
The index of the audio stream.
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
int64_t video_length
The number of frames in the video stream.
#define AV_FREE_PACKET(av_packet)
Exception when no streams are found in the file.
int height
The height of the video (in pixels)
void SetJson(string value)
Load JSON string into this object.
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
This class represents a fraction.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
auto AV_GET_CODEC_CONTEXT
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
ReaderInfo info
Information about the current media file.
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
#define AV_GET_IMAGE_SIZE(pix_fmt, width, height)
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Exception for frames that are out of bounds.
std::map< string, string > metadata
An optional map/dictionary of metadata for this reader.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
int64_t Count()
Count the frames in the queue.
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
This namespace is the default namespace for all code in the openshot library.
Exception for invalid JSON.
#define AV_GET_CODEC_TYPE(av_stream)
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
This struct holds the associated video frame and starting sample # for an audio packet.
void Open()
Open File - which is called by the constructor automatically.
int video_bit_rate
The bit rate of the video stream (in bytes)
Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
string vcodec
The name of the video codec used to encode / decode the video stream.
void Remove(int64_t frame_number)
Remove a specific frame.
CacheMemory final_cache
Final cache object used to hold final frames.
~FFmpegReader()
Destructor.
int den
Denominator for the fraction.
int channels
The number of audio channels used in the audio stream.
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
int video_stream_index
The index of the video stream.
#define AV_ALLOCATE_IMAGE(av_frame, pix_fmt, width, height)
int max_width
The maximum image width needed by this clip (used for optimizations)
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)