diff --git a/AUTHORS b/AUTHORS index 3fc4d32..1afa518 100644 --- a/AUTHORS +++ b/AUTHORS @@ -5,6 +5,7 @@ Contributors up to version 0.9.3: Frédéric Devernay Stefan Eilemann Gabriele Greco +Joe Jordanis Kiriakidis Martin Lambers Tobias Lange diff --git a/src/media_data.cpp b/src/media_data.cpp index 868f0e2..d8819d7 100644 --- a/src/media_data.cpp +++ b/src/media_data.cpp @@ -3,6 +3,7 @@ * * Copyright (C) 2010-2011 * Martin Lambers + * Joe * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -426,6 +427,53 @@ int audio_blob::sample_bits() const return bits; } +subtitle_box::subtitle_box() : + format(text), + language(), + str(), + presentation_start_time(std::numeric_limits::min()), + presentation_stop_time(std::numeric_limits::min()) +{ +} + +std::string subtitle_box::format_info() const +{ + std::string s; + if (language.empty()) + { + s += "Unknown "; + } + else + { + s += language + " "; + } + switch (format) + { + case text: + s += "(text format)"; + } + return s; +} + +std::string subtitle_box::format_name() const +{ + std::string s; + if (language.empty()) + { + s += "unknown-"; + } + else + { + s += language + "-"; + } + switch (format) + { + case text: + s += "text"; + } + return s; +} + parameters::parameters() : stereo_mode(stereo), stereo_mode_swap(false), diff --git a/src/media_data.h b/src/media_data.h index 24a8946..cfa525c 100644 --- a/src/media_data.h +++ b/src/media_data.h @@ -3,6 +3,7 @@ * * Copyright (C) 2010-2011 * Martin Lambers + * Joe * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -157,6 +158,42 @@ public: int sample_bits() const; }; +class subtitle_box +{ +public: + // Format + typedef enum + { + text, // UTF-8 text + /* The following are not yet supported: + image, // Image in BGRA32 format, with box coordinates + ass // Advanced SubStation Alpha (ASS) format + */ + } format_t; + + format_t format; // Subtitle data format + std::string language; // Language information (empty if unknown) + + // Data + std::string str; + + int64_t presentation_start_time; // Presentation timestamp + int64_t presentation_stop_time; // End of presentation timestamp + + // Constructor + subtitle_box(); + + // Does this box contain valid data? + bool is_valid() const + { + return (!str.empty()); + } + + // Return a string describing the format + std::string format_info() const; // Human readable information + std::string format_name() const; // Short code +}; + class parameters : public s11n { public: @@ -207,7 +244,7 @@ public: // Set all uninitialised values to their defaults void set_defaults(); - + // Convert the stereo mode to and from a string representation static std::string stereo_mode_to_string(stereo_mode_t stereo_mode, bool stereo_mode_swap); static void stereo_mode_from_string(const std::string &s, stereo_mode_t &stereo_mode, bool &stereo_mode_swap); diff --git a/src/media_input.cpp b/src/media_input.cpp index 7c7f364..9522f4f 100644 --- a/src/media_input.cpp +++ b/src/media_input.cpp @@ -4,6 +4,7 @@ * Copyright (C) 2010-2011 * Martin Lambers * Frédéric Devernay + * Joe * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -32,9 +33,9 @@ media_input::media_input() : - _active_video_stream(-1), _active_audio_stream(-1), - _have_active_video_read(false), _have_active_audio_read(false), _last_audio_data_size(0), - _initial_skip(0), _duration(-1) + _active_video_stream(-1), _active_audio_stream(-1), _active_subtitle_stream(-1), + _have_active_video_read(false), _have_active_audio_read(false), _have_active_subtitle_read(false), + _last_audio_data_size(0), _initial_skip(0), _duration(-1) { } @@ -76,6 +77,23 @@ void media_input::get_audio_stream(int stream, int &media_object, int &media_obj } } +void media_input::get_subtitle_stream(int stream, int &media_object, int &media_object_subtitle_stream) const +{ + assert(stream < subtitle_streams()); + + for (size_t i = 0; i < _media_objects.size(); i++) + { + if (_media_objects[i].subtitle_streams() < stream + 1) + { + stream -= _media_objects[i].subtitle_streams(); + continue; + } + media_object = i; + media_object_subtitle_stream = stream; + break; + } +} + // Get the basename of an URL (just the file name, without leading paths) static std::string basename(const std::string &url) { @@ -147,6 +165,16 @@ void media_input::open(const std::vector &urls) + _media_objects[i].audio_blob_template(j).format_info()); } } + for (size_t i = 0; i < _media_objects.size(); i++) + { + std::string pfx = (_media_objects.size() == 1 ? "" : str::from(i + 1) + " - "); + for (int j = 0; j < _media_objects[i].subtitle_streams(); j++) + { + std::string pfx2 = (_media_objects[i].subtitle_streams() == 1 ? "" : str::from(j + 1) + " - "); + _subtitle_stream_names.push_back(pfx + pfx2 + + _media_objects[i].subtitle_box_template(j).format_info()); + } + } // Set duration information _duration = std::numeric_limits::max(); @@ -168,6 +196,14 @@ void media_input::open(const std::vector &urls) _duration = d; } } + for (int j = 0; j < _media_objects[i].subtitle_streams(); j++) + { + int64_t d = _media_objects[i].subtitle_duration(j); + if (d < _duration) + { + _duration = d; + } + } } // Skip advertisement in 3dtv.at movies. Only works for single media objects. @@ -227,6 +263,9 @@ void media_input::open(const std::vector &urls) select_audio_stream(_active_audio_stream); } + // Set active subtitle stream + _active_subtitle_stream = -1; // no subtitles by default + // Print summary msg::inf("Input:"); for (int i = 0; i < video_streams(); i++) @@ -251,6 +290,17 @@ void media_input::open(const std::vector &urls) { msg::inf(" No audio."); } + for (int i = 0; i < subtitle_streams(); i++) + { + int o, s; + get_subtitle_stream(i, o, s); + msg::inf(" Subtitle %s: %s", subtitle_stream_name(i).c_str(), + _media_objects[o].subtitle_box_template(s).format_name().c_str()); + } + if (subtitle_streams() == 0) + { + msg::inf(" No subtitles."); + } msg::inf(" Duration: %g seconds", duration() / 1e6f); if (video_streams() > 0) { @@ -328,6 +378,12 @@ const audio_blob &media_input::audio_blob_template() const return _audio_blob; } +const subtitle_box &media_input::subtitle_box_template() const +{ + assert(_active_subtitle_stream >= 0); + return _subtitle_box; +} + bool media_input::stereo_layout_is_supported(video_frame::stereo_layout_t layout, bool) const { if (video_streams() < 1) @@ -361,6 +417,10 @@ void media_input::set_stereo_layout(video_frame::stereo_layout_t layout, bool sw { (void)finish_audio_blob_read(); } + if (_have_active_subtitle_read) + { + (void)finish_subtitle_box_read(); + } int o, s; get_video_stream(_active_video_stream, o, s); const video_frame &t = _media_objects[o].video_frame_template(s); @@ -393,6 +453,10 @@ void media_input::select_video_stream(int video_stream) { (void)finish_audio_blob_read(); } + if (_have_active_subtitle_read) + { + (void)finish_subtitle_box_read(); + } assert(video_stream >= 0); assert(video_stream < video_streams()); if (_video_frame.stereo_layout == video_frame::separate) @@ -431,6 +495,10 @@ void media_input::select_audio_stream(int audio_stream) { (void)finish_audio_blob_read(); } + if (_have_active_subtitle_read) + { + (void)finish_subtitle_box_read(); + } assert(audio_stream >= 0); assert(audio_stream < audio_streams()); _active_audio_stream = audio_stream; @@ -445,6 +513,34 @@ void media_input::select_audio_stream(int audio_stream) } } +void media_input::select_subtitle_stream(int subtitle_stream) +{ + if (_have_active_video_read) + { + (void)finish_video_frame_read(); + } + if (_have_active_audio_read) + { + (void)finish_audio_blob_read(); + } + if (_have_active_subtitle_read) + { + (void)finish_subtitle_box_read(); + } + assert(subtitle_stream >= 0); + assert(subtitle_stream < subtitle_streams()); + _active_subtitle_stream = subtitle_stream; + int o, s; + get_subtitle_stream(_active_subtitle_stream, o, s); + for (size_t i = 0; i < _media_objects.size(); i++) + { + for (int j = 0; j < _media_objects[i].subtitle_streams(); j++) + { + _media_objects[i].subtitle_stream_set_active(j, (i == static_cast(o) && j == s)); + } + } +} + void media_input::start_video_frame_read() { assert(_active_video_stream >= 0); @@ -544,6 +640,32 @@ audio_blob media_input::finish_audio_blob_read() return _media_objects[o].finish_audio_blob_read(s); } +void media_input::start_subtitle_box_read() +{ + assert(_active_subtitle_stream >= 0); + if (_have_active_subtitle_read) + { + return; + } + int o, s; + get_subtitle_stream(_active_subtitle_stream, o, s); + _media_objects[o].start_subtitle_box_read(s); + _have_active_subtitle_read = true; +} + +subtitle_box media_input::finish_subtitle_box_read() +{ + assert(_active_subtitle_stream >= 0); + int o, s; + get_subtitle_stream(_active_subtitle_stream, o, s); + if (!_have_active_subtitle_read) + { + start_subtitle_box_read(); + } + _have_active_subtitle_read = false; + return _media_objects[o].finish_subtitle_box_read(s); +} + int64_t media_input::tell() { int64_t pos = std::numeric_limits::min(); @@ -571,6 +693,10 @@ void media_input::seek(int64_t pos) { (void)finish_audio_blob_read(); } + if (_have_active_subtitle_read) + { + (void)finish_subtitle_box_read(); + } for (size_t i = 0; i < _media_objects.size(); i++) { _media_objects[i].seek(pos); @@ -589,6 +715,10 @@ void media_input::close() { (void)finish_audio_blob_read(); } + if (_have_active_subtitle_read) + { + (void)finish_subtitle_box_read(); + } for (size_t i = 0; i < _media_objects.size(); i++) { _media_objects[i].close(); @@ -603,10 +733,13 @@ void media_input::close() _tag_values.clear(); _video_stream_names.clear(); _audio_stream_names.clear(); + _subtitle_stream_names.clear(); _active_video_stream = -1; _active_audio_stream = -1; + _active_subtitle_stream = -1; _initial_skip = 0; _duration = -1; _video_frame = video_frame(); _audio_blob = audio_blob(); + _subtitle_box = subtitle_box(); } diff --git a/src/media_input.h b/src/media_input.h index 1857088..968a800 100644 --- a/src/media_input.h +++ b/src/media_input.h @@ -3,6 +3,7 @@ * * Copyright (C) 2010-2011 * Martin Lambers + * Joe * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -37,12 +38,15 @@ private: std::vector _video_stream_names; // Descriptions of available video streams std::vector _audio_stream_names; // Descriptions of available audio streams + std::vector _subtitle_stream_names; // Descriptions of available subtitle streams bool _supports_stereo_layout_separate; // Does this input support the stereo layout 'separate_streams'? int _active_video_stream; // The video stream that is currently active. int _active_audio_stream; // The audio stream that is currently active. + int _active_subtitle_stream; // The subtitle stream that is currently active. bool _have_active_video_read; // Whether a video frame read was started. bool _have_active_audio_read; // Whether a audio blob read was started. + bool _have_active_subtitle_read; // Whether a subtitle box read was started. size_t _last_audio_data_size; // Size of last audio blob read int64_t _initial_skip; // Initial portion of input to skip, in microseconds. @@ -50,10 +54,12 @@ private: video_frame _video_frame; // Video frame template for currently active video stream. audio_blob _audio_blob; // Audio blob template for currently active audio stream. + subtitle_box _subtitle_box; // Subtitle box template for currently active subtitle stream. // Find the media object and its stream index for a given video or audio stream number. void get_video_stream(int stream, int &media_object, int &media_object_video_stream) const; void get_audio_stream(int stream, int &media_object, int &media_object_audio_stream) const; + void get_subtitle_stream(int stream, int &media_object, int &media_object_subtitle_stream) const; public: @@ -89,6 +95,12 @@ public: return _audio_stream_names.size(); } + // Number of subtitle streams in this input. + int subtitle_streams() const + { + return _subtitle_stream_names.size(); + } + // Name of the given video stream. const std::string &video_stream_name(int video_stream) const { @@ -101,6 +113,12 @@ public: return _audio_stream_names[audio_stream]; } + // Name of the given subtitle stream. + const std::string &subtitle_stream_name(int subtitle_stream) const + { + return _subtitle_stream_names[subtitle_stream]; + } + // Initial portion of the input to skip. int64_t initial_skip() const { @@ -126,11 +144,15 @@ public: // that contains all properties but no actual data. const audio_blob &audio_blob_template() const; + // Information about the active subtitle stream, in the form of a subtitle box + // that contains all properties but no actual data. + const subtitle_box &subtitle_box_template() const; + /* - * Access video and audio data + * Access media data */ - /* Set the active video and audio streams. */ + /* Set the active media streams. */ int selected_video_stream() const { return _active_video_stream; @@ -141,6 +163,11 @@ public: return _active_audio_stream; } void select_audio_stream(int audio_stream); + int selected_subtitle_stream() const + { + return _active_subtitle_stream; + } + void select_subtitle_stream(int subtitle_stream); /* Check whether a stereo layout is supported by this input. */ bool stereo_layout_is_supported(video_frame::stereo_layout_t layout, bool swap) const; @@ -161,6 +188,13 @@ public: * An invalid blob means that EOF was reached. */ audio_blob finish_audio_blob_read(); + /* Start to read a subtitle box from the active stream asynchronously + * (in a separate thread). */ + void start_subtitle_box_read(); + /* Wait for the subtitle data reading to finish, and return the box. + * An invalid box means that EOF was reached. */ + subtitle_box finish_subtitle_box_read(); + /* Return the last position in microseconds, of the last packet that was read in an * active stream. If the position is unkown, the minimum possible value is returned. */ int64_t tell(); diff --git a/src/media_object.cpp b/src/media_object.cpp index 1c45417..eb8feb3 100644 --- a/src/media_object.cpp +++ b/src/media_object.cpp @@ -4,6 +4,7 @@ * Copyright (C) 2010-2011 * Martin Lambers * Frédéric Devernay + * Joe * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -113,6 +114,27 @@ public: } }; +// The subtitle decode thread. +// This thread reads packets from its packet queue and decodes them to subtitle boxes. +class subtitle_decode_thread : public thread +{ +private: + std::string _url; + struct ffmpeg_stuff *_ffmpeg; + int _subtitle_stream; + subtitle_box _box; + + int64_t handle_timestamp(int64_t timestamp); + +public: + subtitle_decode_thread(const std::string &url, struct ffmpeg_stuff *ffmpeg, int subtitle_stream); + void run(); + const subtitle_box &box() + { + return _box; + } +}; + // Hide the FFmpeg stuff so that their messy header files cannot cause problems // in other source files. @@ -153,6 +175,16 @@ struct ffmpeg_stuff std::vector audio_blobs; std::vector > audio_buffers; std::vector audio_last_timestamps; + + std::vector subtitle_streams; + std::vector subtitle_codec_ctxs; + std::vector subtitle_box_templates; + std::vector subtitle_codecs; + std::vector > subtitle_packet_queues; + std::vector subtitle_packet_queue_mutexes; + std::vector subtitle_decode_threads; + std::vector > subtitle_box_buffers; + std::vector subtitle_last_timestamps; }; // Use one decoding thread per processor for video decoding. @@ -259,6 +291,25 @@ static int64_t timestamp_helper(int64_t &last_timestamp, int64_t timestamp) return timestamp; } +// Get a stream duration +static int64_t stream_duration(AVStream *stream, AVFormatContext *format) +{ + // Try to get duration from the stream first. If that fails, fall back to + // the value provided by the container. + int64_t duration = stream->duration; + if (duration > 0) + { + AVRational time_base = stream->time_base; + return duration * 1000000 * time_base.num / time_base.den; + } + else + { + duration = format->duration; + return duration * 1000000 / AV_TIME_BASE; + } +} + + media_object::media_object() : _ffmpeg(NULL) { av_register_all(); @@ -554,6 +605,20 @@ void media_object::set_audio_blob_template(int index) } } +void media_object::set_subtitle_box_template(int index) +{ + AVStream *subtitle_stream = _ffmpeg->format_ctx->streams[_ffmpeg->subtitle_streams[index]]; + //AVCodecContext *subtitle_codec_ctx = _ffmpeg->subtitle_codec_ctxs[index]; + subtitle_box &subtitle_box_template = _ffmpeg->subtitle_box_templates[index]; + + AVMetadataTag *tag = av_metadata_get(subtitle_stream->metadata, "language", NULL, AV_METADATA_IGNORE_SUFFIX); + if (tag) + { + subtitle_box_template.language = tag->value; + } + subtitle_box_template.format = subtitle_box::text; +} + void media_object::open(const std::string &url) { assert(!_ffmpeg); @@ -685,15 +750,39 @@ void media_object::open(const std::string &url) _ffmpeg->audio_buffers.push_back(std::vector()); _ffmpeg->audio_last_timestamps.push_back(std::numeric_limits::min()); } + else if (_ffmpeg->format_ctx->streams[i]->codec->codec_type == CODEC_TYPE_SUBTITLE) + { + _ffmpeg->subtitle_streams.push_back(i); + int j = _ffmpeg->subtitle_streams.size() - 1; + msg::dbg(_url + " stream " + str::from(i) + " is subtitle stream " + str::from(j) + "."); + _ffmpeg->subtitle_codec_ctxs.push_back(_ffmpeg->format_ctx->streams[i]->codec); + _ffmpeg->subtitle_codecs.push_back(avcodec_find_decoder(_ffmpeg->subtitle_codec_ctxs[j]->codec_id)); + if (!_ffmpeg->subtitle_codecs[j]) + { + throw exc(_url + " stream " + str::from(i) + ": Unsupported subtitle codec."); + } + if ((e = avcodec_open(_ffmpeg->subtitle_codec_ctxs[j], _ffmpeg->subtitle_codecs[j])) < 0) + { + _ffmpeg->subtitle_codecs[j] = NULL; + throw exc(_url + " stream " + str::from(i) + ": Cannot open subtitle codec: " + my_av_strerror(e)); + } + _ffmpeg->subtitle_box_templates.push_back(subtitle_box()); + set_subtitle_box_template(j); + _ffmpeg->subtitle_decode_threads.push_back(subtitle_decode_thread(_url, _ffmpeg, j)); + _ffmpeg->subtitle_box_buffers.push_back(std::deque()); + _ffmpeg->subtitle_last_timestamps.push_back(std::numeric_limits::min()); + } else { - msg::dbg(_url + " stream " + str::from(i) + " contains neither video nor audio."); + msg::dbg(_url + " stream " + str::from(i) + " contains neither video nor audio nor subtitles."); } } _ffmpeg->video_packet_queues.resize(video_streams()); _ffmpeg->audio_packet_queues.resize(audio_streams()); + _ffmpeg->subtitle_packet_queues.resize(subtitle_streams()); _ffmpeg->video_packet_queue_mutexes.resize(video_streams()); _ffmpeg->audio_packet_queue_mutexes.resize(audio_streams()); + _ffmpeg->subtitle_packet_queue_mutexes.resize(subtitle_streams()); msg::inf(_url + ":"); for (int i = 0; i < video_streams(); i++) @@ -712,7 +801,14 @@ void media_object::open(const std::string &url) audio_blob_template(i).format_name().c_str(), audio_duration(i) / 1e6f); } - if (video_streams() == 0 && audio_streams() == 0) + for (int i = 0; i < subtitle_streams(); i++) + { + msg::inf(" Subtitle stream %d: %s / %s, %g seconds", i, + subtitle_box_template(i).format_info().c_str(), + subtitle_box_template(i).format_name().c_str(), + subtitle_duration(i) / 1e6f); + } + if (video_streams() == 0 && audio_streams() == 0 && subtitle_streams() == 0) { msg::inf(" No usable streams."); } @@ -753,14 +849,19 @@ const std::string &media_object::tag_value(const std::string &tag_name) const return empty; } +int media_object::video_streams() const +{ + return _ffmpeg->video_streams.size(); +} + int media_object::audio_streams() const { return _ffmpeg->audio_streams.size(); } -int media_object::video_streams() const +int media_object::subtitle_streams() const { - return _ffmpeg->video_streams.size(); + return _ffmpeg->subtitle_streams.size(); } void media_object::video_stream_set_active(int index, bool active) @@ -776,6 +877,10 @@ void media_object::video_stream_set_active(int index, bool active) { _ffmpeg->audio_decode_threads[i].finish(); } + for (size_t i = 0; i < _ffmpeg->subtitle_streams.size(); i++) + { + _ffmpeg->subtitle_decode_threads[i].finish(); + } // Stop reading packets _ffmpeg->reader->finish(); // Set status @@ -798,6 +903,10 @@ void media_object::audio_stream_set_active(int index, bool active) { _ffmpeg->audio_decode_threads[i].finish(); } + for (size_t i = 0; i < _ffmpeg->subtitle_streams.size(); i++) + { + _ffmpeg->subtitle_decode_threads[i].finish(); + } // Stop reading packets _ffmpeg->reader->finish(); // Set status @@ -816,6 +925,32 @@ void media_object::audio_stream_set_active(int index, bool active) _ffmpeg->reader->start(); } +void media_object::subtitle_stream_set_active(int index, bool active) +{ + assert(index >= 0); + assert(index < audio_streams()); + // Stop decoder threads + for (size_t i = 0; i < _ffmpeg->video_streams.size(); i++) + { + _ffmpeg->video_decode_threads[i].finish(); + } + for (size_t i = 0; i < _ffmpeg->audio_streams.size(); i++) + { + _ffmpeg->audio_decode_threads[i].finish(); + } + for (size_t i = 0; i < _ffmpeg->subtitle_streams.size(); i++) + { + _ffmpeg->subtitle_decode_threads[i].finish(); + } + // Stop reading packets + _ffmpeg->reader->finish(); + // Set status + _ffmpeg->format_ctx->streams[_ffmpeg->subtitle_streams.at(index)]->discard = + (active ? AVDISCARD_DEFAULT : AVDISCARD_ALL); + // Restart reader + _ffmpeg->reader->start(); +} + const video_frame &media_object::video_frame_template(int video_stream) const { assert(video_stream >= 0); @@ -841,19 +976,9 @@ int64_t media_object::video_duration(int index) const { assert(index >= 0); assert(index < video_streams()); - // Try to get duration from the Codec first. If that fails, fall back to - // the value provided by the container. - int64_t duration = _ffmpeg->format_ctx->streams[_ffmpeg->video_streams.at(index)]->duration; - if (duration > 0) - { - AVRational time_base = _ffmpeg->format_ctx->streams[_ffmpeg->video_streams.at(index)]->time_base; - return duration * 1000000 * time_base.num / time_base.den; - } - else - { - duration = _ffmpeg->format_ctx->duration; - return duration * 1000000 / AV_TIME_BASE; - } + return stream_duration( + _ffmpeg->format_ctx->streams[_ffmpeg->video_streams.at(index)], + _ffmpeg->format_ctx); } const audio_blob &media_object::audio_blob_template(int audio_stream) const @@ -867,19 +992,25 @@ int64_t media_object::audio_duration(int index) const { assert(index >= 0); assert(index < audio_streams()); - // Try to get duration from the Codec first. If that fails, fall back to - // the value provided by the container. - int64_t duration = _ffmpeg->format_ctx->streams[_ffmpeg->audio_streams.at(index)]->duration; - if (duration > 0) - { - AVRational time_base = _ffmpeg->format_ctx->streams[_ffmpeg->audio_streams.at(index)]->time_base; - return duration * 1000000 * time_base.num / time_base.den; - } - else - { - duration = _ffmpeg->format_ctx->duration; - return duration * 1000000 / AV_TIME_BASE; - } + return stream_duration( + _ffmpeg->format_ctx->streams[_ffmpeg->audio_streams.at(index)], + _ffmpeg->format_ctx); +} + +const subtitle_box &media_object::subtitle_box_template(int subtitle_stream) const +{ + assert(subtitle_stream >= 0); + assert(subtitle_stream < subtitle_streams()); + return _ffmpeg->subtitle_box_templates.at(subtitle_stream); +} + +int64_t media_object::subtitle_duration(int index) const +{ + assert(index >= 0); + assert(index < subtitle_streams()); + return stream_duration( + _ffmpeg->format_ctx->streams[_ffmpeg->subtitle_streams.at(index)], + _ffmpeg->format_ctx); } read_thread::read_thread(const std::string &url, struct ffmpeg_stuff *ffmpeg) : @@ -894,6 +1025,7 @@ void read_thread::run() // We need another packet if the number of queued packets for an active stream is below a threshold. const size_t video_stream_low_threshold = 2; // Often, 1 packet results in one video frame const size_t audio_stream_low_threshold = 5; // Often, 3-4 packets are needed for one buffer fill + const size_t subtitle_stream_low_threshold = 1; // Just a guess bool need_another_packet = false; for (size_t i = 0; !need_another_packet && i < _ffmpeg->video_streams.size(); i++) { @@ -913,6 +1045,15 @@ void read_thread::run() _ffmpeg->audio_packet_queue_mutexes[i].unlock(); } } + for (size_t i = 0; !need_another_packet && i < _ffmpeg->subtitle_streams.size(); i++) + { + if (_ffmpeg->format_ctx->streams[_ffmpeg->subtitle_streams[i]]->discard == AVDISCARD_DEFAULT) + { + _ffmpeg->subtitle_packet_queue_mutexes[i].lock(); + need_another_packet = _ffmpeg->subtitle_packet_queues[i].size() < subtitle_stream_low_threshold; + _ffmpeg->subtitle_packet_queue_mutexes[i].unlock(); + } + } if (!need_another_packet) { msg::dbg(_url + ": No need to read more packets."); @@ -989,6 +1130,36 @@ void read_thread::run() _ffmpeg->audio_packet_queue_mutexes[i].unlock(); } } + for (size_t i = 0; i < _ffmpeg->subtitle_streams.size() && !packet_queued; i++) + { + if (packet.stream_index == _ffmpeg->subtitle_streams[i]) + { + _ffmpeg->subtitle_packet_queue_mutexes[i].lock(); + if (_ffmpeg->subtitle_packet_queues[i].empty() + && _ffmpeg->subtitle_last_timestamps[i] == std::numeric_limits::min() + && packet.dts == static_cast(AV_NOPTS_VALUE)) + { + // We have no packet in the queue and no last timestamp, probably + // because we just seeked. We want a packet with a timestamp. + msg::dbg(_url + ": subtitle stream " + str::from(i) + + ": dropping packet because it has no timestamp"); + } + else + { + if (av_dup_packet(&packet) < 0) + { + _ffmpeg->subtitle_packet_queue_mutexes[i].unlock(); + throw exc(_url + ": Cannot duplicate packet."); + } + _ffmpeg->subtitle_packet_queues[i].push_back(packet); + packet_queued = true; + msg::dbg(_url + ": " + + str::from(_ffmpeg->subtitle_packet_queues[i].size()) + + " packets queued in subtitle stream " + str::from(i) + "."); + } + _ffmpeg->subtitle_packet_queue_mutexes[i].unlock(); + } + } if (!packet_queued) { av_free_packet(&packet); @@ -1237,6 +1408,139 @@ audio_blob media_object::finish_audio_blob_read(int audio_stream) return _ffmpeg->audio_decode_threads[audio_stream].blob(); } +subtitle_decode_thread::subtitle_decode_thread(const std::string &url, struct ffmpeg_stuff *ffmpeg, int subtitle_stream) : + _url(url), _ffmpeg(ffmpeg), _subtitle_stream(subtitle_stream), _box() +{ +} + +int64_t subtitle_decode_thread::handle_timestamp(int64_t timestamp) +{ + int64_t ts = timestamp_helper(_ffmpeg->subtitle_last_timestamps[_subtitle_stream], timestamp); + _ffmpeg->pos = ts; + return ts; +} + +void subtitle_decode_thread::run() +{ + if (_ffmpeg->subtitle_box_buffers[_subtitle_stream].empty()) + { + // Read more subtitle data + AVPacket packet, tmppacket; + bool empty; + do + { + _ffmpeg->subtitle_packet_queue_mutexes[_subtitle_stream].lock(); + empty = _ffmpeg->subtitle_packet_queues[_subtitle_stream].empty(); + _ffmpeg->subtitle_packet_queue_mutexes[_subtitle_stream].unlock(); + if (empty) + { + if (_ffmpeg->reader->eof()) + { + _box = subtitle_box(); + return; + } + msg::dbg(_url + ": subtitle stream " + str::from(_subtitle_stream) + ": need to wait for packets..."); + _ffmpeg->reader->start(); + _ffmpeg->reader->finish(); + } + } + while (empty); + _ffmpeg->subtitle_packet_queue_mutexes[_subtitle_stream].lock(); + packet = _ffmpeg->subtitle_packet_queues[_subtitle_stream].front(); + _ffmpeg->subtitle_packet_queues[_subtitle_stream].pop_front(); + _ffmpeg->subtitle_packet_queue_mutexes[_subtitle_stream].unlock(); + _ffmpeg->reader->start(); // Refill the packet queue + + // Decode subtitle data + AVSubtitle subtitle; + int got_subtitle; + tmppacket = packet; + while (tmppacket.size > 0) + { + int len = avcodec_decode_subtitle2(_ffmpeg->subtitle_codec_ctxs[_subtitle_stream], + &subtitle, &got_subtitle, &tmppacket); + if (len < 0) + { + tmppacket.size = 0; + break; + } + tmppacket.data += len; + tmppacket.size -= len; + if (!got_subtitle) + { + continue; + } + // Put it in the subtitle buffer + subtitle_box box = _ffmpeg->subtitle_box_templates[_subtitle_stream]; + int64_t timestamp = subtitle.pts * 1000000 / AV_TIME_BASE + subtitle.start_display_time * 1000; + int64_t duration = (subtitle.end_display_time - subtitle.start_display_time) * 1000; + box.presentation_start_time = timestamp; + box.presentation_stop_time = timestamp + duration; + for (unsigned int i = 0; i < subtitle.num_rects; i++) + { + AVSubtitleRect *rect = subtitle.rects[i]; + switch (rect->type) + { + case SUBTITLE_BITMAP: + // TODO + msg::wrn(_url + ": subtitle stream " + str::from(_subtitle_stream) + + ": bitmap subtitles are not yet supported"); + box.str = "???"; + break; + case SUBTITLE_TEXT: + box.str += rect->text; + box.str += '\n'; + break; + case SUBTITLE_ASS: + // TODO: full support for ASS/SSA + if (rect->text) + { + box.str += rect->text; + box.str += '\n'; + } + else + { + msg::wrn(_url + ": subtitle stream " + str::from(_subtitle_stream) + + ": not text approximation for ASS subtitle found"); + box.str = "???"; + } + break; + case SUBTITLE_NONE: + // Should never happen, but make sure we have a valid subtitle box anyway. + box.str = ' '; + break; + } + } + _ffmpeg->subtitle_box_buffers[_subtitle_stream].push_back(box); + avsubtitle_free(&subtitle); + } + + av_free_packet(&packet); + } + if (_ffmpeg->subtitle_box_buffers[_subtitle_stream].empty()) + { + _box = subtitle_box(); + return; + } + _box = _ffmpeg->subtitle_box_buffers[_subtitle_stream].front(); + _ffmpeg->subtitle_box_buffers[_subtitle_stream].pop_front(); +} + +void media_object::start_subtitle_box_read(int subtitle_stream) +{ + assert(subtitle_stream >= 0); + assert(subtitle_stream < subtitle_streams()); + _ffmpeg->subtitle_decode_threads[subtitle_stream].start(); +} + +subtitle_box media_object::finish_subtitle_box_read(int subtitle_stream) +{ + assert(subtitle_stream >= 0); + assert(subtitle_stream < subtitle_streams()); + _ffmpeg->subtitle_decode_threads[subtitle_stream].finish(); + return _ffmpeg->subtitle_decode_threads[subtitle_stream].box(); +} + int64_t media_object::tell() { return _ffmpeg->pos; @@ -1255,6 +1559,10 @@ void media_object::seek(int64_t dest_pos) { _ffmpeg->audio_decode_threads[i].finish(); } + for (size_t i = 0; i < _ffmpeg->subtitle_streams.size(); i++) + { + _ffmpeg->subtitle_decode_threads[i].finish(); + } // Stop reading packets _ffmpeg->reader->finish(); // Seek @@ -1285,6 +1593,16 @@ void media_object::seek(int64_t dest_pos) } _ffmpeg->audio_packet_queues[i].clear(); } + for (size_t i = 0; i < _ffmpeg->subtitle_streams.size(); i++) + { + avcodec_flush_buffers(_ffmpeg->format_ctx->streams[_ffmpeg->subtitle_streams[i]]->codec); + _ffmpeg->subtitle_box_buffers[i].clear(); + for (size_t j = 0; j < _ffmpeg->subtitle_packet_queues[i].size(); j++) + { + av_free_packet(&_ffmpeg->subtitle_packet_queues[i][j]); + } + _ffmpeg->subtitle_packet_queues[i].clear(); + } // The next read request must update the position for (size_t i = 0; i < _ffmpeg->video_streams.size(); i++) { @@ -1294,6 +1612,10 @@ void media_object::seek(int64_t dest_pos) { _ffmpeg->audio_last_timestamps[i] = std::numeric_limits::min(); } + for (size_t i = 0; i < _ffmpeg->subtitle_streams.size(); i++) + { + _ffmpeg->subtitle_last_timestamps[i] = std::numeric_limits::min(); + } _ffmpeg->pos = std::numeric_limits::min(); // Restart packet reading _ffmpeg->reader->reset(); @@ -1313,6 +1635,10 @@ void media_object::close() { _ffmpeg->audio_decode_threads[i].finish(); } + for (size_t i = 0; i < _ffmpeg->subtitle_streams.size(); i++) + { + _ffmpeg->subtitle_decode_threads[i].finish(); + } // Stop reading packets _ffmpeg->reader->finish(); } @@ -1347,7 +1673,7 @@ void media_object::close() if (_ffmpeg->video_packet_queues[i].size() > 0) { msg::dbg(_url + ": " + str::from(_ffmpeg->video_packet_queues[i].size()) - + " unprocessed video packets in video stream " + str::from(i)); + + " unprocessed packets in video stream " + str::from(i)); } for (size_t j = 0; j < _ffmpeg->video_packet_queues[i].size(); j++) { @@ -1370,7 +1696,7 @@ void media_object::close() if (_ffmpeg->audio_packet_queues[i].size() > 0) { msg::dbg(_url + ": " + str::from(_ffmpeg->audio_packet_queues[i].size()) - + " unprocessed audio packets in audio stream " + str::from(i)); + + " unprocessed packets in audio stream " + str::from(i)); } for (size_t j = 0; j < _ffmpeg->audio_packet_queues[i].size(); j++) { @@ -1381,6 +1707,25 @@ void media_object::close() { av_free(_ffmpeg->audio_tmpbufs[i]); } + for (size_t i = 0; i < _ffmpeg->subtitle_codec_ctxs.size(); i++) + { + if (i < _ffmpeg->subtitle_codecs.size() && _ffmpeg->subtitle_codecs[i]) + { + avcodec_close(_ffmpeg->subtitle_codec_ctxs[i]); + } + } + for (size_t i = 0; i < _ffmpeg->subtitle_packet_queues.size(); i++) + { + if (_ffmpeg->subtitle_packet_queues[i].size() > 0) + { + msg::dbg(_url + ": " + str::from(_ffmpeg->subtitle_packet_queues[i].size()) + + " unprocessed packets in subtitle stream " + str::from(i)); + } + for (size_t j = 0; j < _ffmpeg->subtitle_packet_queues[i].size(); j++) + { + av_free_packet(&_ffmpeg->subtitle_packet_queues[i][j]); + } + } if (_ffmpeg->format_ctx) { av_close_input_file(_ffmpeg->format_ctx); diff --git a/src/media_object.h b/src/media_object.h index 0c72343..f7802fb 100644 --- a/src/media_object.h +++ b/src/media_object.h @@ -3,6 +3,7 @@ * * Copyright (C) 2010-2011 * Martin Lambers + * Joe * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -39,11 +40,13 @@ private: // from the given streams void set_video_frame_template(int video_stream); void set_audio_blob_template(int audio_stream); + void set_subtitle_box_template(int subtitle_stream); // The threaded implementation can access private members friend class read_thread; friend class video_decode_thread; friend class audio_decode_thread; + friend class subtitle_decode_thread; public: @@ -65,13 +68,15 @@ public: const std::string &tag_value(size_t i) const; const std::string &tag_value(const std::string &tag_name) const; - /* Get the number of video and audio streams in the file. */ + /* Get the number of media streams in the file. */ int video_streams() const; int audio_streams() const; + int subtitle_streams() const; - /* Activate a video or audio stream for usage. Inactive streams will not be accessible. */ + /* Activate a media stream for usage. Inactive streams will not be accessible. */ void video_stream_set_active(int video_stream, bool active); void audio_stream_set_active(int audio_stream, bool active); + void subtitle_stream_set_active(int subtitle_stream, bool active); /* Get information about video streams. */ // Return a video frame with all properties filled in (but without any data). @@ -89,10 +94,17 @@ public: // Note that this is only a hint; the properties of actual audio blobs may differ! const audio_blob &audio_blob_template(int audio_stream) const; // Audio stream duration in microseconds. - int64_t audio_duration(int video_stream) const; + int64_t audio_duration(int audio_stream) const; + + /* Get information about subtitle streams. */ + // Return a subtitle box with all properties filled in (but without any data). + // Note that this is only a hint; the properties of actual subtitle boxes may differ! + const subtitle_box &subtitle_box_template(int subtitle_stream) const; + // Subtitle stream duration in microseconds. + int64_t subtitle_duration(int subtitle_stream) const; /* - * Access video and audio data + * Access media data */ /* Start to read a video frame asynchronously (in a separate thread). */ @@ -107,16 +119,22 @@ public: * An invalid blob means that EOF was reached. */ audio_blob finish_audio_blob_read(int audio_stream); + /* Start to read a subtitle box asynchronously (in a separate thread). */ + void start_subtitle_box_read(int subtitle_stream); + /* Wait for the subtitle box reading to finish, and return the box. + * An invalid box means that EOF was reached. */ + subtitle_box finish_subtitle_box_read(int subtitle_stream); + /* Return the last position in microseconds, of the last packet that was read in any * stream. If the position is unkown, the minimum possible value is returned. */ int64_t tell(); /* Seek to the given position in microseconds. This affects all streams. * Make sure that the position is not out of range! - * The real position after seeking is only revealed after reading the next video frame - * or audio blob. This position may differ from the requested position for various - * reasons (seeking is only possible to keyframes, seeking is not supported by the - * stream, ...) */ + * The real position after seeking is only revealed after reading the next video frame, + * audio blob, or subtitle box. This position may differ from the requested position + * for various reasons (seeking is only possible to keyframes, seeking is not supported + * by the stream, ...) */ void seek(int64_t pos); /*