diff --git a/Libraries/LibMedia/CMakeLists.txt b/Libraries/LibMedia/CMakeLists.txt index f10a4c401b3..195ee66f61b 100644 --- a/Libraries/LibMedia/CMakeLists.txt +++ b/Libraries/LibMedia/CMakeLists.txt @@ -22,13 +22,17 @@ target_link_libraries(LibMedia PRIVATE LibCore LibCrypto LibRIFF LibIPC LibGfx L if (NOT ANDROID) target_sources(LibMedia PRIVATE Audio/FFmpegLoader.cpp + FFmpeg/FFmpegDemuxer.cpp FFmpeg/FFmpegIOContext.cpp FFmpeg/FFmpegVideoDecoder.cpp ) target_link_libraries(LibMedia PRIVATE PkgConfig::AVCODEC PkgConfig::AVFORMAT PkgConfig::AVUTIL) else() # FIXME: Need to figure out how to build or replace ffmpeg libs on Android and Windows - target_sources(LibMedia PRIVATE FFmpeg/FFmpegVideoDecoderStub.cpp) + target_sources(LibMedia PRIVATE + FFmpeg/FFmpegDemuxerStub.cpp + FFmpeg/FFmpegVideoDecoderStub.cpp + ) endif() if (LADYBIRD_AUDIO_BACKEND STREQUAL "PULSE") diff --git a/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp b/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp index 907272c344b..ffa07cca9ff 100644 --- a/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp +++ b/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp @@ -49,7 +49,7 @@ DecoderErrorOr> MatroskaDemuxer::get_tracks_for_type(TrackType typ switch (type) { case TrackType::Video: if (auto video_track = track_entry.video_track(); video_track.has_value()) - track.set_video_data({ TRY(duration()), video_track->pixel_width, video_track->pixel_height }); + track.set_video_data({ TRY(duration(track)), video_track->pixel_width, video_track->pixel_height }); break; default: break; @@ -148,10 +148,11 @@ DecoderErrorOr MatroskaDemuxer::get_next_sample_for_track(Track track) status.frame_index = 0; } auto cicp = TRY(m_reader.track_for_track_number(track.identifier()))->video_track()->color_format.to_cicp(); - return Sample(status.block->timestamp(), status.block->frame(status.frame_index++), VideoSampleData(cicp)); + auto sample_data = DECODER_TRY_ALLOC(ByteBuffer::copy(status.block->frame(status.frame_index++))); + return Sample(status.block->timestamp(), move(sample_data), VideoSampleData(cicp)); } -DecoderErrorOr MatroskaDemuxer::duration() +DecoderErrorOr MatroskaDemuxer::duration(Track) { auto duration = TRY(m_reader.segment_information()).duration(); return duration.value_or(AK::Duration::zero()); diff --git a/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h b/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h index 9502d7d41ab..7d4a2349f06 100644 --- a/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h +++ b/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h @@ -31,7 +31,7 @@ public: DecoderErrorOr> seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional earliest_available_sample = OptionalNone()) override; - DecoderErrorOr duration() override; + DecoderErrorOr duration(Track track) override; DecoderErrorOr get_codec_id_for_track(Track track) override; diff --git a/Libraries/LibMedia/Demuxer.h b/Libraries/LibMedia/Demuxer.h index f3632799e11..c62e3c1182c 100644 --- a/Libraries/LibMedia/Demuxer.h +++ b/Libraries/LibMedia/Demuxer.h @@ -33,7 +33,7 @@ public: // in the case that the timestamp is closer to the current time than the nearest keyframe. virtual DecoderErrorOr> seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional earliest_available_sample = OptionalNone()) = 0; - virtual DecoderErrorOr duration() = 0; + virtual DecoderErrorOr duration(Track track) = 0; }; } diff --git a/Libraries/LibMedia/FFmpeg/FFmpegDemuxer.cpp b/Libraries/LibMedia/FFmpeg/FFmpegDemuxer.cpp new file mode 100644 index 00000000000..34d0327b2bd --- /dev/null +++ b/Libraries/LibMedia/FFmpeg/FFmpegDemuxer.cpp @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2025, Luke Wilde + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include + +namespace Media::FFmpeg { + +FFmpegDemuxer::FFmpegDemuxer(NonnullOwnPtr stream, NonnullOwnPtr io_context) + : m_stream(move(stream)) + , m_io_context(move(io_context)) +{ +} + +FFmpegDemuxer::~FFmpegDemuxer() +{ + if (m_packet != nullptr) + av_packet_free(&m_packet); + if (m_codec_context != nullptr) + avcodec_free_context(&m_codec_context); + if (m_format_context != nullptr) + avformat_close_input(&m_format_context); +} + +ErrorOr> FFmpegDemuxer::create(NonnullOwnPtr stream) +{ + auto io_context = TRY(Media::FFmpeg::FFmpegIOContext::create(*stream)); + auto demuxer = make(move(stream), move(io_context)); + + // Open the container + demuxer->m_format_context = avformat_alloc_context(); + if (demuxer->m_format_context == nullptr) + return Error::from_string_literal("Failed to allocate format context"); + demuxer->m_format_context->pb = demuxer->m_io_context->avio_context(); + if (avformat_open_input(&demuxer->m_format_context, nullptr, nullptr, nullptr) < 0) + return Error::from_string_literal("Failed to open input for format parsing"); + + // Read stream info; doing this is required for headerless formats like MPEG + if (avformat_find_stream_info(demuxer->m_format_context, nullptr) < 0) + return Error::from_string_literal("Failed to find stream info"); + + demuxer->m_packet = av_packet_alloc(); + if (demuxer->m_packet == nullptr) + return Error::from_string_literal("Failed to allocate packet"); + + return demuxer; +} + +DecoderErrorOr FFmpegDemuxer::duration_of_track_in_milliseconds(Track const& track) +{ + VERIFY(track.identifier() < m_format_context->nb_streams); + auto* stream = m_format_context->streams[track.identifier()]; + + if (stream->duration >= 0) { + auto time_base = av_q2d(stream->time_base); + double duration_in_milliseconds = static_cast(stream->duration) * time_base * 1000.0; + return AK::Duration::from_milliseconds(AK::round_to(duration_in_milliseconds)); + } + + // If the stream doesn't specify the duration, fallback to what the container says the duration is. + // If the container doesn't know the duration, then we're out of luck. Return an error. + if (m_format_context->duration < 0) + return DecoderError::format(DecoderErrorCategory::Unknown, "Negative stream duration"); + + double duration_in_milliseconds = (static_cast(m_format_context->duration) / AV_TIME_BASE) * 1000.0; + return AK::Duration::from_milliseconds(AK::round_to(duration_in_milliseconds)); +} + +DecoderErrorOr> FFmpegDemuxer::get_tracks_for_type(TrackType type) +{ + AVMediaType media_type; + + switch (type) { + case TrackType::Video: + media_type = AVMediaType::AVMEDIA_TYPE_VIDEO; + break; + case TrackType::Audio: + media_type = AVMediaType::AVMEDIA_TYPE_AUDIO; + break; + case TrackType::Subtitles: + media_type = AVMediaType::AVMEDIA_TYPE_SUBTITLE; + break; + } + + // Find the best stream to play within the container + int best_stream_index = av_find_best_stream(m_format_context, media_type, -1, -1, nullptr, 0); + if (best_stream_index == AVERROR_STREAM_NOT_FOUND) + return DecoderError::format(DecoderErrorCategory::Unknown, "No stream for given type found in container"); + if (best_stream_index == AVERROR_DECODER_NOT_FOUND) + return DecoderError::format(DecoderErrorCategory::Unknown, "No suitable decoder found for stream"); + if (best_stream_index < 0) + return DecoderError::format(DecoderErrorCategory::Unknown, "Failed to find a stream for the given type"); + + auto* stream = m_format_context->streams[best_stream_index]; + + Track track(type, best_stream_index); + + if (type == TrackType::Video) { + track.set_video_data({ + .duration = TRY(duration_of_track_in_milliseconds(track)), + .pixel_width = static_cast(stream->codecpar->width), + .pixel_height = static_cast(stream->codecpar->height), + }); + } + + Vector tracks; + tracks.append(move(track)); + return tracks; +} + +DecoderErrorOr> FFmpegDemuxer::seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional earliest_available_sample) +{ + // FIXME: What do we do with this here? + (void)earliest_available_sample; + + VERIFY(track.identifier() < m_format_context->nb_streams); + auto* stream = m_format_context->streams[track.identifier()]; + auto time_base = av_q2d(stream->time_base); + auto time_in_seconds = static_cast(timestamp.to_milliseconds()) / 1000.0 / time_base; + auto sample_timestamp = AK::round_to(time_in_seconds); + + if (av_seek_frame(m_format_context, stream->index, sample_timestamp, AVSEEK_FLAG_BACKWARD) < 0) + return DecoderError::format(DecoderErrorCategory::Unknown, "Failed to seek"); + + return timestamp; +} + +DecoderErrorOr FFmpegDemuxer::duration(Track track) +{ + return duration_of_track_in_milliseconds(track); +} + +DecoderErrorOr FFmpegDemuxer::get_codec_id_for_track(Track track) +{ + VERIFY(track.identifier() < m_format_context->nb_streams); + auto* stream = m_format_context->streams[track.identifier()]; + return media_codec_id_from_ffmpeg_codec_id(stream->codecpar->codec_id); +} + +DecoderErrorOr FFmpegDemuxer::get_codec_initialization_data_for_track(Track track) +{ + VERIFY(track.identifier() < m_format_context->nb_streams); + auto* stream = m_format_context->streams[track.identifier()]; + return ReadonlyBytes { stream->codecpar->extradata, static_cast(stream->codecpar->extradata_size) }; +} + +DecoderErrorOr FFmpegDemuxer::get_next_sample_for_track(Track track) +{ + VERIFY(track.identifier() < m_format_context->nb_streams); + auto* stream = m_format_context->streams[track.identifier()]; + + for (;;) { + auto read_frame_error = av_read_frame(m_format_context, m_packet); + if (read_frame_error < 0) { + if (read_frame_error == AVERROR_EOF) + return DecoderError::format(DecoderErrorCategory::EndOfStream, "End of stream"); + + return DecoderError::format(DecoderErrorCategory::Unknown, "Failed to read frame"); + } + if (m_packet->stream_index != stream->index) { + av_packet_unref(m_packet); + continue; + } + + auto color_primaries = static_cast(stream->codecpar->color_primaries); + auto transfer_characteristics = static_cast(stream->codecpar->color_trc); + auto matrix_coefficients = static_cast(stream->codecpar->color_space); + auto color_range = [stream] { + switch (stream->codecpar->color_range) { + case AVColorRange::AVCOL_RANGE_MPEG: + return VideoFullRangeFlag::Studio; + case AVColorRange::AVCOL_RANGE_JPEG: + return VideoFullRangeFlag::Full; + default: + return VideoFullRangeFlag::Unspecified; + } + }(); + + auto time_base = av_q2d(stream->time_base); + double timestamp_in_milliseconds = static_cast(m_packet->pts) * time_base * 1000.0; + + // Copy the packet data so that we have a permanent reference to it whilst the Sample is alive, which allows us + // to wipe the packet afterwards. + auto packet_data = DECODER_TRY_ALLOC(ByteBuffer::copy(m_packet->data, m_packet->size)); + + auto sample = Sample( + AK::Duration::from_milliseconds(AK::round_to(timestamp_in_milliseconds)), + move(packet_data), + VideoSampleData(CodingIndependentCodePoints(color_primaries, transfer_characteristics, matrix_coefficients, color_range))); + + // Wipe the packet now that the data is safe. + av_packet_unref(m_packet); + return sample; + } +} + +} diff --git a/Libraries/LibMedia/FFmpeg/FFmpegDemuxer.h b/Libraries/LibMedia/FFmpeg/FFmpegDemuxer.h new file mode 100644 index 00000000000..97a91e25de5 --- /dev/null +++ b/Libraries/LibMedia/FFmpeg/FFmpegDemuxer.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2025, Luke Wilde + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include + +extern "C" { +#include +#include +} + +namespace Media::FFmpeg { + +class FFmpegDemuxer : public Demuxer { +public: + static ErrorOr> create(NonnullOwnPtr stream); + + FFmpegDemuxer(NonnullOwnPtr stream, NonnullOwnPtr); + virtual ~FFmpegDemuxer() override; + + virtual DecoderErrorOr> get_tracks_for_type(TrackType type) override; + + virtual DecoderErrorOr> seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional earliest_available_sample = OptionalNone()) override; + + virtual DecoderErrorOr duration(Track track) override; + + virtual DecoderErrorOr get_codec_id_for_track(Track track) override; + + virtual DecoderErrorOr get_codec_initialization_data_for_track(Track track) override; + + virtual DecoderErrorOr get_next_sample_for_track(Track track) override; + +private: + DecoderErrorOr duration_of_track_in_milliseconds(Track const& track); + + NonnullOwnPtr m_stream; + AVCodecContext* m_codec_context { nullptr }; + AVFormatContext* m_format_context { nullptr }; + NonnullOwnPtr m_io_context; + AVPacket* m_packet { nullptr }; +}; + +} diff --git a/Libraries/LibMedia/FFmpeg/FFmpegDemuxerStub.cpp b/Libraries/LibMedia/FFmpeg/FFmpegDemuxerStub.cpp new file mode 100644 index 00000000000..a13f264cde2 --- /dev/null +++ b/Libraries/LibMedia/FFmpeg/FFmpegDemuxerStub.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2025, Luke Wilde + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include + +namespace Media::FFmpeg { + +DecoderErrorOr> FFmpegDemuxer::get_tracks_for_type(TrackType type) +{ + (void)type; + return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform"); +} + +DecoderErrorOr> FFmpegDemuxer::seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional earliest_available_sample = OptionalNone()) +{ + (void)track; + (void)timestamp; + (void)earliest_available_sample; + return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform"); +} + +DecoderErrorOr FFmpegDemuxer::duration() +{ + return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform"); +} + +DecoderErrorOr FFmpegDemuxer::get_codec_id_for_track(Track track) +{ + (void)track; + return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform"); +} + +DecoderErrorOr FFmpegDemuxer::get_codec_initialization_data_for_track(Track track) +{ + (void)track; + return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform"); +} + +DecoderErrorOr FFmpegDemuxer::get_next_sample_for_track(Track track) +{ + (void)track; + return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform"); +} + +} diff --git a/Libraries/LibMedia/FFmpeg/FFmpegHelpers.h b/Libraries/LibMedia/FFmpeg/FFmpegHelpers.h index 74ebc5cb9a6..d06cd117c8c 100644 --- a/Libraries/LibMedia/FFmpeg/FFmpegHelpers.h +++ b/Libraries/LibMedia/FFmpeg/FFmpegHelpers.h @@ -14,7 +14,7 @@ extern "C" { namespace Media::FFmpeg { -static inline AVCodecID ffmpeg_codec_id_from_serenity_codec_id(CodecID codec) +static inline AVCodecID ffmpeg_codec_id_from_media_codec_id(CodecID codec) { switch (codec) { case CodecID::VP8: @@ -45,4 +45,35 @@ static inline AVCodecID ffmpeg_codec_id_from_serenity_codec_id(CodecID codec) } } +static inline CodecID media_codec_id_from_ffmpeg_codec_id(AVCodecID codec) +{ + switch (codec) { + case AV_CODEC_ID_VP8: + return CodecID::VP8; + case AV_CODEC_ID_VP9: + return CodecID::VP9; + case AV_CODEC_ID_H261: + return CodecID::H261; + case AV_CODEC_ID_MPEG2VIDEO: + // FIXME: This could also map to CodecID::MPEG1 + return CodecID::H262; + case AV_CODEC_ID_H263: + return CodecID::H263; + case AV_CODEC_ID_H264: + return CodecID::H264; + case AV_CODEC_ID_HEVC: + return CodecID::H265; + case AV_CODEC_ID_AV1: + return CodecID::AV1; + case AV_CODEC_ID_THEORA: + return CodecID::Theora; + case AV_CODEC_ID_VORBIS: + return CodecID::Vorbis; + case AV_CODEC_ID_OPUS: + return CodecID::Opus; + default: + return CodecID::Unknown; + } +} + } diff --git a/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp b/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp index bfae686a7f1..6b6396cb891 100644 --- a/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp +++ b/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp @@ -47,7 +47,7 @@ DecoderErrorOr> FFmpegVideoDecoder::try_create } }; - auto ff_codec_id = ffmpeg_codec_id_from_serenity_codec_id(codec_id); + auto ff_codec_id = ffmpeg_codec_id_from_media_codec_id(codec_id); auto const* codec = avcodec_find_decoder(ff_codec_id); if (!codec) return DecoderError::format(DecoderErrorCategory::NotImplemented, "Could not find FFmpeg decoder for codec {}", codec_id); diff --git a/Libraries/LibMedia/PlaybackManager.cpp b/Libraries/LibMedia/PlaybackManager.cpp index 64f51a2bc1e..412904c468d 100644 --- a/Libraries/LibMedia/PlaybackManager.cpp +++ b/Libraries/LibMedia/PlaybackManager.cpp @@ -5,8 +5,9 @@ */ #include +#include #include -#include +#include #include #include @@ -26,21 +27,15 @@ namespace Media { _fatal_expression.release_value(); \ }) -DecoderErrorOr> PlaybackManager::from_file(StringView filename) -{ - auto demuxer = TRY(Matroska::MatroskaDemuxer::from_file(filename)); - return create(move(demuxer)); -} - -DecoderErrorOr> PlaybackManager::from_mapped_file(NonnullOwnPtr mapped_file) -{ - auto demuxer = TRY(Matroska::MatroskaDemuxer::from_mapped_file(move(mapped_file))); - return create(move(demuxer)); -} - DecoderErrorOr> PlaybackManager::from_data(ReadonlyBytes data) { - auto demuxer = TRY(Matroska::MatroskaDemuxer::from_data(data)); + auto stream = make(data); + return from_stream(move(stream)); +} + +DecoderErrorOr> PlaybackManager::from_stream(NonnullOwnPtr stream) +{ + auto demuxer = MUST(FFmpeg::FFmpegDemuxer::create(move(stream))); return create(move(demuxer)); } @@ -93,7 +88,7 @@ AK::Duration PlaybackManager::duration() { auto duration_result = ({ auto demuxer_locker = Threading::MutexLocker(m_decoder_mutex); - m_demuxer->duration(); + m_demuxer->duration(m_selected_video_track); }); if (duration_result.is_error()) { dispatch_decoder_error(duration_result.release_error()); diff --git a/Libraries/LibMedia/PlaybackManager.h b/Libraries/LibMedia/PlaybackManager.h index a99cbbbdfe0..0682deef9db 100644 --- a/Libraries/LibMedia/PlaybackManager.h +++ b/Libraries/LibMedia/PlaybackManager.h @@ -110,10 +110,8 @@ public: static constexpr SeekMode DEFAULT_SEEK_MODE = SeekMode::Accurate; - static DecoderErrorOr> from_file(StringView file); - static DecoderErrorOr> from_mapped_file(NonnullOwnPtr file); - static DecoderErrorOr> from_data(ReadonlyBytes data); + static DecoderErrorOr> from_stream(NonnullOwnPtr stream); PlaybackManager(NonnullOwnPtr& demuxer, Track video_track, NonnullOwnPtr&& decoder, VideoFrameQueue&& frame_queue); ~PlaybackManager(); diff --git a/Libraries/LibMedia/Sample.h b/Libraries/LibMedia/Sample.h index ce8e5f7c74a..fedf5195cd7 100644 --- a/Libraries/LibMedia/Sample.h +++ b/Libraries/LibMedia/Sample.h @@ -17,7 +17,7 @@ class Sample final { public: using AuxiliaryData = Variant; - Sample(AK::Duration timestamp, ReadonlyBytes data, AuxiliaryData auxiliary_data) + Sample(AK::Duration timestamp, ByteBuffer data, AuxiliaryData auxiliary_data) : m_timestamp(timestamp) , m_data(data) , m_auxiliary_data(auxiliary_data) @@ -25,12 +25,12 @@ public: } AK::Duration timestamp() const { return m_timestamp; } - ReadonlyBytes const& data() const { return m_data; } + ByteBuffer const& data() const { return m_data; } AuxiliaryData const& auxiliary_data() const { return m_auxiliary_data; } private: AK::Duration m_timestamp; - ReadonlyBytes m_data; + ByteBuffer m_data; AuxiliaryData m_auxiliary_data; };