LibMedia: Demux videos with FFmpeg

This gives us access to container types other than Matroska, the
biggest one being MP4.

Libraries/LibMedia/CMakeLists.txt

@@ -22,13 +22,17 @@ target_link_libraries(LibMedia PRIVATE LibCore LibCrypto LibRIFF LibIPC LibGfx L
 if (NOT ANDROID)
     target_sources(LibMedia PRIVATE
             Audio/FFmpegLoader.cpp
+            FFmpeg/FFmpegDemuxer.cpp
             FFmpeg/FFmpegIOContext.cpp
             FFmpeg/FFmpegVideoDecoder.cpp
     )
     target_link_libraries(LibMedia PRIVATE PkgConfig::AVCODEC PkgConfig::AVFORMAT PkgConfig::AVUTIL)
 else()
     # FIXME: Need to figure out how to build or replace ffmpeg libs on Android and Windows
-    target_sources(LibMedia PRIVATE FFmpeg/FFmpegVideoDecoderStub.cpp)
+    target_sources(LibMedia PRIVATE
+            FFmpeg/FFmpegDemuxerStub.cpp
+            FFmpeg/FFmpegVideoDecoderStub.cpp
+    )
 endif()
 
 if (LADYBIRD_AUDIO_BACKEND STREQUAL "PULSE")

Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp

@@ -49,7 +49,7 @@ DecoderErrorOr<Vector<Track>> MatroskaDemuxer::get_tracks_for_type(TrackType typ
         switch (type) {
         case TrackType::Video:
             if (auto video_track = track_entry.video_track(); video_track.has_value())
-                track.set_video_data({ TRY(duration()), video_track->pixel_width, video_track->pixel_height });
+                track.set_video_data({ TRY(duration(track)), video_track->pixel_width, video_track->pixel_height });
             break;
         default:
             break;
@@ -148,10 +148,11 @@ DecoderErrorOr<Sample> MatroskaDemuxer::get_next_sample_for_track(Track track)
         status.frame_index = 0;
     }
     auto cicp = TRY(m_reader.track_for_track_number(track.identifier()))->video_track()->color_format.to_cicp();
-    return Sample(status.block->timestamp(), status.block->frame(status.frame_index++), VideoSampleData(cicp));
+    auto sample_data = DECODER_TRY_ALLOC(ByteBuffer::copy(status.block->frame(status.frame_index++)));
+    return Sample(status.block->timestamp(), move(sample_data), VideoSampleData(cicp));
 }
 
-DecoderErrorOr<AK::Duration> MatroskaDemuxer::duration()
+DecoderErrorOr<AK::Duration> MatroskaDemuxer::duration(Track)
 {
     auto duration = TRY(m_reader.segment_information()).duration();
     return duration.value_or(AK::Duration::zero());

Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h

@@ -31,7 +31,7 @@ public:
 
     DecoderErrorOr<Optional<AK::Duration>> seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional<AK::Duration> earliest_available_sample = OptionalNone()) override;
 
-    DecoderErrorOr<AK::Duration> duration() override;
+    DecoderErrorOr<AK::Duration> duration(Track track) override;
 
     DecoderErrorOr<CodecID> get_codec_id_for_track(Track track) override;
 

Libraries/LibMedia/Demuxer.h

@@ -33,7 +33,7 @@ public:
     // in the case that the timestamp is closer to the current time than the nearest keyframe.
     virtual DecoderErrorOr<Optional<AK::Duration>> seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional<AK::Duration> earliest_available_sample = OptionalNone()) = 0;
 
-    virtual DecoderErrorOr<AK::Duration> duration() = 0;
+    virtual DecoderErrorOr<AK::Duration> duration(Track track) = 0;
 };
 
 }

Libraries/LibMedia/FFmpeg/FFmpegDemuxer.cpp

@@ -0,0 +1,202 @@
+/*
+ * Copyright (c) 2025, Luke Wilde <luke@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Math.h>
+#include <AK/Stream.h>
+#include <LibMedia/FFmpeg/FFmpegDemuxer.h>
+#include <LibMedia/FFmpeg/FFmpegHelpers.h>
+
+namespace Media::FFmpeg {
+
+FFmpegDemuxer::FFmpegDemuxer(NonnullOwnPtr<SeekableStream> stream, NonnullOwnPtr<Media::FFmpeg::FFmpegIOContext> io_context)
+    : m_stream(move(stream))
+    , m_io_context(move(io_context))
+{
+}
+
+FFmpegDemuxer::~FFmpegDemuxer()
+{
+    if (m_packet != nullptr)
+        av_packet_free(&m_packet);
+    if (m_codec_context != nullptr)
+        avcodec_free_context(&m_codec_context);
+    if (m_format_context != nullptr)
+        avformat_close_input(&m_format_context);
+}
+
+ErrorOr<NonnullOwnPtr<FFmpegDemuxer>> FFmpegDemuxer::create(NonnullOwnPtr<SeekableStream> stream)
+{
+    auto io_context = TRY(Media::FFmpeg::FFmpegIOContext::create(*stream));
+    auto demuxer = make<FFmpegDemuxer>(move(stream), move(io_context));
+
+    // Open the container
+    demuxer->m_format_context = avformat_alloc_context();
+    if (demuxer->m_format_context == nullptr)
+        return Error::from_string_literal("Failed to allocate format context");
+    demuxer->m_format_context->pb = demuxer->m_io_context->avio_context();
+    if (avformat_open_input(&demuxer->m_format_context, nullptr, nullptr, nullptr) < 0)
+        return Error::from_string_literal("Failed to open input for format parsing");
+
+    // Read stream info; doing this is required for headerless formats like MPEG
+    if (avformat_find_stream_info(demuxer->m_format_context, nullptr) < 0)
+        return Error::from_string_literal("Failed to find stream info");
+
+    demuxer->m_packet = av_packet_alloc();
+    if (demuxer->m_packet == nullptr)
+        return Error::from_string_literal("Failed to allocate packet");
+
+    return demuxer;
+}
+
+DecoderErrorOr<AK::Duration> FFmpegDemuxer::duration_of_track_in_milliseconds(Track const& track)
+{
+    VERIFY(track.identifier() < m_format_context->nb_streams);
+    auto* stream = m_format_context->streams[track.identifier()];
+
+    if (stream->duration >= 0) {
+        auto time_base = av_q2d(stream->time_base);
+        double duration_in_milliseconds = static_cast<double>(stream->duration) * time_base * 1000.0;
+        return AK::Duration::from_milliseconds(AK::round_to<int64_t>(duration_in_milliseconds));
+    }
+
+    // If the stream doesn't specify the duration, fallback to what the container says the duration is.
+    // If the container doesn't know the duration, then we're out of luck. Return an error.
+    if (m_format_context->duration < 0)
+        return DecoderError::format(DecoderErrorCategory::Unknown, "Negative stream duration");
+
+    double duration_in_milliseconds = (static_cast<double>(m_format_context->duration) / AV_TIME_BASE) * 1000.0;
+    return AK::Duration::from_milliseconds(AK::round_to<int64_t>(duration_in_milliseconds));
+}
+
+DecoderErrorOr<Vector<Track>> FFmpegDemuxer::get_tracks_for_type(TrackType type)
+{
+    AVMediaType media_type;
+
+    switch (type) {
+    case TrackType::Video:
+        media_type = AVMediaType::AVMEDIA_TYPE_VIDEO;
+        break;
+    case TrackType::Audio:
+        media_type = AVMediaType::AVMEDIA_TYPE_AUDIO;
+        break;
+    case TrackType::Subtitles:
+        media_type = AVMediaType::AVMEDIA_TYPE_SUBTITLE;
+        break;
+    }
+
+    // Find the best stream to play within the container
+    int best_stream_index = av_find_best_stream(m_format_context, media_type, -1, -1, nullptr, 0);
+    if (best_stream_index == AVERROR_STREAM_NOT_FOUND)
+        return DecoderError::format(DecoderErrorCategory::Unknown, "No stream for given type found in container");
+    if (best_stream_index == AVERROR_DECODER_NOT_FOUND)
+        return DecoderError::format(DecoderErrorCategory::Unknown, "No suitable decoder found for stream");
+    if (best_stream_index < 0)
+        return DecoderError::format(DecoderErrorCategory::Unknown, "Failed to find a stream for the given type");
+
+    auto* stream = m_format_context->streams[best_stream_index];
+
+    Track track(type, best_stream_index);
+
+    if (type == TrackType::Video) {
+        track.set_video_data({
+            .duration = TRY(duration_of_track_in_milliseconds(track)),
+            .pixel_width = static_cast<u64>(stream->codecpar->width),
+            .pixel_height = static_cast<u64>(stream->codecpar->height),
+        });
+    }
+
+    Vector<Track> tracks;
+    tracks.append(move(track));
+    return tracks;
+}
+
+DecoderErrorOr<Optional<AK::Duration>> FFmpegDemuxer::seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional<AK::Duration> earliest_available_sample)
+{
+    // FIXME: What do we do with this here?
+    (void)earliest_available_sample;
+
+    VERIFY(track.identifier() < m_format_context->nb_streams);
+    auto* stream = m_format_context->streams[track.identifier()];
+    auto time_base = av_q2d(stream->time_base);
+    auto time_in_seconds = static_cast<double>(timestamp.to_milliseconds()) / 1000.0 / time_base;
+    auto sample_timestamp = AK::round_to<int64_t>(time_in_seconds);
+
+    if (av_seek_frame(m_format_context, stream->index, sample_timestamp, AVSEEK_FLAG_BACKWARD) < 0)
+        return DecoderError::format(DecoderErrorCategory::Unknown, "Failed to seek");
+
+    return timestamp;
+}
+
+DecoderErrorOr<AK::Duration> FFmpegDemuxer::duration(Track track)
+{
+    return duration_of_track_in_milliseconds(track);
+}
+
+DecoderErrorOr<CodecID> FFmpegDemuxer::get_codec_id_for_track(Track track)
+{
+    VERIFY(track.identifier() < m_format_context->nb_streams);
+    auto* stream = m_format_context->streams[track.identifier()];
+    return media_codec_id_from_ffmpeg_codec_id(stream->codecpar->codec_id);
+}
+
+DecoderErrorOr<ReadonlyBytes> FFmpegDemuxer::get_codec_initialization_data_for_track(Track track)
+{
+    VERIFY(track.identifier() < m_format_context->nb_streams);
+    auto* stream = m_format_context->streams[track.identifier()];
+    return ReadonlyBytes { stream->codecpar->extradata, static_cast<size_t>(stream->codecpar->extradata_size) };
+}
+
+DecoderErrorOr<Sample> FFmpegDemuxer::get_next_sample_for_track(Track track)
+{
+    VERIFY(track.identifier() < m_format_context->nb_streams);
+    auto* stream = m_format_context->streams[track.identifier()];
+
+    for (;;) {
+        auto read_frame_error = av_read_frame(m_format_context, m_packet);
+        if (read_frame_error < 0) {
+            if (read_frame_error == AVERROR_EOF)
+                return DecoderError::format(DecoderErrorCategory::EndOfStream, "End of stream");
+
+            return DecoderError::format(DecoderErrorCategory::Unknown, "Failed to read frame");
+        }
+        if (m_packet->stream_index != stream->index) {
+            av_packet_unref(m_packet);
+            continue;
+        }
+
+        auto color_primaries = static_cast<ColorPrimaries>(stream->codecpar->color_primaries);
+        auto transfer_characteristics = static_cast<TransferCharacteristics>(stream->codecpar->color_trc);
+        auto matrix_coefficients = static_cast<MatrixCoefficients>(stream->codecpar->color_space);
+        auto color_range = [stream] {
+            switch (stream->codecpar->color_range) {
+            case AVColorRange::AVCOL_RANGE_MPEG:
+                return VideoFullRangeFlag::Studio;
+            case AVColorRange::AVCOL_RANGE_JPEG:
+                return VideoFullRangeFlag::Full;
+            default:
+                return VideoFullRangeFlag::Unspecified;
+            }
+        }();
+
+        auto time_base = av_q2d(stream->time_base);
+        double timestamp_in_milliseconds = static_cast<double>(m_packet->pts) * time_base * 1000.0;
+
+        // Copy the packet data so that we have a permanent reference to it whilst the Sample is alive, which allows us
+        // to wipe the packet afterwards.
+        auto packet_data = DECODER_TRY_ALLOC(ByteBuffer::copy(m_packet->data, m_packet->size));
+
+        auto sample = Sample(
+            AK::Duration::from_milliseconds(AK::round_to<int64_t>(timestamp_in_milliseconds)),
+            move(packet_data),
+            VideoSampleData(CodingIndependentCodePoints(color_primaries, transfer_characteristics, matrix_coefficients, color_range)));
+
+        // Wipe the packet now that the data is safe.
+        av_packet_unref(m_packet);
+        return sample;
+    }
+}
+
+}

Libraries/LibMedia/FFmpeg/FFmpegDemuxer.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2025, Luke Wilde <luke@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Error.h>
+#include <AK/NonnullOwnPtr.h>
+#include <LibMedia/Demuxer.h>
+#include <LibMedia/FFmpeg/FFmpegIOContext.h>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+}
+
+namespace Media::FFmpeg {
+
+class FFmpegDemuxer : public Demuxer {
+public:
+    static ErrorOr<NonnullOwnPtr<FFmpegDemuxer>> create(NonnullOwnPtr<SeekableStream> stream);
+
+    FFmpegDemuxer(NonnullOwnPtr<SeekableStream> stream, NonnullOwnPtr<Media::FFmpeg::FFmpegIOContext>);
+    virtual ~FFmpegDemuxer() override;
+
+    virtual DecoderErrorOr<Vector<Track>> get_tracks_for_type(TrackType type) override;
+
+    virtual DecoderErrorOr<Optional<AK::Duration>> seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional<AK::Duration> earliest_available_sample = OptionalNone()) override;
+
+    virtual DecoderErrorOr<AK::Duration> duration(Track track) override;
+
+    virtual DecoderErrorOr<CodecID> get_codec_id_for_track(Track track) override;
+
+    virtual DecoderErrorOr<ReadonlyBytes> get_codec_initialization_data_for_track(Track track) override;
+
+    virtual DecoderErrorOr<Sample> get_next_sample_for_track(Track track) override;
+
+private:
+    DecoderErrorOr<AK::Duration> duration_of_track_in_milliseconds(Track const& track);
+
+    NonnullOwnPtr<SeekableStream> m_stream;
+    AVCodecContext* m_codec_context { nullptr };
+    AVFormatContext* m_format_context { nullptr };
+    NonnullOwnPtr<Media::FFmpeg::FFmpegIOContext> m_io_context;
+    AVPacket* m_packet { nullptr };
+};
+
+}

Libraries/LibMedia/FFmpeg/FFmpegDemuxerStub.cpp

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2025, Luke Wilde <luke@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibMedia/FFmpeg/FFmpegDemuxer.h>
+
+namespace Media::FFmpeg {
+
+DecoderErrorOr<Vector<Track>> FFmpegDemuxer::get_tracks_for_type(TrackType type)
+{
+    (void)type;
+    return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform");
+}
+
+DecoderErrorOr<Optional<AK::Duration>> FFmpegDemuxer::seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional<AK::Duration> earliest_available_sample = OptionalNone())
+{
+    (void)track;
+    (void)timestamp;
+    (void)earliest_available_sample;
+    return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform");
+}
+
+DecoderErrorOr<AK::Duration> FFmpegDemuxer::duration()
+{
+    return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform");
+}
+
+DecoderErrorOr<CodecID> FFmpegDemuxer::get_codec_id_for_track(Track track)
+{
+    (void)track;
+    return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform");
+}
+
+DecoderErrorOr<ReadonlyBytes> FFmpegDemuxer::get_codec_initialization_data_for_track(Track track)
+{
+    (void)track;
+    return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform");
+}
+
+DecoderErrorOr<Sample> FFmpegDemuxer::get_next_sample_for_track(Track track)
+{
+    (void)track;
+    return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform");
+}
+
+}

Libraries/LibMedia/FFmpeg/FFmpegHelpers.h

@@ -14,7 +14,7 @@ extern "C" {
 
 namespace Media::FFmpeg {
 
-static inline AVCodecID ffmpeg_codec_id_from_serenity_codec_id(CodecID codec)
+static inline AVCodecID ffmpeg_codec_id_from_media_codec_id(CodecID codec)
 {
     switch (codec) {
     case CodecID::VP8:
@@ -45,4 +45,35 @@ static inline AVCodecID ffmpeg_codec_id_from_serenity_codec_id(CodecID codec)
     }
 }
 
+static inline CodecID media_codec_id_from_ffmpeg_codec_id(AVCodecID codec)
+{
+    switch (codec) {
+    case AV_CODEC_ID_VP8:
+        return CodecID::VP8;
+    case AV_CODEC_ID_VP9:
+        return CodecID::VP9;
+    case AV_CODEC_ID_H261:
+        return CodecID::H261;
+    case AV_CODEC_ID_MPEG2VIDEO:
+        // FIXME: This could also map to CodecID::MPEG1
+        return CodecID::H262;
+    case AV_CODEC_ID_H263:
+        return CodecID::H263;
+    case AV_CODEC_ID_H264:
+        return CodecID::H264;
+    case AV_CODEC_ID_HEVC:
+        return CodecID::H265;
+    case AV_CODEC_ID_AV1:
+        return CodecID::AV1;
+    case AV_CODEC_ID_THEORA:
+        return CodecID::Theora;
+    case AV_CODEC_ID_VORBIS:
+        return CodecID::Vorbis;
+    case AV_CODEC_ID_OPUS:
+        return CodecID::Opus;
+    default:
+        return CodecID::Unknown;
+    }
+}
+
 }

Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp

@@ -47,7 +47,7 @@ DecoderErrorOr<NonnullOwnPtr<FFmpegVideoDecoder>> FFmpegVideoDecoder::try_create
         }
     };
 
-    auto ff_codec_id = ffmpeg_codec_id_from_serenity_codec_id(codec_id);
+    auto ff_codec_id = ffmpeg_codec_id_from_media_codec_id(codec_id);
     auto const* codec = avcodec_find_decoder(ff_codec_id);
     if (!codec)
         return DecoderError::format(DecoderErrorCategory::NotImplemented, "Could not find FFmpeg decoder for codec {}", codec_id);

Libraries/LibMedia/PlaybackManager.cpp

@@ -5,8 +5,9 @@
  */
 
 #include <AK/Format.h>
+#include <LibCore/MappedFile.h>
 #include <LibCore/Timer.h>
-#include <LibMedia/Containers/Matroska/MatroskaDemuxer.h>
+#include <LibMedia/FFmpeg/FFmpegDemuxer.h>
 #include <LibMedia/FFmpeg/FFmpegVideoDecoder.h>
 #include <LibMedia/VideoFrame.h>
 
@@ -26,21 +27,15 @@ namespace Media {
         _fatal_expression.release_value();                                                           \
     })
 
-DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> PlaybackManager::from_file(StringView filename)
-{
-    auto demuxer = TRY(Matroska::MatroskaDemuxer::from_file(filename));
-    return create(move(demuxer));
-}
-
-DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> PlaybackManager::from_mapped_file(NonnullOwnPtr<Core::MappedFile> mapped_file)
-{
-    auto demuxer = TRY(Matroska::MatroskaDemuxer::from_mapped_file(move(mapped_file)));
-    return create(move(demuxer));
-}
-
 DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> PlaybackManager::from_data(ReadonlyBytes data)
 {
-    auto demuxer = TRY(Matroska::MatroskaDemuxer::from_data(data));
+    auto stream = make<FixedMemoryStream>(data);
+    return from_stream(move(stream));
+}
+
+DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> PlaybackManager::from_stream(NonnullOwnPtr<SeekableStream> stream)
+{
+    auto demuxer = MUST(FFmpeg::FFmpegDemuxer::create(move(stream)));
     return create(move(demuxer));
 }
 
@@ -93,7 +88,7 @@ AK::Duration PlaybackManager::duration()
 {
     auto duration_result = ({
         auto demuxer_locker = Threading::MutexLocker(m_decoder_mutex);
-        m_demuxer->duration();
+        m_demuxer->duration(m_selected_video_track);
     });
     if (duration_result.is_error()) {
         dispatch_decoder_error(duration_result.release_error());

Libraries/LibMedia/PlaybackManager.h

@@ -110,10 +110,8 @@ public:
 
     static constexpr SeekMode DEFAULT_SEEK_MODE = SeekMode::Accurate;
 
-    static DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> from_file(StringView file);
-    static DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> from_mapped_file(NonnullOwnPtr<Core::MappedFile> file);
-
     static DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> from_data(ReadonlyBytes data);
+    static DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> from_stream(NonnullOwnPtr<SeekableStream> stream);
 
     PlaybackManager(NonnullOwnPtr<Demuxer>& demuxer, Track video_track, NonnullOwnPtr<VideoDecoder>&& decoder, VideoFrameQueue&& frame_queue);
     ~PlaybackManager();

Libraries/LibMedia/Sample.h

@@ -17,7 +17,7 @@ class Sample final {
 public:
     using AuxiliaryData = Variant<VideoSampleData>;
 
-    Sample(AK::Duration timestamp, ReadonlyBytes data, AuxiliaryData auxiliary_data)
+    Sample(AK::Duration timestamp, ByteBuffer data, AuxiliaryData auxiliary_data)
         : m_timestamp(timestamp)
         , m_data(data)
         , m_auxiliary_data(auxiliary_data)
@@ -25,12 +25,12 @@ public:
     }
 
     AK::Duration timestamp() const { return m_timestamp; }
-    ReadonlyBytes const& data() const { return m_data; }
+    ByteBuffer const& data() const { return m_data; }
     AuxiliaryData const& auxiliary_data() const { return m_auxiliary_data; }
 
 private:
     AK::Duration m_timestamp;
-    ReadonlyBytes m_data;
+    ByteBuffer m_data;
     AuxiliaryData m_auxiliary_data;
 };