From dd188aafb9a47a4f6192c411807774ac8a5cf6ee Mon Sep 17 00:00:00 2001
From: Timothy Flynn <trflynn89@pm.me>
Date: Thu, 20 Apr 2023 06:40:31 -0400
Subject: [PATCH] LibWeb: Track decoded video frame positions along with the
 frame image

This will be needed by the layout node, which may change what is painted
when the position of the frame image is not the same as the element's
current time.
---
 Userland/Libraries/LibWeb/HTML/HTMLVideoElement.cpp   |  4 ++--
 Userland/Libraries/LibWeb/HTML/HTMLVideoElement.h     | 11 ++++++++---
 Userland/Libraries/LibWeb/HTML/VideoTrack.cpp         |  9 +++++----
 Userland/Libraries/LibWeb/Painting/VideoPaintable.cpp |  2 +-
 4 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.cpp b/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.cpp
index 2471351da4e..adfec5c96e0 100644
--- a/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.cpp
+++ b/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.cpp
@@ -83,9 +83,9 @@ void HTMLVideoElement::set_video_track(JS::GCPtr<HTML::VideoTrack> video_track)
     m_video_track = video_track;
 }
 
-void HTMLVideoElement::set_current_frame(Badge<VideoTrack>, RefPtr<Gfx::Bitmap> frame)
+void HTMLVideoElement::set_current_frame(Badge<VideoTrack>, RefPtr<Gfx::Bitmap> frame, double position)
 {
-    m_current_frame = move(frame);
+    m_current_frame = { move(frame), position };
     layout_node()->set_needs_display();
 }
 
diff --git a/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.h b/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.h
index 95262163db0..3de683d52a5 100644
--- a/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.h
+++ b/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.h
@@ -13,6 +13,11 @@
 
 namespace Web::HTML {
 
+struct VideoFrame {
+    RefPtr<Gfx::Bitmap> frame;
+    double position { 0.0 };
+};
+
 class HTMLVideoElement final : public HTMLMediaElement {
     WEB_PLATFORM_OBJECT(HTMLVideoElement, HTMLMediaElement);
 
@@ -30,8 +35,8 @@ public:
 
     void set_video_track(JS::GCPtr<VideoTrack>);
 
-    void set_current_frame(Badge<VideoTrack>, RefPtr<Gfx::Bitmap> frame);
-    RefPtr<Gfx::Bitmap> const& current_frame() const { return m_current_frame; }
+    void set_current_frame(Badge<VideoTrack>, RefPtr<Gfx::Bitmap> frame, double position);
+    VideoFrame const& current_frame() const { return m_current_frame; }
 
     void set_layout_mouse_position(Badge<Painting::VideoPaintable>, Optional<CSSPixelPoint> mouse_position) { m_mouse_position = move(mouse_position); }
     Optional<CSSPixelPoint> const& layout_mouse_position(Badge<Painting::VideoPaintable>) const { return m_mouse_position; }
@@ -56,7 +61,7 @@ private:
     virtual void on_seek(double, MediaSeekMode) override;
 
     JS::GCPtr<HTML::VideoTrack> m_video_track;
-    RefPtr<Gfx::Bitmap> m_current_frame;
+    VideoFrame m_current_frame;
 
     u32 m_video_width { 0 };
     u32 m_video_height { 0 };
diff --git a/Userland/Libraries/LibWeb/HTML/VideoTrack.cpp b/Userland/Libraries/LibWeb/HTML/VideoTrack.cpp
index abd58ed0ebc..a1eda89da04 100644
--- a/Userland/Libraries/LibWeb/HTML/VideoTrack.cpp
+++ b/Userland/Libraries/LibWeb/HTML/VideoTrack.cpp
@@ -30,11 +30,12 @@ VideoTrack::VideoTrack(JS::Realm& realm, JS::NonnullGCPtr<HTMLMediaElement> medi
     , m_playback_manager(move(playback_manager))
 {
     m_playback_manager->on_video_frame = [this](auto frame) {
-        if (is<HTMLVideoElement>(*m_media_element))
-            verify_cast<HTMLVideoElement>(*m_media_element).set_current_frame({}, move(frame));
+        auto playback_position = static_cast<double>(position().to_milliseconds()) / 1000.0;
 
-        auto playback_position_ms = static_cast<double>(position().to_milliseconds());
-        m_media_element->set_current_playback_position(playback_position_ms / 1000.0);
+        if (is<HTMLVideoElement>(*m_media_element))
+            verify_cast<HTMLVideoElement>(*m_media_element).set_current_frame({}, move(frame), playback_position);
+
+        m_media_element->set_current_playback_position(playback_position);
     };
 
     m_playback_manager->on_playback_state_change = [this]() {
diff --git a/Userland/Libraries/LibWeb/Painting/VideoPaintable.cpp b/Userland/Libraries/LibWeb/Painting/VideoPaintable.cpp
index 8dd1b571cd1..4213838ec07 100644
--- a/Userland/Libraries/LibWeb/Painting/VideoPaintable.cpp
+++ b/Userland/Libraries/LibWeb/Painting/VideoPaintable.cpp
@@ -76,7 +76,7 @@ void VideoPaintable::paint(PaintContext& context, PaintPhase phase) const
 
     auto paint_user_agent_controls = video_element.has_attribute(HTML::AttributeNames::controls) || video_element.is_scripting_disabled();
 
-    if (auto const& bitmap = layout_box().dom_node().current_frame()) {
+    if (auto const& bitmap = layout_box().dom_node().current_frame().frame) {
         context.painter().draw_scaled_bitmap(video_rect.to_type<int>(), *bitmap, bitmap->rect(), 1.0f, to_gfx_scaling_mode(computed_values().image_rendering()));
 
         if (paint_user_agent_controls)