Wip ratelimiting subs

app/src/main/java/com/futo/platformplayer/Settings.kt

@@ -140,7 +140,11 @@ class Settings : FragmentedStorageFileJson() {
                 return FeedStyle.THUMBNAIL;
         }
 
-        @FormField("Background Update", FieldForm.DROPDOWN, "Experimental background update for subscriptions cache (requires restart)", 6)
+        @FormField("Fetch on app boot", FieldForm.TOGGLE, "Shortly after opening the app, start fetching subscriptions.", 6)
+        @Serializable(with = FlexibleBooleanSerializer::class)
+        var fetchOnAppBoot: Boolean = true;
+
+        @FormField("Background Update", FieldForm.DROPDOWN, "Experimental background update for subscriptions cache (requires restart)", 7)
         @DropdownFieldOptionsId(R.array.background_interval)
         var subscriptionsBackgroundUpdateInterval: Int = 0;
 
@@ -156,7 +160,7 @@ class Settings : FragmentedStorageFileJson() {
         };
 
 
-        @FormField("Subscription Concurrency", FieldForm.DROPDOWN, "Specify how many threads are used to fetch channels (requires restart)", 7)
+        @FormField("Subscription Concurrency", FieldForm.DROPDOWN, "Specify how many threads are used to fetch channels (requires restart)", 8)
         @DropdownFieldOptionsId(R.array.thread_count)
         var subscriptionConcurrency: Int = 3;
 

app/src/main/java/com/futo/platformplayer/api/media/models/ResultCapabilities.kt

@@ -27,6 +27,7 @@ class ResultCapabilities(
         const val TYPE_VIDEOS = "VIDEOS";
         const val TYPE_STREAMS = "STREAMS";
         const val TYPE_LIVE = "LIVE";
+        const val TYPE_POSTS = "POSTS";
         const val TYPE_MIXED = "MIXED";
 
         const val ORDER_CHONOLOGICAL = "CHRONOLOGICAL";

app/src/main/java/com/futo/platformplayer/api/media/platforms/js/SourcePluginConfig.kt

@@ -41,6 +41,7 @@ class SourcePluginConfig(
     val constants: HashMap<String, String> = hashMapOf(),
 
     //TODO: These should be vals...but prob for serialization reasons cannot be changed.
+    var subscriptionRateLimit: Int? = null,
     var enableInSearch: Boolean = true,
     var enableInHome: Boolean = true,
     var supportedClaimTypes: List<Int> = listOf()

app/src/main/java/com/futo/platformplayer/models/Subscription.kt

@@ -3,6 +3,7 @@ package com.futo.platformplayer.models
 import com.futo.platformplayer.api.media.models.channels.IPlatformChannel
 import com.futo.platformplayer.api.media.models.channels.SerializedChannel
 import com.futo.platformplayer.api.media.models.contents.IPlatformContent
+import com.futo.platformplayer.getNowDiffDays
 import com.futo.platformplayer.serializers.OffsetDateTimeSerializer
 import java.time.OffsetDateTime
 
@@ -10,17 +11,37 @@ import java.time.OffsetDateTime
 class Subscription {
     var channel: SerializedChannel;
 
+    //Last found content
     @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
     var lastVideo : OffsetDateTime = OffsetDateTime.MAX;
     @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
     var lastLiveStream : OffsetDateTime = OffsetDateTime.MAX;
+    @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
+    var lastPost : OffsetDateTime = OffsetDateTime.MAX;
 
+    //Last update time
+    @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
+    var lastVideoUpdate : OffsetDateTime = OffsetDateTime.MIN;
+    @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
+    var lastStreamUpdate : OffsetDateTime = OffsetDateTime.MIN;
+    @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
+    var lastLiveStreamUpdate : OffsetDateTime = OffsetDateTime.MIN;
+    @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
+    var lastPostUpdate : OffsetDateTime = OffsetDateTime.MIN;
+
+    //Last video interval
     var uploadInterval : Int = 0;
+    var uploadPostInterval : Int = 0;
+
 
     constructor(channel : SerializedChannel) {
         this.channel = channel;
     }
 
+    fun shouldFetchStreams() = lastLiveStream.getNowDiffDays() < 7;
+    fun shouldFetchLiveStreams() = lastLiveStream.getNowDiffDays() < 14;
+    fun shouldFetchPosts() = lastPost.getNowDiffDays() < 2;
+
     fun updateChannel(channel: IPlatformChannel) {
         this.channel = SerializedChannel.fromChannel(channel);
     }

app/src/main/java/com/futo/platformplayer/states/StateApp.kt

@@ -28,6 +28,7 @@ import com.futo.platformplayer.R
 import com.futo.platformplayer.activities.CaptchaActivity
 import com.futo.platformplayer.activities.IWithResultLauncher
 import com.futo.platformplayer.activities.MainActivity
+import com.futo.platformplayer.api.media.Serializer
 import com.futo.platformplayer.api.media.platforms.js.JSClient
 import com.futo.platformplayer.api.media.platforms.js.internal.JSHttpClient
 import com.futo.platformplayer.background.BackgroundWorker
@@ -44,7 +45,10 @@ import com.futo.platformplayer.receivers.AudioNoisyReceiver
 import com.futo.platformplayer.services.DownloadService
 import com.futo.platformplayer.stores.FragmentedStorage
 import com.futo.platformplayer.stores.v2.ManagedStore
+import com.stripe.android.core.utils.encodeToJson
 import kotlinx.coroutines.*
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
 import java.io.File
 import java.time.OffsetDateTime
 import java.util.*
@@ -429,10 +433,22 @@ class StateApp {
             StatePlaylists.instance.toMigrateCheck()
         ).flatten(), 0);
 
+        if(Settings.instance.subscriptions.fetchOnAppBoot) {
+            val subRequestCounts = StateSubscriptions.instance.getSubscriptionRequestCount();
+            val reqCountStr = subRequestCounts.map { "    ${it.key.config.name}: ${it.value}/${it.key.config.subscriptionRateLimit}" }.joinToString("\n");
+            if (!subRequestCounts.any { clientCount ->
+                    clientCount.key.config.subscriptionRateLimit
+                        ?.let { rateLimit -> clientCount.value > rateLimit } == true
+                }) {
+                Logger.w(TAG, "Subscriptions request on boot, request counts:\n${reqCountStr}");
             scope.launch {
                 delay(5000);
                 StateSubscriptions.instance.updateSubscriptionFeed(scope, false);
             }
+        }
+        else
+            Logger.w(TAG, "Too many subscription requests required:\n${reqCountStr}");
+    }
 
         val interval = Settings.instance.subscriptions.getSubscriptionsBackgroundIntervalMinutes();
         scheduleBackgroundWork(context, interval != 0, interval);

app/src/main/java/com/futo/platformplayer/states/StatePlatform.kt

@@ -664,19 +664,24 @@ class StatePlatform {
                 toQuery.add(ResultCapabilities.TYPE_STREAMS);
             if(clientCapabilities.hasType(ResultCapabilities.TYPE_LIVE))
                 toQuery.add(ResultCapabilities.TYPE_LIVE);
+            if(clientCapabilities.hasType(ResultCapabilities.TYPE_POSTS))
+                toQuery.add(ResultCapabilities.TYPE_POSTS);
 
             if(isSubscriptionOptimized) {
                 val sub = StateSubscriptions.instance.getSubscription(channelUrl);
                 if(sub != null) {
-                    val daysSinceLiveStream = sub.lastLiveStream.getNowDiffDays()
+                    if(!sub.shouldFetchStreams()) {
-                    if(daysSinceLiveStream > 7) {
+                        Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 7 days, skipping live streams [${sub.lastLiveStream.getNowDiffDays()} days ago]");
-                        Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 7 days, skipping live streams [${daysSinceLiveStream} days ago]");
                         toQuery.remove(ResultCapabilities.TYPE_LIVE);
                     }
-                    if(daysSinceLiveStream > 14) {
+                    if(!sub.shouldFetchLiveStreams()) {
-                        Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 15 days, skipping streams [${daysSinceLiveStream} days ago]");
+                        Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 15 days, skipping streams [${sub.lastLiveStream.getNowDiffDays()} days ago]");
                         toQuery.remove(ResultCapabilities.TYPE_STREAMS);
                     }
+                    if(!sub.shouldFetchPosts()) {
+                        Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 5 days, skipping posts [${sub.lastPost.getNowDiffDays()} days ago]");
+                        toQuery.remove(ResultCapabilities.TYPE_POSTS);
+                    }
                 }
             }
 

app/src/main/java/com/futo/platformplayer/states/StateSubscriptions.kt

@@ -2,6 +2,7 @@ package com.futo.platformplayer.states
 
 import com.futo.platformplayer.Settings
 import com.futo.platformplayer.UIDialogs
+import com.futo.platformplayer.api.media.models.ResultCapabilities
 import com.futo.platformplayer.api.media.models.channels.IPlatformChannel
 import com.futo.platformplayer.api.media.models.channels.SerializedChannel
 import com.futo.platformplayer.api.media.models.contents.IPlatformContent
@@ -18,6 +19,7 @@ import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
 import com.futo.platformplayer.exceptions.ChannelException
 import com.futo.platformplayer.findNonRuntimeException
 import com.futo.platformplayer.fragment.mainactivity.main.PolycentricProfile
+import com.futo.platformplayer.getNowDiffDays
 import com.futo.platformplayer.logging.Logger
 import com.futo.platformplayer.models.Subscription
 import com.futo.platformplayer.polycentric.PolycentricCache
@@ -219,6 +221,31 @@ class StateSubscriptions {
         }
     }
 
+    fun getSubscriptionRequestCount(): Map<JSClient, Int> {
+        val subs = getSubscriptions();
+        val pluginReqCounts = mutableMapOf<JSClient, Int>();
+
+        for(sub in subs) {
+            val client = StatePlatform.instance.getChannelClientOrNull(sub.channel.url);
+            if(client !is JSClient)
+                continue;
+
+            val channelCaps = client.getChannelCapabilities();
+            if(!pluginReqCounts.containsKey(client))
+                pluginReqCounts[client] = 1;
+            else
+                pluginReqCounts[client] = pluginReqCounts[client]!! + 1;
+
+            if(channelCaps.hasType(ResultCapabilities.TYPE_STREAMS) && sub.shouldFetchStreams())
+                pluginReqCounts[client] = pluginReqCounts[client]!! + 1;
+            if(channelCaps.hasType(ResultCapabilities.TYPE_LIVE) && sub.shouldFetchLiveStreams())
+                pluginReqCounts[client] = pluginReqCounts[client]!! + 1;
+            if(channelCaps.hasType(ResultCapabilities.TYPE_POSTS) && sub.shouldFetchPosts())
+                pluginReqCounts[client] = pluginReqCounts[client]!! + 1;
+        }
+        return pluginReqCounts;
+    }
+
     fun getSubscriptionsFeed(allowFailure: Boolean = false): IPager<IPlatformContent> {
         val result = getSubscriptionsFeedWithExceptions(allowFailure, true);
         if(result.second.any())

app/src/unstable/assets/sources/youtube

@@ -1 +1 @@
-Subproject commit eba1bb5d2ccf438a2f025a204d2d443e4b76b144
+Subproject commit 0a0b84437015cb282fae49e60ae69afe1b45768d