From c70dbb56c8a60720bc728e64b61ba0ec4a3d354b Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 19 Oct 2023 20:05:22 +0200 Subject: [PATCH] Wip ratelimiting subs --- .../java/com/futo/platformplayer/Settings.kt | 8 ++++-- .../api/media/models/ResultCapabilities.kt | 1 + .../media/platforms/js/SourcePluginConfig.kt | 1 + .../platformplayer/models/Subscription.kt | 21 +++++++++++++++ .../futo/platformplayer/states/StateApp.kt | 22 ++++++++++++--- .../platformplayer/states/StatePlatform.kt | 15 +++++++---- .../states/StateSubscriptions.kt | 27 +++++++++++++++++++ app/src/unstable/assets/sources/youtube | 2 +- 8 files changed, 86 insertions(+), 11 deletions(-) diff --git a/app/src/main/java/com/futo/platformplayer/Settings.kt b/app/src/main/java/com/futo/platformplayer/Settings.kt index f8c9c743..ca27483d 100644 --- a/app/src/main/java/com/futo/platformplayer/Settings.kt +++ b/app/src/main/java/com/futo/platformplayer/Settings.kt @@ -140,7 +140,11 @@ class Settings : FragmentedStorageFileJson() { return FeedStyle.THUMBNAIL; } - @FormField("Background Update", FieldForm.DROPDOWN, "Experimental background update for subscriptions cache (requires restart)", 6) + @FormField("Fetch on app boot", FieldForm.TOGGLE, "Shortly after opening the app, start fetching subscriptions.", 6) + @Serializable(with = FlexibleBooleanSerializer::class) + var fetchOnAppBoot: Boolean = true; + + @FormField("Background Update", FieldForm.DROPDOWN, "Experimental background update for subscriptions cache (requires restart)", 7) @DropdownFieldOptionsId(R.array.background_interval) var subscriptionsBackgroundUpdateInterval: Int = 0; @@ -156,7 +160,7 @@ class Settings : FragmentedStorageFileJson() { }; - @FormField("Subscription Concurrency", FieldForm.DROPDOWN, "Specify how many threads are used to fetch channels (requires restart)", 7) + @FormField("Subscription Concurrency", FieldForm.DROPDOWN, "Specify how many threads are used to fetch channels (requires restart)", 8) @DropdownFieldOptionsId(R.array.thread_count) var subscriptionConcurrency: Int = 3; diff --git a/app/src/main/java/com/futo/platformplayer/api/media/models/ResultCapabilities.kt b/app/src/main/java/com/futo/platformplayer/api/media/models/ResultCapabilities.kt index 6fa88c25..dec92d58 100644 --- a/app/src/main/java/com/futo/platformplayer/api/media/models/ResultCapabilities.kt +++ b/app/src/main/java/com/futo/platformplayer/api/media/models/ResultCapabilities.kt @@ -27,6 +27,7 @@ class ResultCapabilities( const val TYPE_VIDEOS = "VIDEOS"; const val TYPE_STREAMS = "STREAMS"; const val TYPE_LIVE = "LIVE"; + const val TYPE_POSTS = "POSTS"; const val TYPE_MIXED = "MIXED"; const val ORDER_CHONOLOGICAL = "CHRONOLOGICAL"; diff --git a/app/src/main/java/com/futo/platformplayer/api/media/platforms/js/SourcePluginConfig.kt b/app/src/main/java/com/futo/platformplayer/api/media/platforms/js/SourcePluginConfig.kt index 5cadbd98..f7331f1d 100644 --- a/app/src/main/java/com/futo/platformplayer/api/media/platforms/js/SourcePluginConfig.kt +++ b/app/src/main/java/com/futo/platformplayer/api/media/platforms/js/SourcePluginConfig.kt @@ -41,6 +41,7 @@ class SourcePluginConfig( val constants: HashMap = hashMapOf(), //TODO: These should be vals...but prob for serialization reasons cannot be changed. + var subscriptionRateLimit: Int? = null, var enableInSearch: Boolean = true, var enableInHome: Boolean = true, var supportedClaimTypes: List = listOf() diff --git a/app/src/main/java/com/futo/platformplayer/models/Subscription.kt b/app/src/main/java/com/futo/platformplayer/models/Subscription.kt index dbf59fd4..0e869f15 100644 --- a/app/src/main/java/com/futo/platformplayer/models/Subscription.kt +++ b/app/src/main/java/com/futo/platformplayer/models/Subscription.kt @@ -3,6 +3,7 @@ package com.futo.platformplayer.models import com.futo.platformplayer.api.media.models.channels.IPlatformChannel import com.futo.platformplayer.api.media.models.channels.SerializedChannel import com.futo.platformplayer.api.media.models.contents.IPlatformContent +import com.futo.platformplayer.getNowDiffDays import com.futo.platformplayer.serializers.OffsetDateTimeSerializer import java.time.OffsetDateTime @@ -10,17 +11,37 @@ import java.time.OffsetDateTime class Subscription { var channel: SerializedChannel; + //Last found content @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class) var lastVideo : OffsetDateTime = OffsetDateTime.MAX; @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class) var lastLiveStream : OffsetDateTime = OffsetDateTime.MAX; + @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class) + var lastPost : OffsetDateTime = OffsetDateTime.MAX; + //Last update time + @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class) + var lastVideoUpdate : OffsetDateTime = OffsetDateTime.MIN; + @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class) + var lastStreamUpdate : OffsetDateTime = OffsetDateTime.MIN; + @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class) + var lastLiveStreamUpdate : OffsetDateTime = OffsetDateTime.MIN; + @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class) + var lastPostUpdate : OffsetDateTime = OffsetDateTime.MIN; + + //Last video interval var uploadInterval : Int = 0; + var uploadPostInterval : Int = 0; + constructor(channel : SerializedChannel) { this.channel = channel; } + fun shouldFetchStreams() = lastLiveStream.getNowDiffDays() < 7; + fun shouldFetchLiveStreams() = lastLiveStream.getNowDiffDays() < 14; + fun shouldFetchPosts() = lastPost.getNowDiffDays() < 2; + fun updateChannel(channel: IPlatformChannel) { this.channel = SerializedChannel.fromChannel(channel); } diff --git a/app/src/main/java/com/futo/platformplayer/states/StateApp.kt b/app/src/main/java/com/futo/platformplayer/states/StateApp.kt index 98ab2cb4..eb3c50bb 100644 --- a/app/src/main/java/com/futo/platformplayer/states/StateApp.kt +++ b/app/src/main/java/com/futo/platformplayer/states/StateApp.kt @@ -28,6 +28,7 @@ import com.futo.platformplayer.R import com.futo.platformplayer.activities.CaptchaActivity import com.futo.platformplayer.activities.IWithResultLauncher import com.futo.platformplayer.activities.MainActivity +import com.futo.platformplayer.api.media.Serializer import com.futo.platformplayer.api.media.platforms.js.JSClient import com.futo.platformplayer.api.media.platforms.js.internal.JSHttpClient import com.futo.platformplayer.background.BackgroundWorker @@ -44,7 +45,10 @@ import com.futo.platformplayer.receivers.AudioNoisyReceiver import com.futo.platformplayer.services.DownloadService import com.futo.platformplayer.stores.FragmentedStorage import com.futo.platformplayer.stores.v2.ManagedStore +import com.stripe.android.core.utils.encodeToJson import kotlinx.coroutines.* +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json import java.io.File import java.time.OffsetDateTime import java.util.* @@ -429,10 +433,22 @@ class StateApp { StatePlaylists.instance.toMigrateCheck() ).flatten(), 0); - scope.launch { - delay(5000); - StateSubscriptions.instance.updateSubscriptionFeed(scope, false); + if(Settings.instance.subscriptions.fetchOnAppBoot) { + val subRequestCounts = StateSubscriptions.instance.getSubscriptionRequestCount(); + val reqCountStr = subRequestCounts.map { " ${it.key.config.name}: ${it.value}/${it.key.config.subscriptionRateLimit}" }.joinToString("\n"); + if (!subRequestCounts.any { clientCount -> + clientCount.key.config.subscriptionRateLimit + ?.let { rateLimit -> clientCount.value > rateLimit } == true + }) { + Logger.w(TAG, "Subscriptions request on boot, request counts:\n${reqCountStr}"); + scope.launch { + delay(5000); + StateSubscriptions.instance.updateSubscriptionFeed(scope, false); + } } + else + Logger.w(TAG, "Too many subscription requests required:\n${reqCountStr}"); + } val interval = Settings.instance.subscriptions.getSubscriptionsBackgroundIntervalMinutes(); scheduleBackgroundWork(context, interval != 0, interval); diff --git a/app/src/main/java/com/futo/platformplayer/states/StatePlatform.kt b/app/src/main/java/com/futo/platformplayer/states/StatePlatform.kt index 31ca3ff6..68ec3cd4 100644 --- a/app/src/main/java/com/futo/platformplayer/states/StatePlatform.kt +++ b/app/src/main/java/com/futo/platformplayer/states/StatePlatform.kt @@ -664,19 +664,24 @@ class StatePlatform { toQuery.add(ResultCapabilities.TYPE_STREAMS); if(clientCapabilities.hasType(ResultCapabilities.TYPE_LIVE)) toQuery.add(ResultCapabilities.TYPE_LIVE); + if(clientCapabilities.hasType(ResultCapabilities.TYPE_POSTS)) + toQuery.add(ResultCapabilities.TYPE_POSTS); if(isSubscriptionOptimized) { val sub = StateSubscriptions.instance.getSubscription(channelUrl); if(sub != null) { - val daysSinceLiveStream = sub.lastLiveStream.getNowDiffDays() - if(daysSinceLiveStream > 7) { - Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 7 days, skipping live streams [${daysSinceLiveStream} days ago]"); + if(!sub.shouldFetchStreams()) { + Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 7 days, skipping live streams [${sub.lastLiveStream.getNowDiffDays()} days ago]"); toQuery.remove(ResultCapabilities.TYPE_LIVE); } - if(daysSinceLiveStream > 14) { - Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 15 days, skipping streams [${daysSinceLiveStream} days ago]"); + if(!sub.shouldFetchLiveStreams()) { + Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 15 days, skipping streams [${sub.lastLiveStream.getNowDiffDays()} days ago]"); toQuery.remove(ResultCapabilities.TYPE_STREAMS); } + if(!sub.shouldFetchPosts()) { + Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 5 days, skipping posts [${sub.lastPost.getNowDiffDays()} days ago]"); + toQuery.remove(ResultCapabilities.TYPE_POSTS); + } } } diff --git a/app/src/main/java/com/futo/platformplayer/states/StateSubscriptions.kt b/app/src/main/java/com/futo/platformplayer/states/StateSubscriptions.kt index 6a244aaf..7721cb60 100644 --- a/app/src/main/java/com/futo/platformplayer/states/StateSubscriptions.kt +++ b/app/src/main/java/com/futo/platformplayer/states/StateSubscriptions.kt @@ -2,6 +2,7 @@ package com.futo.platformplayer.states import com.futo.platformplayer.Settings import com.futo.platformplayer.UIDialogs +import com.futo.platformplayer.api.media.models.ResultCapabilities import com.futo.platformplayer.api.media.models.channels.IPlatformChannel import com.futo.platformplayer.api.media.models.channels.SerializedChannel import com.futo.platformplayer.api.media.models.contents.IPlatformContent @@ -18,6 +19,7 @@ import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException import com.futo.platformplayer.exceptions.ChannelException import com.futo.platformplayer.findNonRuntimeException import com.futo.platformplayer.fragment.mainactivity.main.PolycentricProfile +import com.futo.platformplayer.getNowDiffDays import com.futo.platformplayer.logging.Logger import com.futo.platformplayer.models.Subscription import com.futo.platformplayer.polycentric.PolycentricCache @@ -219,6 +221,31 @@ class StateSubscriptions { } } + fun getSubscriptionRequestCount(): Map { + val subs = getSubscriptions(); + val pluginReqCounts = mutableMapOf(); + + for(sub in subs) { + val client = StatePlatform.instance.getChannelClientOrNull(sub.channel.url); + if(client !is JSClient) + continue; + + val channelCaps = client.getChannelCapabilities(); + if(!pluginReqCounts.containsKey(client)) + pluginReqCounts[client] = 1; + else + pluginReqCounts[client] = pluginReqCounts[client]!! + 1; + + if(channelCaps.hasType(ResultCapabilities.TYPE_STREAMS) && sub.shouldFetchStreams()) + pluginReqCounts[client] = pluginReqCounts[client]!! + 1; + if(channelCaps.hasType(ResultCapabilities.TYPE_LIVE) && sub.shouldFetchLiveStreams()) + pluginReqCounts[client] = pluginReqCounts[client]!! + 1; + if(channelCaps.hasType(ResultCapabilities.TYPE_POSTS) && sub.shouldFetchPosts()) + pluginReqCounts[client] = pluginReqCounts[client]!! + 1; + } + return pluginReqCounts; + } + fun getSubscriptionsFeed(allowFailure: Boolean = false): IPager { val result = getSubscriptionsFeedWithExceptions(allowFailure, true); if(result.second.any()) diff --git a/app/src/unstable/assets/sources/youtube b/app/src/unstable/assets/sources/youtube index eba1bb5d..0a0b8443 160000 --- a/app/src/unstable/assets/sources/youtube +++ b/app/src/unstable/assets/sources/youtube @@ -1 +1 @@ -Subproject commit eba1bb5d2ccf438a2f025a204d2d443e4b76b144 +Subproject commit 0a0b84437015cb282fae49e60ae69afe1b45768d