mirrors/grayjay
1
0
Fork 0
mirror of https://gitlab.futo.org/videostreaming/grayjay.git synced 2025-04-20 03:24:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Wip ratelimiting subs

Browse source
This commit is contained in:
Kelvin 2023-10-19 20:05:22 +02:00
parent f65e293e45
commit c70dbb56c8
8 changed files with 86 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
app/src/main/java/com/futo/platformplayer/Settings.kt
View file

@ -140,7 +140,11 @@ class Settings : FragmentedStorageFileJson() {
return FeedStyle.THUMBNAIL;
}
@FormField("Background Update", FieldForm.DROPDOWN, "Experimental background update for subscriptions cache (requires restart)", 6)
@FormField("Fetch on app boot", FieldForm.TOGGLE, "Shortly after opening the app, start fetching subscriptions.", 6)
@Serializable(with = FlexibleBooleanSerializer::class)
var fetchOnAppBoot: Boolean = true;
@FormField("Background Update", FieldForm.DROPDOWN, "Experimental background update for subscriptions cache (requires restart)", 7)
@DropdownFieldOptionsId(R.array.background_interval)
var subscriptionsBackgroundUpdateInterval: Int = 0;
@ -156,7 +160,7 @@ class Settings : FragmentedStorageFileJson() {
};
@FormField("Subscription Concurrency", FieldForm.DROPDOWN, "Specify how many threads are used to fetch channels (requires restart)", 7)
@FormField("Subscription Concurrency", FieldForm.DROPDOWN, "Specify how many threads are used to fetch channels (requires restart)", 8)
@DropdownFieldOptionsId(R.array.thread_count)
var subscriptionConcurrency: Int = 3;

1
app/src/main/java/com/futo/platformplayer/api/media/models/ResultCapabilities.kt
View file

@ -27,6 +27,7 @@ class ResultCapabilities(
const val TYPE_VIDEOS = "VIDEOS";
const val TYPE_STREAMS = "STREAMS";
const val TYPE_LIVE = "LIVE";
const val TYPE_POSTS = "POSTS";
const val TYPE_MIXED = "MIXED";
const val ORDER_CHONOLOGICAL = "CHRONOLOGICAL";

1
app/src/main/java/com/futo/platformplayer/api/media/platforms/js/SourcePluginConfig.kt
View file

@ -41,6 +41,7 @@ class SourcePluginConfig(
val constants: HashMap<String, String> = hashMapOf(),
//TODO: These should be vals...but prob for serialization reasons cannot be changed.
var subscriptionRateLimit: Int? = null,
var enableInSearch: Boolean = true,
var enableInHome: Boolean = true,
var supportedClaimTypes: List<Int> = listOf()

21
app/src/main/java/com/futo/platformplayer/models/Subscription.kt
View file

@ -3,6 +3,7 @@ package com.futo.platformplayer.models
import com.futo.platformplayer.api.media.models.channels.IPlatformChannel
import com.futo.platformplayer.api.media.models.channels.SerializedChannel
import com.futo.platformplayer.api.media.models.contents.IPlatformContent
import com.futo.platformplayer.getNowDiffDays
import com.futo.platformplayer.serializers.OffsetDateTimeSerializer
import java.time.OffsetDateTime
@ -10,17 +11,37 @@ import java.time.OffsetDateTime
class Subscription {
var channel: SerializedChannel;
//Last found content
@kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
var lastVideo : OffsetDateTime = OffsetDateTime.MAX;
@kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
var lastLiveStream : OffsetDateTime = OffsetDateTime.MAX;
@kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
var lastPost : OffsetDateTime = OffsetDateTime.MAX;
//Last update time
@kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
var lastVideoUpdate : OffsetDateTime = OffsetDateTime.MIN;
@kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
var lastStreamUpdate : OffsetDateTime = OffsetDateTime.MIN;
@kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
var lastLiveStreamUpdate : OffsetDateTime = OffsetDateTime.MIN;
@kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
var lastPostUpdate : OffsetDateTime = OffsetDateTime.MIN;
//Last video interval
var uploadInterval : Int = 0;
var uploadPostInterval : Int = 0;
constructor(channel : SerializedChannel) {
this.channel = channel;
}
fun shouldFetchStreams() = lastLiveStream.getNowDiffDays() < 7;
fun shouldFetchLiveStreams() = lastLiveStream.getNowDiffDays() < 14;
fun shouldFetchPosts() = lastPost.getNowDiffDays() < 2;
fun updateChannel(channel: IPlatformChannel) {
this.channel = SerializedChannel.fromChannel(channel);
}

22
app/src/main/java/com/futo/platformplayer/states/StateApp.kt
View file

@ -28,6 +28,7 @@ import com.futo.platformplayer.R
import com.futo.platformplayer.activities.CaptchaActivity
import com.futo.platformplayer.activities.IWithResultLauncher
import com.futo.platformplayer.activities.MainActivity
import com.futo.platformplayer.api.media.Serializer
import com.futo.platformplayer.api.media.platforms.js.JSClient
import com.futo.platformplayer.api.media.platforms.js.internal.JSHttpClient
import com.futo.platformplayer.background.BackgroundWorker
@ -44,7 +45,10 @@ import com.futo.platformplayer.receivers.AudioNoisyReceiver
import com.futo.platformplayer.services.DownloadService
import com.futo.platformplayer.stores.FragmentedStorage
import com.futo.platformplayer.stores.v2.ManagedStore
import com.stripe.android.core.utils.encodeToJson
import kotlinx.coroutines.*
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import java.io.File
import java.time.OffsetDateTime
import java.util.*
@ -429,10 +433,22 @@ class StateApp {
StatePlaylists.instance.toMigrateCheck()
).flatten(), 0);
scope.launch {
delay(5000);
StateSubscriptions.instance.updateSubscriptionFeed(scope, false);
if(Settings.instance.subscriptions.fetchOnAppBoot) {
val subRequestCounts = StateSubscriptions.instance.getSubscriptionRequestCount();
val reqCountStr = subRequestCounts.map { " ${it.key.config.name}: ${it.value}/${it.key.config.subscriptionRateLimit}" }.joinToString("\n");
if (!subRequestCounts.any { clientCount ->
clientCount.key.config.subscriptionRateLimit
?.let { rateLimit -> clientCount.value > rateLimit } == true
}) {
Logger.w(TAG, "Subscriptions request on boot, request counts:\n${reqCountStr}");
scope.launch {
delay(5000);
StateSubscriptions.instance.updateSubscriptionFeed(scope, false);
}
}
else
Logger.w(TAG, "Too many subscription requests required:\n${reqCountStr}");
}
val interval = Settings.instance.subscriptions.getSubscriptionsBackgroundIntervalMinutes();
scheduleBackgroundWork(context, interval != 0, interval);

15
app/src/main/java/com/futo/platformplayer/states/StatePlatform.kt
View file

@ -664,19 +664,24 @@ class StatePlatform {
toQuery.add(ResultCapabilities.TYPE_STREAMS);
if(clientCapabilities.hasType(ResultCapabilities.TYPE_LIVE))
toQuery.add(ResultCapabilities.TYPE_LIVE);
if(clientCapabilities.hasType(ResultCapabilities.TYPE_POSTS))
toQuery.add(ResultCapabilities.TYPE_POSTS);
if(isSubscriptionOptimized) {
val sub = StateSubscriptions.instance.getSubscription(channelUrl);
if(sub != null) {
val daysSinceLiveStream = sub.lastLiveStream.getNowDiffDays()
if(daysSinceLiveStream > 7) {
Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 7 days, skipping live streams [${daysSinceLiveStream} days ago]");
if(!sub.shouldFetchStreams()) {
Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 7 days, skipping live streams [${sub.lastLiveStream.getNowDiffDays()} days ago]");
toQuery.remove(ResultCapabilities.TYPE_LIVE);
}
if(daysSinceLiveStream > 14) {
Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 15 days, skipping streams [${daysSinceLiveStream} days ago]");
if(!sub.shouldFetchLiveStreams()) {
Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 15 days, skipping streams [${sub.lastLiveStream.getNowDiffDays()} days ago]");
toQuery.remove(ResultCapabilities.TYPE_STREAMS);
}
if(!sub.shouldFetchPosts()) {
Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 5 days, skipping posts [${sub.lastPost.getNowDiffDays()} days ago]");
toQuery.remove(ResultCapabilities.TYPE_POSTS);
}
}
}

27
app/src/main/java/com/futo/platformplayer/states/StateSubscriptions.kt
View file

@ -2,6 +2,7 @@ package com.futo.platformplayer.states
import com.futo.platformplayer.Settings
import com.futo.platformplayer.UIDialogs
import com.futo.platformplayer.api.media.models.ResultCapabilities
import com.futo.platformplayer.api.media.models.channels.IPlatformChannel
import com.futo.platformplayer.api.media.models.channels.SerializedChannel
import com.futo.platformplayer.api.media.models.contents.IPlatformContent
@ -18,6 +19,7 @@ import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
import com.futo.platformplayer.exceptions.ChannelException
import com.futo.platformplayer.findNonRuntimeException
import com.futo.platformplayer.fragment.mainactivity.main.PolycentricProfile
import com.futo.platformplayer.getNowDiffDays
import com.futo.platformplayer.logging.Logger
import com.futo.platformplayer.models.Subscription
import com.futo.platformplayer.polycentric.PolycentricCache
@ -219,6 +221,31 @@ class StateSubscriptions {
}
}
fun getSubscriptionRequestCount(): Map<JSClient, Int> {
val subs = getSubscriptions();
val pluginReqCounts = mutableMapOf<JSClient, Int>();
for(sub in subs) {
val client = StatePlatform.instance.getChannelClientOrNull(sub.channel.url);
if(client !is JSClient)
continue;
val channelCaps = client.getChannelCapabilities();
if(!pluginReqCounts.containsKey(client))
pluginReqCounts[client] = 1;
else
pluginReqCounts[client] = pluginReqCounts[client]!! + 1;
if(channelCaps.hasType(ResultCapabilities.TYPE_STREAMS) && sub.shouldFetchStreams())
pluginReqCounts[client] = pluginReqCounts[client]!! + 1;
if(channelCaps.hasType(ResultCapabilities.TYPE_LIVE) && sub.shouldFetchLiveStreams())
pluginReqCounts[client] = pluginReqCounts[client]!! + 1;
if(channelCaps.hasType(ResultCapabilities.TYPE_POSTS) && sub.shouldFetchPosts())
pluginReqCounts[client] = pluginReqCounts[client]!! + 1;
}
return pluginReqCounts;
}
fun getSubscriptionsFeed(allowFailure: Boolean = false): IPager<IPlatformContent> {
val result = getSubscriptionsFeedWithExceptions(allowFailure, true);
if(result.second.any())

2
app/src/unstable/assets/sources/youtube

@ -1 +1 @@
Subproject commit eba1bb5d2ccf438a2f025a204d2d443e4b76b144
Subproject commit 0a0b84437015cb282fae49e60ae69afe1b45768d