mirrors/grayjay
1
0
Fork 0
mirror of https://gitlab.futo.org/videostreaming/grayjay.git synced 2025-04-20 03:24:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Working DBCache, test plugin

Browse source
This commit is contained in:
Kelvin 2023-11-30 20:58:37 +01:00
parent aeb29c54cd
commit c5541b1747
18 changed files with 266 additions and 254 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
app/src/main/java/com/futo/platformplayer/Settings.kt
View file

@ -8,7 +8,6 @@ import android.webkit.CookieManager
import androidx.lifecycle.lifecycleScope
import com.futo.platformplayer.activities.*
import com.futo.platformplayer.api.http.ManagedHttpClient
import com.futo.platformplayer.cache.ChannelContentCache
import com.futo.platformplayer.constructs.Event0
import com.futo.platformplayer.fragment.mainactivity.bottombar.MenuBottomBarFragment
import com.futo.platformplayer.logging.Logger
@ -276,7 +275,7 @@ class Settings : FragmentedStorageFileJson() {
@FormField(R.string.clear_channel_cache, FieldForm.BUTTON, R.string.clear_channel_cache_description, 14)
fun clearChannelCache() {
UIDialogs.toast(SettingsActivity.getActivity()!!, "Started clearing..");
ChannelContentCache.instance.clear();
StateCache.instance.clear();
UIDialogs.toast(SettingsActivity.getActivity()!!, "Finished clearing");
}
}

74
app/src/main/java/com/futo/platformplayer/SettingsDev.kt
View file

@ -2,6 +2,7 @@ package com.futo.platformplayer
import android.content.Context
import android.webkit.CookieManager
import androidx.lifecycle.lifecycleScope
import androidx.work.Constraints
import androidx.work.Data
import androidx.work.ExistingPeriodicWorkPolicy
@ -20,12 +21,12 @@ import com.futo.platformplayer.api.media.platforms.js.SourcePluginConfig
import com.futo.platformplayer.api.media.platforms.js.SourcePluginDescriptor
import com.futo.platformplayer.api.media.structures.IPager
import com.futo.platformplayer.background.BackgroundWorker
import com.futo.platformplayer.cache.ChannelContentCache
import com.futo.platformplayer.engine.V8Plugin
import com.futo.platformplayer.logging.Logger
import com.futo.platformplayer.serializers.FlexibleBooleanSerializer
import com.futo.platformplayer.states.StateAnnouncement
import com.futo.platformplayer.states.StateApp
import com.futo.platformplayer.states.StateCache
import com.futo.platformplayer.states.StateDeveloper
import com.futo.platformplayer.states.StateDownloads
import com.futo.platformplayer.states.StateSubscriptions
@ -82,26 +83,74 @@ class SettingsDev : FragmentedStorageFileJson() {
var backgroundSubscriptionFetching: Boolean = false;
}
@FormField(R.string.cache, FieldForm.GROUP, -1, 3)
val cache: Cache = Cache();
@Serializable
class Cache {
@FormField(R.string.subscriptions_cache_5000, FieldForm.BUTTON, -1, 1)
fun subscriptionsCache5000() {
StateApp.instance.scope.launch(Dispatchers.IO) {
try {
val subsCache =
StateSubscriptions.instance.getSubscriptionsFeedWithExceptions(cacheScope = this)?.first;
var total = 0;
var page = 0;
var lastToast = System.currentTimeMillis();
while(subsCache!!.hasMorePages() && total < 5000) {
subsCache!!.nextPage();
total += subsCache!!.getResults().size;
page++;
if(page % 10 == 0)
withContext(Dispatchers.Main) {
val diff = System.currentTimeMillis() - lastToast;
lastToast = System.currentTimeMillis();
UIDialogs.toast(
SettingsActivity.getActivity()!!,
"Page: ${page}, Total: ${total}, Speed: ${diff}ms"
);
}
Thread.sleep(250);
}
withContext(Dispatchers.Main) {
UIDialogs.toast(
SettingsActivity.getActivity()!!,
"FINISHED Page: ${page}, Total: ${total}"
);
}
}
catch(ex: Throwable) {
Logger.e("SettingsDev", ex.message, ex);
Logger.i("SettingsDev", "Failed: ${ex.message}");
}
}
}
}
@FormField(R.string.crash_me, FieldForm.BUTTON,
R.string.crashes_the_application_on_purpose, 2)
R.string.crashes_the_application_on_purpose, 3)
fun crashMe() {
throw java.lang.IllegalStateException("This is an uncaught exception triggered on purpose!");
}
@FormField(R.string.delete_announcements, FieldForm.BUTTON,
R.string.delete_all_announcements, 2)
R.string.delete_all_announcements, 3)
fun deleteAnnouncements() {
StateAnnouncement.instance.deleteAllAnnouncements();
}
@FormField(R.string.clear_cookies, FieldForm.BUTTON,
R.string.clear_all_cookies_from_the_cookieManager, 2)
R.string.clear_all_cookies_from_the_cookieManager, 3)
fun clearCookies() {
val cookieManager: CookieManager = CookieManager.getInstance()
cookieManager.removeAllCookies(null);
}
@FormField(R.string.test_background_worker, FieldForm.BUTTON,
R.string.test_background_worker_description, 3)
R.string.test_background_worker_description, 4)
fun triggerBackgroundUpdate() {
val act = SettingsActivity.getActivity()!!;
UIDialogs.toast(SettingsActivity.getActivity()!!, "Starting test background worker");
@ -113,10 +162,10 @@ class SettingsDev : FragmentedStorageFileJson() {
wm.enqueue(req);
}
@FormField(R.string.clear_channel_cache, FieldForm.BUTTON,
R.string.test_background_worker_description, 3)
R.string.test_background_worker_description, 4)
fun clearChannelContentCache() {
UIDialogs.toast(SettingsActivity.getActivity()!!, "Clearing cache");
ChannelContentCache.instance.clearToday();
StateCache.instance.clearToday();
UIDialogs.toast(SettingsActivity.getActivity()!!, "Cleared");
}
@ -363,6 +412,17 @@ class SettingsDev : FragmentedStorageFileJson() {
}
}
@Contextual
@Transient
@FormField(R.string.info, FieldForm.GROUP, -1, 19)
var info = Info();
@Serializable
class Info {
@FormField(R.string.dev_info_channel_cache_size, FieldForm.READONLYTEXT, -1, 1, "channelCacheSize")
var channelCacheStartupCount = StateCache.instance.channelCacheStartupCount;
}
//region BOILERPLATE
override fun encode(): String {
return Json.encodeToString(this);

213
app/src/main/java/com/futo/platformplayer/cache/ChannelContentCache.kt vendored
View file

@ -1,213 +0,0 @@
package com.futo.platformplayer.cache
import com.futo.platformplayer.api.media.models.contents.IPlatformContent
import com.futo.platformplayer.api.media.models.video.SerializedPlatformContent
import com.futo.platformplayer.api.media.structures.DedupContentPager
import com.futo.platformplayer.api.media.structures.IPager
import com.futo.platformplayer.api.media.structures.PlatformContentPager
import com.futo.platformplayer.logging.Logger
import com.futo.platformplayer.polycentric.PolycentricCache
import com.futo.platformplayer.resolveChannelUrl
import com.futo.platformplayer.serializers.PlatformContentSerializer
import com.futo.platformplayer.states.StatePlatform
import com.futo.platformplayer.states.StateSubscriptions
import com.futo.platformplayer.stores.FragmentedStorage
import com.futo.platformplayer.stores.v2.ManagedStore
import com.futo.platformplayer.toSafeFileName
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import java.time.OffsetDateTime
import kotlin.streams.toList
import kotlin.system.measureTimeMillis
class ChannelContentCache {
private val _targetCacheSize = 3000;
val _channelCacheDir = FragmentedStorage.getOrCreateDirectory("channelCache");
val _channelContents: HashMap<String, ManagedStore<SerializedPlatformContent>>;
init {
val allFiles = _channelCacheDir.listFiles() ?: arrayOf();
val initializeTime = measureTimeMillis {
_channelContents = HashMap(allFiles
.filter { it.isDirectory }
.parallelStream().map {
Pair(it.name, FragmentedStorage.storeJson(_channelCacheDir, it.name, PlatformContentSerializer())
.withoutBackup()
.load())
}.toList().associate { it })
}
val minDays = OffsetDateTime.now().minusDays(10);
val totalItems = _channelContents.map { it.value.count() }.sum();
val toTrim = totalItems - _targetCacheSize;
val trimmed: Int;
if(toTrim > 0) {
val redundantContent = _channelContents.flatMap { it.value.getItems().filter { it.datetime != null && it.datetime!!.isBefore(minDays) }.drop(9) }
.sortedBy { it.datetime!! }.take(toTrim);
for(content in redundantContent)
uncacheContent(content);
trimmed = redundantContent.size;
}
else trimmed = 0;
Logger.i(TAG, "ChannelContentCache time: ${initializeTime}ms channels: ${allFiles.size}, videos: ${totalItems}, trimmed: ${trimmed}, total: ${totalItems - trimmed}");
}
fun clear() {
synchronized(_channelContents) {
for(channel in _channelContents)
for(content in channel.value.getItems())
uncacheContent(content);
}
}
fun clearToday() {
val yesterday = OffsetDateTime.now().minusDays(1);
synchronized(_channelContents) {
for(channel in _channelContents)
for(content in channel.value.getItems().filter { it.datetime?.isAfter(yesterday) == true })
uncacheContent(content);
}
}
fun getChannelCachePager(channelUrl: String): PlatformContentPager {
val validID = channelUrl.toSafeFileName();
val validStores = _channelContents
.filter { it.key == validID }
.map { it.value };
val items = validStores.flatMap { it.getItems() }
.sortedByDescending { it.datetime };
return PlatformContentPager(items, Math.min(150, items.size));
}
fun getSubscriptionCachePager(): DedupContentPager {
Logger.i(TAG, "Subscriptions CachePager get subscriptions");
val subs = StateSubscriptions.instance.getSubscriptions();
Logger.i(TAG, "Subscriptions CachePager polycentric urls");
val allUrls = subs.map {
val otherUrls = PolycentricCache.instance.getCachedProfile(it.channel.url)?.profile?.ownedClaims?.mapNotNull { c -> c.claim.resolveChannelUrl() } ?: listOf();
if(!otherUrls.contains(it.channel.url))
return@map listOf(listOf(it.channel.url), otherUrls).flatten();
else
return@map otherUrls;
}.flatten().distinct();
Logger.i(TAG, "Subscriptions CachePager compiling");
val validSubIds = allUrls.map { it.toSafeFileName() }.toHashSet();
val validStores = _channelContents
.filter { validSubIds.contains(it.key) }
.map { it.value };
val items = validStores.flatMap { it.getItems() }
.sortedByDescending { it.datetime };
return DedupContentPager(PlatformContentPager(items, Math.min(30, items.size)), StatePlatform.instance.getEnabledClients().map { it.id });
}
fun uncacheContent(content: SerializedPlatformContent) {
val store = getContentStore(content);
store?.delete(content);
}
fun cacheContents(contents: List<IPlatformContent>): List<IPlatformContent> {
return contents.filter { cacheContent(it) };
}
fun cacheContent(content: IPlatformContent, doUpdate: Boolean = false): Boolean {
if(content.author.url.isEmpty())
return false;
val channelId = content.author.url.toSafeFileName();
val store = getContentStore(channelId).let {
if(it == null) {
Logger.i(TAG, "New Channel Cache for channel ${content.author.name}");
val store = FragmentedStorage.storeJson<SerializedPlatformContent>(_channelCacheDir, channelId, PlatformContentSerializer()).load();
_channelContents.put(channelId, store);
return@let store;
}
else return@let it;
}
val serialized = SerializedPlatformContent.fromContent(content);
val existing = store.findItems { it.url == content.url };
if(existing.isEmpty() || doUpdate) {
if(existing.isNotEmpty())
existing.forEach { store.delete(it) };
store.save(serialized);
}
return existing.isEmpty();
}
private fun getContentStore(content: IPlatformContent): ManagedStore<SerializedPlatformContent>? {
val channelId = content.author.url.toSafeFileName();
return getContentStore(channelId);
}
private fun getContentStore(channelId: String): ManagedStore<SerializedPlatformContent>? {
return synchronized(_channelContents) {
var channelStore = _channelContents.get(channelId);
return@synchronized channelStore;
}
}
companion object {
private val TAG = "ChannelCache";
private val _lock = Object();
private var _instance: ChannelContentCache? = null;
val instance: ChannelContentCache get() {
synchronized(_lock) {
if(_instance == null) {
_instance = ChannelContentCache();
}
}
return _instance!!;
}
fun cachePagerResults(scope: CoroutineScope, pager: IPager<IPlatformContent>, onNewCacheHit: ((IPlatformContent)->Unit)? = null): IPager<IPlatformContent> {
return ChannelVideoCachePager(pager, scope, onNewCacheHit);
}
}
class ChannelVideoCachePager(val pager: IPager<IPlatformContent>, private val scope: CoroutineScope, private val onNewCacheItem: ((IPlatformContent)->Unit)? = null): IPager<IPlatformContent> {
init {
val results = pager.getResults();
Logger.i(TAG, "Caching ${results.size} subscription initial results [${pager.hashCode()}]");
scope.launch(Dispatchers.IO) {
try {
val newCacheItems = instance.cacheContents(results);
if(onNewCacheItem != null)
newCacheItems.forEach { onNewCacheItem!!(it) }
} catch (e: Throwable) {
Logger.e(TAG, "Failed to cache videos.", e);
}
}
}
override fun hasMorePages(): Boolean {
return pager.hasMorePages();
}
override fun nextPage() {
pager.nextPage();
val results = pager.getResults();
Logger.i(TAG, "Caching ${results.size} subscription results");
scope.launch(Dispatchers.IO) {
try {
val newCacheItems = instance.cacheContents(results);
if(onNewCacheItem != null)
newCacheItems.forEach { onNewCacheItem!!(it) }
} catch (e: Throwable) {
Logger.e(TAG, "Failed to cache videos.", e);
}
}
}
override fun getResults(): List<IPlatformContent> {
val results = pager.getResults();
return results;
}
}
}

4
app/src/main/java/com/futo/platformplayer/fragment/channel/tab/ChannelContentsFragment.kt
View file

@ -24,7 +24,6 @@ import com.futo.platformplayer.api.media.structures.IPager
import com.futo.platformplayer.api.media.structures.IRefreshPager
import com.futo.platformplayer.api.media.structures.IReplacerPager
import com.futo.platformplayer.api.media.structures.MultiPager
import com.futo.platformplayer.cache.ChannelContentCache
import com.futo.platformplayer.constructs.Event1
import com.futo.platformplayer.constructs.Event2
import com.futo.platformplayer.constructs.TaskHandler
@ -32,6 +31,7 @@ import com.futo.platformplayer.engine.exceptions.PluginException
import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
import com.futo.platformplayer.fragment.mainactivity.main.FeedView
import com.futo.platformplayer.fragment.mainactivity.main.PolycentricProfile
import com.futo.platformplayer.states.StateCache
import com.futo.platformplayer.states.StatePolycentric
import com.futo.platformplayer.states.StateSubscriptions
import com.futo.platformplayer.views.FeedStyle
@ -78,7 +78,7 @@ class ChannelContentsFragment : Fragment(), IChannelTabFragment {
private val _taskLoadVideos = TaskHandler<IPlatformChannel, IPager<IPlatformContent>>({lifecycleScope}, {
val livePager = getContentPager(it);
return@TaskHandler if(_channel?.let { StateSubscriptions.instance.isSubscribed(it) } == true)
ChannelContentCache.cachePagerResults(lifecycleScope, livePager);
StateCache.cachePagerResults(lifecycleScope, livePager);
else livePager;
}).success { livePager ->
setLoading(false);

7
app/src/main/java/com/futo/platformplayer/fragment/mainactivity/main/FeedView.kt
View file

@ -352,6 +352,7 @@ abstract class FeedView<TFragment, TResult, TConverted, TPager, TViewHolder> : L
}
private fun loadPagerInternal(pager: TPager, cache: ItemCache<TResult>? = null) {
Logger.i(TAG, "Setting new internal pager on feed");
_cache = cache;
detachPagerEvents();
@ -397,6 +398,7 @@ abstract class FeedView<TFragment, TResult, TConverted, TPager, TViewHolder> : L
}
}
var _lastNextPage = false;
private fun loadNextPage() {
synchronized(_pager_lock) {
val pager: TPager = recyclerData.pager ?: return;
@ -405,9 +407,14 @@ abstract class FeedView<TFragment, TResult, TConverted, TPager, TViewHolder> : L
//loadCachedPage();
if (pager.hasMorePages()) {
_lastNextPage = true;
setLoading(true);
_nextPageHandler.run(pager);
}
else if(_lastNextPage) {
Logger.i(TAG, "End of page reached");
_lastNextPage = false;
}
}
}

24
app/src/main/java/com/futo/platformplayer/fragment/mainactivity/main/SubscriptionsFeedFragment.kt
View file

@ -15,13 +15,13 @@ import com.futo.platformplayer.api.media.models.video.IPlatformVideo
import com.futo.platformplayer.api.media.platforms.js.JSClient
import com.futo.platformplayer.api.media.structures.EmptyPager
import com.futo.platformplayer.api.media.structures.IPager
import com.futo.platformplayer.cache.ChannelContentCache
import com.futo.platformplayer.constructs.TaskHandler
import com.futo.platformplayer.engine.exceptions.PluginException
import com.futo.platformplayer.exceptions.ChannelException
import com.futo.platformplayer.exceptions.RateLimitException
import com.futo.platformplayer.logging.Logger
import com.futo.platformplayer.states.StateApp
import com.futo.platformplayer.states.StateCache
import com.futo.platformplayer.states.StatePlatform
import com.futo.platformplayer.states.StateSubscriptions
import com.futo.platformplayer.stores.FragmentedStorage
@ -132,8 +132,10 @@ class SubscriptionsFeedFragment : MainFragment() {
if(StateSubscriptions.instance.getOldestUpdateTime().getNowDiffMinutes() > 5 && Settings.instance.subscriptions.fetchOnTabOpen)
loadResults(false);
else if(recyclerData.results.size == 0)
else if(recyclerData.results.size == 0) {
loadCache();
setLoading(false);
}
}
val announcementsView = _announcementsView;
@ -306,12 +308,18 @@ class SubscriptionsFeedFragment : MainFragment() {
private fun loadCache() {
Logger.i(TAG, "Subscriptions load cache");
val cachePager = ChannelContentCache.instance.getSubscriptionCachePager();
val results = cachePager.getResults();
Logger.i(TAG, "Subscriptions show cache (${results.size})");
setTextCentered(if (results.isEmpty()) context.getString(R.string.no_results_found_swipe_down_to_refresh) else null);
setPager(cachePager);
fragment.lifecycleScope.launch(Dispatchers.IO) {
Logger.i(TAG, "Subscriptions retrieving cache");
val cachePager = StateCache.instance.getSubscriptionCachePager();
Logger.i(TAG, "Subscriptions retrieved cache");
withContext(Dispatchers.Main) {
val results = cachePager.getResults();
Logger.i(TAG, "Subscriptions show cache (${results.size})");
setTextCentered(if (results.isEmpty()) context.getString(R.string.no_results_found_swipe_down_to_refresh) else null);
setPager(cachePager);
}
}
}
private fun loadResults(withRefetch: Boolean = false) {
setLoading(true);

3
app/src/main/java/com/futo/platformplayer/states/StateApp.kt
View file

@ -26,7 +26,6 @@ import com.futo.platformplayer.api.media.models.video.SerializedPlatformVideo
import com.futo.platformplayer.api.media.platforms.js.DevJSClient
import com.futo.platformplayer.api.media.platforms.js.JSClient
import com.futo.platformplayer.background.BackgroundWorker
import com.futo.platformplayer.cache.ChannelContentCache
import com.futo.platformplayer.casting.StateCasting
import com.futo.platformplayer.constructs.Event0
import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
@ -387,7 +386,7 @@ class StateApp {
try {
Logger.i(TAG, "MainApp Started: Initializing [ChannelContentCache]");
val time = measureTimeMillis {
ChannelContentCache.instance;
StateCache.instance;
}
Logger.i(TAG, "ChannelContentCache initialized in ${time}ms");
} catch (e: Throwable) {

81
app/src/main/java/com/futo/platformplayer/states/StateCache.kt
View file

@ -6,7 +6,6 @@ import com.futo.platformplayer.api.media.structures.DedupContentPager
import com.futo.platformplayer.api.media.structures.IPager
import com.futo.platformplayer.api.media.structures.MultiChronoContentPager
import com.futo.platformplayer.api.media.structures.PlatformContentPager
import com.futo.platformplayer.cache.ChannelContentCache
import com.futo.platformplayer.logging.Logger
import com.futo.platformplayer.polycentric.PolycentricCache
import com.futo.platformplayer.resolveChannelUrl
@ -16,12 +15,18 @@ import com.futo.platformplayer.stores.db.ManagedDBStore
import com.futo.platformplayer.stores.db.types.DBChannelCache
import com.futo.platformplayer.stores.db.types.DBHistory
import com.futo.platformplayer.toSafeFileName
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import java.time.OffsetDateTime
import kotlin.system.measureTimeMillis
class StateCache {
private val _channelCache = ManagedDBStore.create("channelCache", DBChannelCache.Descriptor(), PlatformContentSerializer())
.load();
val channelCacheStartupCount = _channelCache.count();
fun clear() {
_channelCache.deleteAll();
}
@ -36,6 +41,12 @@ class StateCache {
it.obj;
}
}
fun getChannelCachePager(channelUrls: List<String>): IPager<IPlatformContent> {
val pagers = MultiChronoContentPager(channelUrls.map { _channelCache.queryPager(DBChannelCache.Index::channelUrl, it, 20) {
it.obj;
} }, false, 20);
return DedupContentPager(pagers, StatePlatform.instance.getEnabledClients().map { it.id });
}
fun getSubscriptionCachePager(): DedupContentPager {
Logger.i(TAG, "Subscriptions CachePager get subscriptions");
val subs = StateSubscriptions.instance.getSubscriptions();
@ -47,10 +58,15 @@ class StateCache {
else
return@map otherUrls;
}.flatten().distinct();
Logger.i(TAG, "Subscriptions CachePager compiling");
val pagers = MultiChronoContentPager(allUrls.map { getChannelCachePager(it) }, false, 20);
return DedupContentPager(pagers, StatePlatform.instance.getEnabledClients().map { it.id });
Logger.i(TAG, "Subscriptions CachePager get pagers");
val pagers = allUrls.parallelStream().map { getChannelCachePager(it) }.toList();
Logger.i(TAG, "Subscriptions CachePager compiling");
val pager = MultiChronoContentPager(pagers, false, 20);
pager.initialize();
Logger.i(TAG, "Subscriptions CachePager compiled");
return DedupContentPager(pager, StatePlatform.instance.getEnabledClients().map { it.id });
}
@ -63,8 +79,8 @@ class StateCache {
if(item != null)
_channelCache.delete(item);
}
fun cacheContents(contents: List<IPlatformContent>): List<IPlatformContent> {
return contents.filter { cacheContent(it) };
fun cacheContents(contents: List<IPlatformContent>, doUpdate: Boolean = false): List<IPlatformContent> {
return contents.filter { cacheContent(it, doUpdate) };
}
fun cacheContent(content: IPlatformContent, doUpdate: Boolean = false): Boolean {
if(content.author.url.isEmpty())
@ -102,5 +118,58 @@ class StateCache {
_instance = null;
}
}
fun cachePagerResults(scope: CoroutineScope, pager: IPager<IPlatformContent>, onNewCacheHit: ((IPlatformContent)->Unit)? = null): IPager<IPlatformContent> {
return ChannelContentCachePager(pager, scope, onNewCacheHit);
}
}
class ChannelContentCachePager(val pager: IPager<IPlatformContent>, private val scope: CoroutineScope, private val onNewCacheItem: ((IPlatformContent)->Unit)? = null): IPager<IPlatformContent> {
init {
val results = pager.getResults();
Logger.i(TAG, "Caching ${results.size} subscription initial results [${pager.hashCode()}]");
scope.launch(Dispatchers.IO) {
try {
val newCacheItems = StateCache.instance.cacheContents(results, true);
if(onNewCacheItem != null)
newCacheItems.forEach { onNewCacheItem!!(it) }
} catch (e: Throwable) {
Logger.e(TAG, "Failed to cache videos.", e);
}
}
}
override fun hasMorePages(): Boolean {
return pager.hasMorePages();
}
override fun nextPage() {
pager.nextPage();
val results = pager.getResults();
scope.launch(Dispatchers.IO) {
try {
val newCacheItemsCount: Int;
val ms = measureTimeMillis {
val newCacheItems = instance.cacheContents(results, true);
newCacheItemsCount = newCacheItems.size;
if(onNewCacheItem != null)
newCacheItems.forEach { onNewCacheItem!!(it) }
}
Logger.i(TAG, "Caching ${results.size} subscription results, updated ${newCacheItemsCount} (${ms}ms)");
} catch (e: Throwable) {
Logger.e(TAG, "Failed to cache ${results.size} videos.", e);
}
}
}
override fun getResults(): List<IPlatformContent> {
val results = pager.getResults();
return results;
}
}
}

1
app/src/main/java/com/futo/platformplayer/states/StateSubscriptions.kt
View file

@ -10,7 +10,6 @@ import com.futo.platformplayer.api.media.platforms.js.JSClient
import com.futo.platformplayer.api.media.platforms.js.SourcePluginConfig
import com.futo.platformplayer.api.media.structures.*
import com.futo.platformplayer.api.media.structures.ReusablePager.Companion.asReusable
import com.futo.platformplayer.cache.ChannelContentCache
import com.futo.platformplayer.constructs.Event0
import com.futo.platformplayer.constructs.Event1
import com.futo.platformplayer.constructs.Event2

2
app/src/main/java/com/futo/platformplayer/stores/db/ManagedDBStore.kt
View file

@ -7,6 +7,7 @@ import androidx.sqlite.db.SimpleSQLiteQuery
import com.futo.platformplayer.api.media.structures.AdhocPager
import com.futo.platformplayer.api.media.structures.IPager
import com.futo.platformplayer.assume
import com.futo.platformplayer.logging.Logger
import com.futo.platformplayer.states.StateApp
import com.futo.platformplayer.stores.v2.JsonStoreSerializer
import com.futo.platformplayer.stores.v2.StoreSerializer
@ -264,6 +265,7 @@ class ManagedDBStore<I: ManagedDBIndex<T>, T, D: ManagedDBDatabase<T, I, DA>, DA
fun queryPager(field: KProperty<*>, obj: Any, pageSize: Int): IPager<I> = queryPager(validateFieldName(field), obj, pageSize);
fun queryPager(field: String, obj: Any, pageSize: Int): IPager<I> {
return AdhocPager({
Logger.i("ManagedDBStore", "Next Page [query: ${obj}](${it}) ${pageSize}");
queryPage(field, obj, it - 1, pageSize);
});
}

11
app/src/main/java/com/futo/platformplayer/stores/db/types/DBChannelCache.kt
View file

@ -3,7 +3,9 @@ package com.futo.platformplayer.stores.db.types
import androidx.room.ColumnInfo
import androidx.room.Dao
import androidx.room.Database
import androidx.room.Entity
import androidx.room.Ignore
import androidx.room.Index
import androidx.room.PrimaryKey
import com.futo.platformplayer.api.media.models.video.SerializedPlatformContent
import com.futo.platformplayer.models.HistoryVideo
@ -25,7 +27,7 @@ class DBChannelCache {
//These classes solely exist for bounding generics for type erasure
@Dao
interface DBDAO: ManagedDBDAOBase<SerializedPlatformContent, Index> {}
@Database(entities = [Index::class], version = 2)
@Database(entities = [Index::class], version = 4)
abstract class DB: ManagedDBDatabase<SerializedPlatformContent, Index, DBDAO>() {
abstract override fun base(): DBDAO;
}
@ -37,6 +39,11 @@ class DBChannelCache {
override fun indexClass(): KClass<Index> = Index::class;
}
@Entity(TABLE_NAME, indices = [
androidx.room.Index(value = ["url"]),
androidx.room.Index(value = ["channelUrl"]),
androidx.room.Index(value = ["datetime"], orders = [androidx.room.Index.Order.DESC])
])
class Index: ManagedDBIndex<SerializedPlatformContent> {
@ColumnIndex
@PrimaryKey(true)
@ -49,7 +56,7 @@ class DBChannelCache {
var channelUrl: String? = null;
@ColumnIndex
@ColumnOrdered(0)
@ColumnOrdered(0, true)
var datetime: Long? = null;

11
app/src/main/java/com/futo/platformplayer/subscription/CachedSubscriptionAlgorithm.kt
View file

@ -5,10 +5,10 @@ import com.futo.platformplayer.api.media.platforms.js.JSClient
import com.futo.platformplayer.api.media.structures.DedupContentPager
import com.futo.platformplayer.api.media.structures.IPager
import com.futo.platformplayer.api.media.structures.PlatformContentPager
import com.futo.platformplayer.cache.ChannelContentCache
import com.futo.platformplayer.models.Subscription
import com.futo.platformplayer.polycentric.PolycentricCache
import com.futo.platformplayer.resolveChannelUrl
import com.futo.platformplayer.states.StateCache
import com.futo.platformplayer.states.StatePlatform
import com.futo.platformplayer.states.StateSubscriptions
import com.futo.platformplayer.toSafeFileName
@ -27,13 +27,16 @@ class CachedSubscriptionAlgorithm(pageSize: Int = 150, scope: CoroutineScope, al
override fun getSubscriptions(subs: Map<Subscription, List<String>>): Result {
val validSubIds = subs.flatMap { it.value } .map { it.toSafeFileName() }.toHashSet();
val validStores = ChannelContentCache.instance._channelContents
/*
val validStores = StateCache.instance._channelContents
.filter { validSubIds.contains(it.key) }
.map { it.value };
.map { it.value };*/
/*
val items = validStores.flatMap { it.getItems() }
.sortedByDescending { it.datetime };
*/
return Result(DedupContentPager(PlatformContentPager(items, Math.min(_pageSize, items.size)), StatePlatform.instance.getEnabledClients().map { it.id }), listOf());
return Result(DedupContentPager(StateCache.instance.getChannelCachePager(subs.flatMap { it.value }.distinct()), StatePlatform.instance.getEnabledClients().map { it.id }), listOf());
}
}

6
app/src/main/java/com/futo/platformplayer/subscription/SimpleSubscriptionAlgorithm.kt
View file

@ -8,7 +8,6 @@ import com.futo.platformplayer.api.media.platforms.js.SourcePluginConfig
import com.futo.platformplayer.api.media.structures.DedupContentPager
import com.futo.platformplayer.api.media.structures.IPager
import com.futo.platformplayer.api.media.structures.MultiChronoContentPager
import com.futo.platformplayer.cache.ChannelContentCache
import com.futo.platformplayer.engine.exceptions.PluginException
import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
import com.futo.platformplayer.engine.exceptions.ScriptCriticalException
@ -17,6 +16,7 @@ import com.futo.platformplayer.findNonRuntimeException
import com.futo.platformplayer.logging.Logger
import com.futo.platformplayer.models.Subscription
import com.futo.platformplayer.polycentric.PolycentricCache
import com.futo.platformplayer.states.StateCache
import com.futo.platformplayer.states.StatePlatform
import com.futo.platformplayer.states.StatePolycentric
import com.futo.platformplayer.states.StateSubscriptions
@ -157,7 +157,7 @@ class SimpleSubscriptionAlgorithm(
val time = measureTimeMillis {
pager = StatePlatform.instance.getChannelContent(platformClient, url, true, threadPool.poolSize, toIgnore);
pager = ChannelContentCache.cachePagerResults(scope, pager!!) {
pager = StateCache.cachePagerResults(scope, pager!!) {
onNewCacheHit.emit(sub, it);
};
@ -176,7 +176,7 @@ class SimpleSubscriptionAlgorithm(
throw channelEx;
else {
Logger.i(StateSubscriptions.TAG, "Channel ${sub.channel.name} failed, substituting with cache");
pager = ChannelContentCache.instance.getChannelCachePager(sub.channel.url);
pager = StateCache.instance.getChannelCachePager(sub.channel.url);
}
}
}

8
app/src/main/java/com/futo/platformplayer/subscription/SubscriptionsTaskFetchAlgorithm.kt
View file

@ -11,7 +11,6 @@ import com.futo.platformplayer.api.media.structures.DedupContentPager
import com.futo.platformplayer.api.media.structures.EmptyPager
import com.futo.platformplayer.api.media.structures.IPager
import com.futo.platformplayer.api.media.structures.MultiChronoContentPager
import com.futo.platformplayer.cache.ChannelContentCache
import com.futo.platformplayer.engine.exceptions.PluginException
import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
import com.futo.platformplayer.engine.exceptions.ScriptCriticalException
@ -21,6 +20,7 @@ import com.futo.platformplayer.fragment.mainactivity.main.SubscriptionsFeedFragm
import com.futo.platformplayer.logging.Logger
import com.futo.platformplayer.models.Subscription
import com.futo.platformplayer.states.StateApp
import com.futo.platformplayer.states.StateCache
import com.futo.platformplayer.states.StatePlatform
import com.futo.platformplayer.states.StateSubscriptions
import kotlinx.coroutines.CoroutineScope
@ -108,7 +108,7 @@ abstract class SubscriptionsTaskFetchAlgorithm(
val sub = if(!entry.value.isEmpty()) entry.value[0].task.sub else null;
val liveTasks = entry.value.filter { !it.task.fromCache };
val cachedTasks = entry.value.filter { it.task.fromCache };
val livePager = if(!liveTasks.isEmpty()) ChannelContentCache.cachePagerResults(scope, MultiChronoContentPager(liveTasks.map { it.pager!! }, true).apply { this.initialize() }, {
val livePager = if(!liveTasks.isEmpty()) StateCache.cachePagerResults(scope, MultiChronoContentPager(liveTasks.map { it.pager!! }, true).apply { this.initialize() }, {
onNewCacheHit.emit(sub!!, it);
}) else null;
val cachedPager = if(!cachedTasks.isEmpty()) MultiChronoContentPager(cachedTasks.map { it.pager!! }, true).apply { this.initialize() } else null;
@ -142,7 +142,7 @@ abstract class SubscriptionsTaskFetchAlgorithm(
return@submit SubscriptionTaskResult(task, null, null);
else {
cachedChannels.add(task.url);
return@submit SubscriptionTaskResult(task, ChannelContentCache.instance.getChannelCachePager(task.url), null);
return@submit SubscriptionTaskResult(task, StateCache.instance.getChannelCachePager(task.url), null);
}
}
}
@ -197,7 +197,7 @@ abstract class SubscriptionsTaskFetchAlgorithm(
throw channelEx;
else {
Logger.i(StateSubscriptions.TAG, "Channel ${task.sub.channel.name} failed, substituting with cache");
pager = ChannelContentCache.instance.getChannelCachePager(task.sub.channel.url);
pager = StateCache.instance.getChannelCachePager(task.sub.channel.url);
taskEx = ex;
return@submit SubscriptionTaskResult(task, pager, taskEx);
}

3
app/src/main/res/values/strings.xml
View file

@ -399,6 +399,7 @@
<string name="version_code">Version Code</string>
<string name="version_name">Version Name</string>
<string name="version_type">Version Type</string>
<string name="dev_info_channel_cache_size">Channel Cache Size (Startup)</string>
<string name="when_watching_a_video_in_preview_mode_resume_at_the_position_when_opening_the_video_code">When watching a video in preview mode, resume at the position when opening the video code</string>
<string name="please_enable_logging_to_submit_logs">Please enable logging to submit logs</string>
<string name="embedded_plugins_reinstalled_a_reboot_is_recommended">Embedded plugins reinstalled, a reboot is recommended</string>
@ -424,6 +425,7 @@
<string name="developer_mode">Developer Mode</string>
<string name="development_server">Development Server</string>
<string name="experimental">Experimental</string>
<string name="cache">Cache</string>
<string name="fill_storage_till_error">Fill storage till error</string>
<string name="inject">Inject</string>
<string name="injects_a_test_source_config_local_into_v8">Injects a test source config (local) into V8</string>
@ -432,6 +434,7 @@
<string name="removes_all_subscriptions">Removes all subscriptions</string>
<string name="settings_related_to_development_server_be_careful_as_it_may_open_your_phone_to_security_vulnerabilities">Settings related to development server, be careful as it may open your phone to security vulnerabilities</string>
<string name="start_server">Start Server</string>
<string name="subscriptions_cache_5000">Subscriptions Cache 5000</string>
<string name="start_server_on_boot">Start Server on boot</string>
<string name="starts_a_devServer_on_port_11337_may_expose_vulnerabilities">Starts a DevServer on port 11337, may expose vulnerabilities.</string>
<string name="test_v8_communication_speed">Test V8 Communication speed</string>

24
app/src/unstable/assets/sources/test/TestConfig.json Normal file
View file

@ -0,0 +1,24 @@
{
"name": "Testing",
"description": "Just for testing.",
"author": "FUTO",
"authorUrl": "https://futo.org",
"platformUrl": "https://odysee.com",
"sourceUrl": "https://plugins.grayjay.app/Test/TestConfig.json",
"repositoryUrl": "https://futo.org",
"scriptUrl": "./TestScript.js",
"version": 31,
"iconUrl": "./odysee.png",
"id": "1c05bfc3-08b9-42d0-93d3-6d52e0fd34d8",
"scriptSignature": "",
"scriptPublicKey": "",
"packages": ["Http"],
"allowEval": false,
"allowUrls": [],
"supportedClaimTypes": []
}

45
app/src/unstable/assets/sources/test/TestScript.js Normal file
View file

@ -0,0 +1,45 @@
var config = {};
//Source Methods
source.enable = function(conf){
config = conf ?? {};
//log(config);
}
source.getHome = function() {
return new ContentPager([
source.getContentDetails("whatever")
]);
};
//Video
source.isContentDetailsUrl = function(url) {
return REGEX_DETAILS_URL.test(url)
};
source.getContentDetails = function(url) {
return new PlatformVideoDetails({
id: new PlatformID("Test", "Something", config.id),
name: "Test Video",
thumbnails: new Thumbnails([]),
author: new PlatformAuthorLink(new PlatformID("Test", "TestID", config.id),
"TestAuthor",
"None",
""),
datetime: parseInt(new Date().getTime() / 1000),
duration: 0,
viewCount: 0,
url: "",
isLive: false,
description: "",
rating: new RatingLikes(0),
video: new VideoSourceDescriptor([
new HLSSource({
name: "HLS",
url: "",
duration: 0,
priority: true
})
])
});
};
log("LOADED");

BIN
app/src/unstable/assets/sources/test/odysee.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 46 KiB