DBStore improvements, query like support, more unittest, refactor into StateHistory, history indexes

app/src/androidTest/java/com/futo/platformplayer/ManagedDBStoreTests.kt

@@ -203,35 +203,105 @@ class ManagedDBStoreTests {
     }
     @Test
     fun queryPager() {
-        val store = ManagedDBStore.create("test", Descriptor())
-            .load(context, true);
-        store.deleteAll();
-
         val testStr = UUID.randomUUID().toString();
-
+        testQuery(100, { i, testObject ->
-        val testResults = createSequence(store, 100, { i, testObject ->
             if(i % 2 == 0)
                 testObject.someStr = testStr;
-        });
+        }) {
-        val pager = store.queryPager(DBTOs.TestIndex::someString, testStr, 10);
+            val pager = it.queryPager(DBTOs.TestIndex::someString, testStr, 10);
 
-        val items = pager.getResults().toMutableList();
+            val items = pager.getResults().toMutableList();
-        while(pager.hasMorePages()) {
+            while(pager.hasMorePages()) {
-            pager.nextPage();
+                pager.nextPage();
-            items.addAll(pager.getResults());
+                items.addAll(pager.getResults());
+            }
+            Assert.assertEquals(50, items.size);
+            for(i in 0 until 50) {
+                val k = i * 2;
+                Assert.assertEquals(k, items[i].someNum);
+            }
         }
-        Assert.assertEquals(50, items.size);
-        for(i in 0 until 50) {
-            val k = i * 2;
-            Assert.assertEquals(k, items[i].someNum);
-        }
-
-        store.shutdown();
     }
 
 
 
+    @Test
+    fun queryLike() {
+        val testStr = UUID.randomUUID().toString();
+        val testStrLike = testStr.substring(0, 8) + "Testing" + testStr.substring(8, testStr.length);
+        testQuery(100, { i, testObject ->
+            if(i % 2 == 0)
+                testObject.someStr = testStrLike;
+        }) {
+            val results = it.queryLike(DBTOs.TestIndex::someString, "%Testing%");
 
+            Assert.assertEquals(50, results.size);
+        }
+    }
+    @Test
+    fun queryLikePager() {
+        val testStr = UUID.randomUUID().toString();
+        val testStrLike = testStr.substring(0, 8) + "Testing" + testStr.substring(8, testStr.length);
+        testQuery(100, { i, testObject ->
+            if(i % 2 == 0)
+                testObject.someStr = testStrLike;
+
+        }) {
+            val pager = it.queryLikePager(DBTOs.TestIndex::someString, "%Testing%", 10);
+            val items = pager.getResults().toMutableList();
+            while(pager.hasMorePages()) {
+                pager.nextPage();
+                items.addAll(pager.getResults());
+            }
+            Assert.assertEquals(50, items.size);
+            for(i in 0 until 50) {
+                val k = i * 2;
+                Assert.assertEquals(k, items[i].someNum);
+            }
+        }
+    }
+
+    @Test
+    fun queryGreater() {
+        testQuery(100, { i, testObject ->
+            testObject.someNum = i;
+        }) {
+            val results = it.queryGreater(DBTOs.TestIndex::someNum, 51);
+            Assert.assertEquals(48, results.size);
+        }
+    }
+    @Test
+    fun querySmaller() {
+        testQuery(100, { i, testObject ->
+            testObject.someNum = i;
+        }) {
+            val results = it.querySmaller(DBTOs.TestIndex::someNum, 30);
+            Assert.assertEquals(30, results.size);
+        }
+    }
+    @Test
+    fun queryBetween() {
+        testQuery(100, { i, testObject ->
+            testObject.someNum = i;
+        }) {
+            val results = it.queryBetween(DBTOs.TestIndex::someNum, 30, 65);
+            Assert.assertEquals(34, results.size);
+        }
+    }
+
+
+    private fun testQuery(items: Int, modifier: (Int, DBTOs.TestObject)->Unit, testing: (ManagedDBStore<DBTOs.TestIndex, DBTOs.TestObject, DBTOs.DB, DBTOs.DBDAO>)->Unit) {
+        val store = ManagedDBStore.create("test", Descriptor())
+            .load(context, true);
+        store.deleteAll();
+        createSequence(store, items, modifier);
+        try {
+            testing(store);
+        }
+        finally {
+            store.shutdown();
+        }
+    }
 
 
     private fun createSequence(store: ManagedDBStore<DBTOs.TestIndex, DBTOs.TestObject, DBTOs.DB, DBTOs.DBDAO>, count: Int, modifier: ((Int, DBTOs.TestObject)->Unit)? = null): List<DBTOs.TestIndex> {

app/src/main/java/com/futo/platformplayer/fragment/channel/tab/ChannelContentsFragment.kt

@@ -106,7 +106,7 @@ class ChannelContentsFragment : Fragment(), IChannelTabFragment {
         }
 
         val posBefore = _results.size;
-        val toAdd = it.filter { it is IPlatformVideo }.map { it as IPlatformVideo };
+        val toAdd = it.filter { it is IPlatformVideo }.map { it as IPlatformVideo }
         _results.addAll(toAdd);
         _adapterResults?.let { adapterVideo -> adapterVideo.notifyItemRangeInserted(adapterVideo.childToParentPosition(posBefore), toAdd.size); };
     }.exception<Throwable> {

app/src/main/java/com/futo/platformplayer/fragment/mainactivity/main/HistoryFragment.kt

@@ -12,6 +12,7 @@ import androidx.core.widget.addTextChangedListener
 import androidx.recyclerview.widget.LinearLayoutManager
 import androidx.recyclerview.widget.RecyclerView
 import com.futo.platformplayer.*
+import com.futo.platformplayer.states.StateHistory
 import com.futo.platformplayer.states.StatePlayer
 import com.futo.platformplayer.states.StatePlaylists
 import com.futo.platformplayer.views.others.TagsView
@@ -58,7 +59,7 @@ class HistoryFragment : MainFragment() {
 
         tagsView.onClick.subscribe { timeMinutesToErase ->
             UIDialogs.showConfirmationDialog(requireContext(), getString(R.string.are_you_sure_delete_historical), {
-                StatePlaylists.instance.removeHistoryRange(timeMinutesToErase.second as Long);
+                StateHistory.instance.removeHistoryRange(timeMinutesToErase.second as Long);
                 UIDialogs.toast(view.context, timeMinutesToErase.first + " " + getString(R.string.removed));
                 adapter.updateFilteredVideos();
                 adapter.notifyDataSetChanged();

app/src/main/java/com/futo/platformplayer/fragment/mainactivity/main/VideoDetailView.kt

@@ -778,7 +778,7 @@ class VideoDetailView : ConstraintLayout {
     suspend fun getHistoryIndex(video: IPlatformVideo): DBHistory.Index = withContext(Dispatchers.IO){
         val current = _historyIndex;
         if(current == null || current.url != video.url) {
-            val index = StatePlaylists.instance.getHistoryByVideo(video, true);
+            val index = StateHistory.instance.getHistoryByVideo(video, true)!!;
             _historyIndex = index;
             return@withContext index;
         }
@@ -1290,7 +1290,7 @@ class VideoDetailView : ConstraintLayout {
             val historyItem = getHistoryIndex(videoDetail);
 
             withContext(Dispatchers.Main) {
-                _historicalPosition = StatePlaylists.instance.updateHistoryPosition(video,  historyItem,false, (toResume.toFloat() / 1000.0f).toLong());
+                _historicalPosition = StateHistory.instance.updateHistoryPosition(video,  historyItem,false, (toResume.toFloat() / 1000.0f).toLong());
                 Logger.i(TAG, "Historical position: $_historicalPosition, last position: $lastPositionMilliseconds");
                 if (_historicalPosition > 60 && video.duration - _historicalPosition > 5 && Math.abs(_historicalPosition - lastPositionMilliseconds / 1000) > 5.0) {
                     _layoutResume.visibility = View.VISIBLE;
@@ -2107,7 +2107,7 @@ class VideoDetailView : ConstraintLayout {
         if (updateHistory && (_lastPositionSaveTime == -1L || currentTime - _lastPositionSaveTime > 5000)) {
             fragment.lifecycleScope.launch(Dispatchers.IO) {
                 val history = getHistoryIndex(v);
-                StatePlaylists.instance.updateHistoryPosition(v, history, true, (positionMilliseconds.toFloat() / 1000.0f).toLong());
+                StateHistory.instance.updateHistoryPosition(v, history, true, (positionMilliseconds.toFloat() / 1000.0f).toLong());
             }
             _lastPositionSaveTime = currentTime;
         }

app/src/main/java/com/futo/platformplayer/states/StateApp.kt

@@ -526,10 +526,8 @@ class StateApp {
 
         StatePlaylists.instance.toMigrateCheck();
 
-        StatePlaylists.instance._historyDBStore.deleteAll();
+        if(StateHistory.instance.shouldMigrateLegacyHistory())
-
+            StateHistory.instance.migrateLegacyHistory();
-        if(StatePlaylists.instance.shouldMigrateLegacyHistory())
-            StatePlaylists.instance.migrateLegacyHistory();
 
 
         if(false) {
@@ -544,64 +542,13 @@ class StateApp {
             testHistoryDB(4000);
             Logger.i(TAG, "TEST:--------(6000)---------");
             testHistoryDB(6000);
-            */
             Logger.i(TAG, "TEST:--------(100000)---------");
             scope.launch(Dispatchers.Default) {
-                testHistoryDB(100000);
+                StateHistory.instance.testHistoryDB(100000);
             }
+            */
         }
     }
-    fun testHistoryDB(count: Int) {
-        Logger.i(TAG, "TEST: Starting tests");
-        StatePlaylists.instance._historyDBStore.deleteAll();
-
-        val testHistoryItem = StatePlaylists.instance.getHistoryLegacy().first();
-        val testItemJson = testHistoryItem.video.toJson();
-        val now = OffsetDateTime.now();
-
-        val testSet = (0..count).map { HistoryVideo(Json.decodeFromString<SerializedPlatformVideo>(testItemJson.replace(testHistoryItem.video.url, UUID.randomUUID().toString())), it.toLong(), now.minusHours(it.toLong())) }
-
-
-        Logger.i(TAG, "TEST: Inserting (${testSet.size})");
-        val insertMS = measureTimeMillis {
-            for(item in testSet)
-                StatePlaylists.instance._historyDBStore.insert(item);
-        };
-        Logger.i(TAG, "TEST: Inserting in ${insertMS}ms");
-        /*
-        var fetched: List<DBHistory.Index>? = null;
-        val fetchMS = measureTimeMillis {
-            fetched = StatePlaylists.instance._historyDBStore.getAll();
-            Logger.i(TAG, "TEST: Fetched: ${fetched?.size}");
-        };
-        Logger.i(TAG, "TEST: Fetch speed ${fetchMS}MS");
-        val deserializeMS = measureTimeMillis {
-            val deserialized = StatePlaylists.instance._historyDBStore.convertObjects(fetched!!);
-            Logger.i(TAG, "TEST: Deserialized: ${deserialized.size}");
-        };
-        Logger.i(TAG, "TEST: Deserialize speed ${deserializeMS}MS");
-        */
-        var fetchedIndex: List<DBHistory.Index>? = null;
-        val fetchIndexMS = measureTimeMillis {
-            fetchedIndex = StatePlaylists.instance._historyDBStore.getAllIndexes();
-            Logger.i(TAG, "TEST: Fetched Index: ${fetchedIndex!!.size}");
-        };
-        Logger.i(TAG, "TEST: Fetched Index speed ${fetchIndexMS}ms");
-        val fetchFromIndex = measureTimeMillis {
-            for(preItem in testSet) {
-                val item = StatePlaylists.instance.historyIndex[preItem.video.url];
-                if(item == null)
-                    throw IllegalStateException("Missing item [${preItem.video.url}]");
-                if(item.url != preItem.video.url)
-                    throw IllegalStateException("Mismatch item [${preItem.video.url}]");
-            }
-        };
-        Logger.i(TAG, "TEST: Index Lookup speed ${fetchFromIndex}ms");
-
-        val page1 = StatePlaylists.instance._historyDBStore.getPage(0, 20);
-        val page2 = StatePlaylists.instance._historyDBStore.getPage(1, 20);
-        val page3 = StatePlaylists.instance._historyDBStore.getPage(2, 20);
-    }
 
     fun mainAppStartedWithExternalFiles(context: Context) {
         if(!Settings.instance.didFirstStart) {

app/src/main/java/com/futo/platformplayer/states/StateCache.kt

@@ -5,16 +5,12 @@ import com.futo.platformplayer.api.media.models.video.SerializedPlatformContent
 import com.futo.platformplayer.api.media.structures.DedupContentPager
 import com.futo.platformplayer.api.media.structures.IPager
 import com.futo.platformplayer.api.media.structures.MultiChronoContentPager
-import com.futo.platformplayer.api.media.structures.PlatformContentPager
 import com.futo.platformplayer.logging.Logger
 import com.futo.platformplayer.polycentric.PolycentricCache
 import com.futo.platformplayer.resolveChannelUrl
 import com.futo.platformplayer.serializers.PlatformContentSerializer
-import com.futo.platformplayer.stores.FragmentedStorage
 import com.futo.platformplayer.stores.db.ManagedDBStore
-import com.futo.platformplayer.stores.db.types.DBChannelCache
+import com.futo.platformplayer.stores.db.types.DBSubscriptionCache
-import com.futo.platformplayer.stores.db.types.DBHistory
-import com.futo.platformplayer.toSafeFileName
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
@@ -22,27 +18,27 @@ import java.time.OffsetDateTime
 import kotlin.system.measureTimeMillis
 
 class StateCache {
-    private val _channelCache = ManagedDBStore.create("channelCache", DBChannelCache.Descriptor(), PlatformContentSerializer())
+    private val _subscriptionCache = ManagedDBStore.create("subscriptionCache", DBSubscriptionCache.Descriptor(), PlatformContentSerializer())
         .load();
 
-    val channelCacheStartupCount = _channelCache.count();
+    val channelCacheStartupCount = _subscriptionCache.count();
 
     fun clear() {
-        _channelCache.deleteAll();
+        _subscriptionCache.deleteAll();
     }
     fun clearToday() {
-        val today = _channelCache.queryGreater(DBChannelCache.Index::datetime, OffsetDateTime.now().toEpochSecond());
+        val today = _subscriptionCache.queryGreater(DBSubscriptionCache.Index::datetime, OffsetDateTime.now().toEpochSecond());
         for(content in today)
-            _channelCache.delete(content);
+            _subscriptionCache.delete(content);
     }
 
     fun getChannelCachePager(channelUrl: String): IPager<IPlatformContent> {
-        return _channelCache.queryPager(DBChannelCache.Index::channelUrl, channelUrl, 20) {
+        return _subscriptionCache.queryPager(DBSubscriptionCache.Index::channelUrl, channelUrl, 20) {
             it.obj;
         }
     }
     fun getChannelCachePager(channelUrls: List<String>): IPager<IPlatformContent> {
-        val pagers = MultiChronoContentPager(channelUrls.map { _channelCache.queryPager(DBChannelCache.Index::channelUrl, it, 20) {
+        val pagers = MultiChronoContentPager(channelUrls.map { _subscriptionCache.queryPager(DBSubscriptionCache.Index::channelUrl, it, 20) {
             it.obj;
         } }, false, 20);
         return DedupContentPager(pagers, StatePlatform.instance.getEnabledClients().map { it.id });
@@ -70,14 +66,14 @@ class StateCache {
     }
 
 
-    fun getCachedContent(url: String): DBChannelCache.Index? {
+    fun getCachedContent(url: String): DBSubscriptionCache.Index? {
-        return _channelCache.query(DBChannelCache.Index::url, url).firstOrNull();
+        return _subscriptionCache.query(DBSubscriptionCache.Index::url, url).firstOrNull();
     }
 
     fun uncacheContent(content: SerializedPlatformContent) {
         val item = getCachedContent(content.url);
         if(item != null)
-            _channelCache.delete(item);
+            _subscriptionCache.delete(item);
     }
     fun cacheContents(contents: List<IPlatformContent>, doUpdate: Boolean = false): List<IPlatformContent> {
         return contents.filter { cacheContent(it, doUpdate) };
@@ -90,11 +86,11 @@ class StateCache {
         val existing = getCachedContent(content.url);
 
         if(existing != null && doUpdate) {
-            _channelCache.update(existing.id!!, serialized);
+            _subscriptionCache.update(existing.id!!, serialized);
             return true;
         }
         else if(existing == null) {
-            _channelCache.insert(serialized);
+            _subscriptionCache.insert(serialized);
             return true;
         }
 

app/src/main/java/com/futo/platformplayer/states/StateHistory.kt

@@ -0,0 +1,215 @@
+package com.futo.platformplayer.states
+
+import com.futo.platformplayer.api.media.models.video.IPlatformVideo
+import com.futo.platformplayer.api.media.models.video.SerializedPlatformVideo
+import com.futo.platformplayer.api.media.structures.IPager
+import com.futo.platformplayer.constructs.Event2
+import com.futo.platformplayer.logging.Logger
+import com.futo.platformplayer.models.HistoryVideo
+import com.futo.platformplayer.stores.FragmentedStorage
+import com.futo.platformplayer.stores.db.ManagedDBStore
+import com.futo.platformplayer.stores.db.types.DBHistory
+import com.futo.platformplayer.stores.v2.ReconstructStore
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import java.time.OffsetDateTime
+import java.util.UUID
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.ConcurrentMap
+import kotlin.system.measureTimeMillis
+
+class StateHistory {
+    //Legacy
+    private val _historyStore = FragmentedStorage.storeJson<HistoryVideo>("history")
+        .withRestore(object: ReconstructStore<HistoryVideo>() {
+            override fun toReconstruction(obj: HistoryVideo): String = obj.toReconString();
+            override suspend fun toObject(id: String, backup: String, reconstructionBuilder: Builder): HistoryVideo
+                    = HistoryVideo.fromReconString(backup, null);
+        })
+        .load();
+
+    private val historyIndex: ConcurrentMap<Any, DBHistory.Index> = ConcurrentHashMap();
+    val _historyDBStore = ManagedDBStore.create("history", DBHistory.Descriptor())
+        .withIndex({ it.url }, historyIndex, false, true)
+        .load();
+
+    var onHistoricVideoChanged = Event2<IPlatformVideo, Long>();
+
+    fun shouldMigrateLegacyHistory(): Boolean {
+        return _historyDBStore.count() == 0 && _historyStore.count() > 0;
+    }
+    fun migrateLegacyHistory() {
+        Logger.i(StatePlaylists.TAG, "Migrating legacy history");
+        _historyDBStore.deleteAll();
+        val allHistory = _historyStore.getItems();
+        Logger.i(StatePlaylists.TAG, "Migrating legacy history (${allHistory.size}) items");
+        for(item in allHistory) {
+            _historyDBStore.insert(item);
+        }
+        _historyStore.deleteAll();
+    }
+
+
+    fun getHistoryPosition(url: String): Long {
+        return historyIndex[url]?.position ?: 0;
+    }
+
+
+    fun updateHistoryPosition(liveObj: IPlatformVideo, index: DBHistory.Index, updateExisting: Boolean, position: Long = -1L): Long {
+        val pos = if(position < 0) 0 else position;
+        if(index.obj == null) throw IllegalStateException("Can only update history with a deserialized db item");
+        val historyVideo = index.obj!!;
+
+        val positionBefore = historyVideo.position;
+        if (updateExisting) {
+            var shouldUpdate = false;
+            if (positionBefore < 30) {
+                shouldUpdate = true;
+            } else {
+                if (position > 30) {
+                    shouldUpdate = true;
+                }
+            }
+
+            if (shouldUpdate) {
+
+                //A unrecovered item
+                if(historyVideo.video.author.id.value == null && historyVideo.video.duration == 0L)
+                    historyVideo.video = SerializedPlatformVideo.fromVideo(liveObj);
+
+                historyVideo.position = pos;
+                historyVideo.date = OffsetDateTime.now();
+                _historyDBStore.update(index.id!!, historyVideo);
+                onHistoricVideoChanged.emit(liveObj, pos);
+            }
+
+            return positionBefore;
+        }
+
+        return positionBefore;
+    }
+
+    fun getHistoryLegacy(): List<HistoryVideo> {
+        return _historyStore.getItems();
+    }
+    fun getHistory() : List<HistoryVideo> {
+        return _historyDBStore.getAllObjects();
+        //return _historyStore.getItems().sortedByDescending { it.date };
+    }
+    fun getHistoryPager(): IPager<HistoryVideo> {
+        return _historyDBStore.getObjectPager();
+    }
+    fun getHistorySearchPager(query: String): IPager<HistoryVideo> {
+        return _historyDBStore.queryLikeObjectPager(DBHistory.Index::url, "%${query}%", 10);
+    }
+    fun getHistoryIndexByUrl(url: String): DBHistory.Index? {
+        return historyIndex[url];
+    }
+    fun getHistoryByVideo(video: IPlatformVideo, create: Boolean = false): DBHistory.Index? {
+        val existing = historyIndex[video.url];
+        if(existing != null)
+            return _historyDBStore.get(existing.id!!);
+        else if(create) {
+            val newHistItem = HistoryVideo(SerializedPlatformVideo.fromVideo(video), 0, OffsetDateTime.now());
+            val id = _historyDBStore.insert(newHistItem);
+            return _historyDBStore.get(id);
+        }
+        return null;
+    }
+
+    fun removeHistory(url: String) {
+        val hist = getHistoryIndexByUrl(url);
+        if(hist != null)
+            _historyDBStore.delete(hist.id!!);
+        /*
+        val hist = _historyStore.findItem { it.video.url == url };
+        if(hist != null)
+            _historyStore.delete(hist);*/
+    }
+
+    fun removeHistoryRange(minutesToDelete: Long) {
+        val now = OffsetDateTime.now().toEpochSecond();
+        val toDelete = _historyDBStore.getAllIndexes().filter { minutesToDelete == -1L || (now - it.datetime) < minutesToDelete * 60 };
+        for(item in toDelete)
+            _historyDBStore.delete(item);
+        /*
+        val now = OffsetDateTime.now();
+        val toDelete = _historyStore.findItems { minutesToDelete == -1L || ChronoUnit.MINUTES.between(it.date, now) < minutesToDelete };
+
+        for(item in toDelete)
+            _historyStore.delete(item);*/
+    }
+
+
+    companion object {
+        val TAG = "StateHistory";
+        private var _instance : StateHistory? = null;
+        val instance : StateHistory
+            get(){
+                if(_instance == null)
+                    _instance = StateHistory();
+                return _instance!!;
+            };
+
+        fun finish() {
+            _instance?.let {
+                _instance = null;
+            }
+        }
+    }
+
+
+    fun testHistoryDB(count: Int) {
+        Logger.i(TAG, "TEST: Starting tests");
+        _historyDBStore.deleteAll();
+
+        val testHistoryItem = getHistoryLegacy().first();
+        val testItemJson = testHistoryItem.video.toJson();
+        val now = OffsetDateTime.now();
+
+        val testSet = (0..count).map { HistoryVideo(Json.decodeFromString<SerializedPlatformVideo>(testItemJson.replace(testHistoryItem.video.url, UUID.randomUUID().toString())), it.toLong(), now.minusHours(it.toLong())) }
+
+
+        Logger.i(TAG, "TEST: Inserting (${testSet.size})");
+        val insertMS = measureTimeMillis {
+            for(item in testSet)
+                _historyDBStore.insert(item);
+        };
+        Logger.i(TAG, "TEST: Inserting in ${insertMS}ms");
+
+        var fetched: List<DBHistory.Index>? = null;
+        val fetchMS = measureTimeMillis {
+            fetched = _historyDBStore.getAll();
+            Logger.i(TAG, "TEST: Fetched: ${fetched?.size}");
+        };
+        Logger.i(TAG, "TEST: Fetch speed ${fetchMS}MS");
+        val deserializeMS = measureTimeMillis {
+            val deserialized = _historyDBStore.convertObjects(fetched!!);
+            Logger.i(TAG, "TEST: Deserialized: ${deserialized.size}");
+        };
+        Logger.i(TAG, "TEST: Deserialize speed ${deserializeMS}MS");
+
+        var fetchedIndex: List<DBHistory.Index>? = null;
+        val fetchIndexMS = measureTimeMillis {
+            fetchedIndex = _historyDBStore.getAllIndexes();
+            Logger.i(TAG, "TEST: Fetched Index: ${fetchedIndex!!.size}");
+        };
+        Logger.i(TAG, "TEST: Fetched Index speed ${fetchIndexMS}ms");
+        val fetchFromIndex = measureTimeMillis {
+            for(preItem in testSet) {
+                val item = historyIndex[preItem.video.url];
+                if(item == null)
+                    throw IllegalStateException("Missing item [${preItem.video.url}]");
+                if(item.url != preItem.video.url)
+                    throw IllegalStateException("Mismatch item [${preItem.video.url}]");
+            }
+        };
+        Logger.i(TAG, "TEST: Index Lookup speed ${fetchFromIndex}ms");
+
+        val page1 = _historyDBStore.getPage(0, 20);
+        val page2 = _historyDBStore.getPage(1, 20);
+        val page3 = _historyDBStore.getPage(2, 20);
+    }
+
+
+}

app/src/main/java/com/futo/platformplayer/states/StatePlaylists.kt

@@ -45,165 +45,17 @@ class StatePlaylists {
                 = SerializedPlatformVideo.fromVideo(StatePlatform.instance.getContentDetails(backup).await() as IPlatformVideoDetails);
         })
         .load();
-    private val _historyStore = FragmentedStorage.storeJson<HistoryVideo>("history")
-        .withRestore(object: ReconstructStore<HistoryVideo>() {
-            override fun toReconstruction(obj: HistoryVideo): String = obj.toReconString();
-            override suspend fun toObject(id: String, backup: String, reconstructionBuilder: Builder): HistoryVideo
-                = HistoryVideo.fromReconString(backup, null);
-        })
-        .load();
     val playlistStore = FragmentedStorage.storeJson<Playlist>("playlists")
         .withRestore(PlaylistBackup())
         .load();
 
-    val historyIndex: ConcurrentMap<Any, DBHistory.Index> = ConcurrentHashMap();
-    val _historyDBStore = ManagedDBStore.create("history", DBHistory.Descriptor())
-        .withIndex({ it.url }, historyIndex, false, true)
-        .load();
-
     val playlistShareDir = FragmentedStorage.getOrCreateDirectory("shares");
 
-    var onHistoricVideoChanged = Event2<IPlatformVideo, Long>();
     val onWatchLaterChanged = Event0();
 
     fun toMigrateCheck(): List<ManagedStore<*>> {
-        return listOf(playlistStore, _watchlistStore, _historyStore);
+        return listOf(playlistStore, _watchlistStore);
     }
-
-    fun shouldMigrateLegacyHistory(): Boolean {
-        return _historyDBStore.count() == 0 && _historyStore.count() > 0;
-    }
-    fun migrateLegacyHistory() {
-        Logger.i(TAG, "Migrating legacy history");
-        _historyDBStore.deleteAll();
-        val allHistory = _historyStore.getItems();
-        Logger.i(TAG, "Migrating legacy history (${allHistory.size}) items");
-        for(item in allHistory) {
-            _historyDBStore.insert(item);
-        }
-    }
-
-
-    fun getHistoryPosition(url: String): Long {
-        return historyIndex[url]?.position ?: 0;
-    }
-    fun updateHistoryPosition(liveObj: IPlatformVideo, index: DBHistory.Index, updateExisting: Boolean, position: Long = -1L): Long {
-        val pos = if(position < 0) 0 else position;
-        if(index.obj == null) throw IllegalStateException("Can only update history with a deserialized db item");
-        val historyVideo = index.obj!!;
-
-        val positionBefore = historyVideo.position;
-        if (updateExisting) {
-            var shouldUpdate = false;
-            if (positionBefore < 30) {
-                shouldUpdate = true;
-            } else {
-                if (position > 30) {
-                    shouldUpdate = true;
-                }
-            }
-
-            if (shouldUpdate) {
-
-                //A unrecovered item
-                if(historyVideo.video.author.id.value == null && historyVideo.video.duration == 0L)
-                    historyVideo.video = SerializedPlatformVideo.fromVideo(liveObj);
-
-                historyVideo.position = pos;
-                historyVideo.date = OffsetDateTime.now();
-                _historyDBStore.update(index.id!!, historyVideo);
-                onHistoricVideoChanged.emit(liveObj, pos);
-            }
-
-            return positionBefore;
-        }
-
-        return positionBefore;
-    }
-    /*
-    fun updateHistoryPosition(video: IPlatformVideo, updateExisting: Boolean, position: Long = -1L): Long {
-        val pos = if(position < 0) 0 else position;
-        val historyVideo = _historyStore.findItem { it.video.url == video.url };
-        if (historyVideo != null) {
-            val positionBefore = historyVideo.position;
-            if (updateExisting) {
-                var shouldUpdate = false;
-                if (positionBefore < 30) {
-                    shouldUpdate = true;
-                } else {
-                    if (position > 30) {
-                        shouldUpdate = true;
-                    }
-                }
-
-                if (shouldUpdate) {
-
-                    //A unrecovered item
-                    if(historyVideo.video.author.id.value == null && historyVideo.video.duration == 0L)
-                        historyVideo.video = SerializedPlatformVideo.fromVideo(video);
-
-                    historyVideo.position = pos;
-                    historyVideo.date = OffsetDateTime.now();
-                    _historyStore.saveAsync(historyVideo);
-                    onHistoricVideoChanged.emit(video, pos);
-                }
-            }
-
-            return positionBefore;
-        } else {
-            val newHistItem = HistoryVideo(SerializedPlatformVideo.fromVideo(video), pos, OffsetDateTime.now());
-            _historyStore.saveAsync(newHistItem);
-            return 0;
-        }
-    }
-*/
-    fun getHistoryLegacy(): List<HistoryVideo> {
-        return _historyStore.getItems();
-    }
-    fun getHistory() : List<HistoryVideo> {
-        return _historyDBStore.getAllObjects();
-        //return _historyStore.getItems().sortedByDescending { it.date };
-    }
-    fun getHistoryPager(): IPager<HistoryVideo> {
-        return _historyDBStore.getObjectPager();
-    }
-    fun getHistoryIndexByUrl(url: String): DBHistory.Index? {
-        return historyIndex[url];
-    }
-    fun getHistoryByVideo(video: IPlatformVideo, create: Boolean = false): DBHistory.Index {
-        val existing = historyIndex[video.url];
-        if(existing != null)
-            return _historyDBStore.get(existing.id!!);
-        else {
-            val newHistItem = HistoryVideo(SerializedPlatformVideo.fromVideo(video), 0, OffsetDateTime.now());
-            val id = _historyDBStore.insert(newHistItem);
-            return _historyDBStore.get(id);
-        }
-    }
-
-    fun removeHistory(url: String) {
-        val hist = getHistoryIndexByUrl(url);
-        if(hist != null)
-            _historyDBStore.delete(hist.id!!);
-        /*
-        val hist = _historyStore.findItem { it.video.url == url };
-        if(hist != null)
-            _historyStore.delete(hist);*/
-    }
-
-    fun removeHistoryRange(minutesToDelete: Long) {
-        val now = OffsetDateTime.now().toEpochSecond();
-        val toDelete = _historyDBStore.getAllIndexes().filter { minutesToDelete == -1L || (now - it.date) < minutesToDelete * 60 };
-        for(item in toDelete)
-            _historyDBStore.delete(item);
-        /*
-        val now = OffsetDateTime.now();
-        val toDelete = _historyStore.findItems { minutesToDelete == -1L || ChronoUnit.MINUTES.between(it.date, now) < minutesToDelete };
-
-        for(item in toDelete)
-            _historyStore.delete(item);*/
-    }
-
     fun getWatchLater() : List<SerializedPlatformVideo> {
         synchronized(_watchlistStore) {
             return _watchlistStore.getItems();

app/src/main/java/com/futo/platformplayer/stores/db/ManagedDBStore.kt

@@ -232,6 +232,12 @@ class ManagedDBStore<I: ManagedDBIndex<T>, T, D: ManagedDBDatabase<T, I, DA>, DA
         val query = SimpleSQLiteQuery(queryStr, arrayOf(obj));
         return deserializeIndexes(dbDaoBase.getMultiple(query));
     }
+    fun queryLike(field: KProperty<*>, obj: String): List<I> = queryLike(validateFieldName(field), obj);
+    fun queryLike(field: String, obj: String): List<I> {
+        val queryStr = "SELECT * FROM ${descriptor.table_name} WHERE ${field} LIKE ?";
+        val query = SimpleSQLiteQuery(queryStr, arrayOf(obj));
+        return deserializeIndexes(dbDaoBase.getMultiple(query));
+    }
     fun queryGreater(field: KProperty<*>, obj: Any): List<I> = queryGreater(validateFieldName(field), obj);
     fun queryGreater(field: String, obj: Any): List<I> {
         val queryStr = "SELECT * FROM ${descriptor.table_name} WHERE ${field} > ?";
@@ -251,17 +257,30 @@ class ManagedDBStore<I: ManagedDBIndex<T>, T, D: ManagedDBDatabase<T, I, DA>, DA
         return deserializeIndexes(dbDaoBase.getMultiple(query));
     }
 
-
+    //Query Pages
+    fun queryPage(field: KProperty<*>, obj: Any, page: Int, pageSize: Int): List<I> = queryPage(validateFieldName(field), obj, page, pageSize);
     fun queryPage(field: String, obj: Any, page: Int, pageSize: Int): List<I> {
         val queryStr = "SELECT * FROM ${descriptor.table_name} WHERE ${field} = ? ${_orderSQL} LIMIT ? OFFSET ?";
         val query = SimpleSQLiteQuery(queryStr, arrayOf(obj, pageSize, page * pageSize));
         return deserializeIndexes(dbDaoBase.getMultiple(query));
     }
-    fun queryPage(field: KProperty<*>, obj: Any, page: Int, pageSize: Int): List<I> = queryPage(validateFieldName(field), obj, page, pageSize);
 
+    fun queryLikePage(field: KProperty<*>, obj: String, page: Int, pageSize: Int): List<I> = queryLikePage(validateFieldName(field), obj, page, pageSize);
+    fun queryLikePage(field: String, obj: String, page: Int, pageSize: Int): List<I> {
+        val queryStr = "SELECT * FROM ${descriptor.table_name} WHERE ${field} LIKE ? ${_orderSQL} LIMIT ? OFFSET ?";
+        val query = SimpleSQLiteQuery(queryStr, arrayOf(obj, pageSize, page * pageSize));
+        return deserializeIndexes(dbDaoBase.getMultiple(query));
+    }
+    fun queryLikeObjectPage(field: String, obj: String, page: Int, pageSize: Int): List<T> {
+        return convertObjects(queryLikePage(field, obj, page, pageSize));
+    }
+
+
+    //Query Page Objects
     fun queryPageObjects(field: String, obj: Any, page: Int, pageSize: Int): List<T> = convertObjects(queryPage(field, obj, page, pageSize));
     fun queryPageObjects(field: KProperty<*>, obj: Any, page: Int, pageSize: Int): List<T> = queryPageObjects(validateFieldName(field), obj, page, pageSize);
 
+    //Query Pager
     fun queryPager(field: KProperty<*>, obj: Any, pageSize: Int): IPager<I> = queryPager(validateFieldName(field), obj, pageSize);
     fun queryPager(field: String, obj: Any, pageSize: Int): IPager<I> {
         return AdhocPager({
@@ -269,6 +288,23 @@ class ManagedDBStore<I: ManagedDBIndex<T>, T, D: ManagedDBDatabase<T, I, DA>, DA
            queryPage(field, obj, it - 1, pageSize);
         });
     }
+
+    fun queryLikePager(field: KProperty<*>, obj: String, pageSize: Int): IPager<I> = queryLikePager(validateFieldName(field), obj, pageSize);
+    fun queryLikePager(field: String, obj: String, pageSize: Int): IPager<I> {
+        return AdhocPager({
+            Logger.i("ManagedDBStore", "Next Page [query: ${obj}](${it}) ${pageSize}");
+            queryLikePage(field, obj, it - 1, pageSize);
+        });
+    }
+    fun queryLikeObjectPager(field: KProperty<*>, obj: String, pageSize: Int): IPager<T> = queryLikeObjectPager(validateFieldName(field), obj, pageSize);
+    fun queryLikeObjectPager(field: String, obj: String, pageSize: Int): IPager<T> {
+        return AdhocPager({
+            Logger.i("ManagedDBStore", "Next Page [query: ${obj}](${it}) ${pageSize}");
+            queryLikeObjectPage(field, obj, it - 1, pageSize);
+        });
+    }
+
+    //Query Pager with convert
     fun <X> queryPager(field: KProperty<*>, obj: Any, pageSize: Int, convert: (I)->X): IPager<X> = queryPager(validateFieldName(field), obj, pageSize, convert);
     fun <X> queryPager(field: String, obj: Any, pageSize: Int, convert: (I)->X): IPager<X> {
         return AdhocPager({
@@ -276,6 +312,7 @@ class ManagedDBStore<I: ManagedDBIndex<T>, T, D: ManagedDBDatabase<T, I, DA>, DA
         });
     }
 
+    //Query Object Pager
     fun queryObjectPager(field: KProperty<*>, obj: Any, pageSize: Int): IPager<T> = queryObjectPager(validateFieldName(field), obj, pageSize);
     fun queryObjectPager(field: String, obj: Any, pageSize: Int): IPager<T> {
         return AdhocPager({
@@ -283,6 +320,7 @@ class ManagedDBStore<I: ManagedDBIndex<T>, T, D: ManagedDBDatabase<T, I, DA>, DA
         });
     }
 
+    //Page
     fun getPage(page: Int, length: Int): List<I> {
         if(_sqlPage == null)
             throw IllegalStateException("DB Store [${name}] does not have ordered fields to provide pages");

app/src/main/java/com/futo/platformplayer/stores/db/types/DBHistory.kt

@@ -28,7 +28,7 @@ class DBHistory {
     //These classes solely exist for bounding generics for type erasure
     @Dao
     interface DBDAO: ManagedDBDAOBase<HistoryVideo, Index> {}
-    @Database(entities = [Index::class], version = 2)
+    @Database(entities = [Index::class], version = 3)
     abstract class DB: ManagedDBDatabase<HistoryVideo, Index, DBDAO>() {
         abstract override fun base(): DBDAO;
     }
@@ -40,32 +40,34 @@ class DBHistory {
         override fun indexClass(): KClass<Index> = Index::class;
     }
 
-    @Entity(TABLE_NAME)
+    @Entity(TABLE_NAME, indices = [
-    class Index: ManagedDBIndex<HistoryVideo> {
+        androidx.room.Index(value = ["url"]),
+        androidx.room.Index(value = ["name"]),
+        androidx.room.Index(value = ["datetime"], orders = [androidx.room.Index.Order.DESC])
+    ])
+    class Index(): ManagedDBIndex<HistoryVideo>() {
         @PrimaryKey(true)
         @ColumnOrdered(1)
         @ColumnIndex
         override var id: Long? = null;
 
         @ColumnIndex
-        var url: String;
+        var url: String = "";
         @ColumnIndex
-        var position: Long;
+        var position: Long = 0;
         @ColumnIndex
         @ColumnOrdered(0, true)
-        var date: Long;
+        var datetime: Long = 0;
+        @ColumnIndex
+        var name: String = "";
 
-        constructor() {
+        constructor(historyVideo: HistoryVideo) : this() {
-            url = "";
-            position = 0;
-            date = 0;
-        }
-        constructor(historyVideo: HistoryVideo) {
             id = null;
             serialized = null;
             url = historyVideo.video.url;
             position = historyVideo.position;
-            date = historyVideo.date.toEpochSecond();
+            datetime = historyVideo.date.toEpochSecond();
+            name = historyVideo.video.name;
         }
     }
 }

app/src/main/java/com/futo/platformplayer/stores/db/types/DBSubscriptionCache.kt

@@ -1,33 +1,28 @@
 package com.futo.platformplayer.stores.db.types
 
-import androidx.room.ColumnInfo
 import androidx.room.Dao
 import androidx.room.Database
 import androidx.room.Entity
-import androidx.room.Ignore
-import androidx.room.Index
 import androidx.room.PrimaryKey
 import com.futo.platformplayer.api.media.models.video.SerializedPlatformContent
-import com.futo.platformplayer.models.HistoryVideo
 import com.futo.platformplayer.stores.db.ColumnIndex
 import com.futo.platformplayer.stores.db.ColumnOrdered
 import com.futo.platformplayer.stores.db.ManagedDBDAOBase
 import com.futo.platformplayer.stores.db.ManagedDBDatabase
 import com.futo.platformplayer.stores.db.ManagedDBDescriptor
 import com.futo.platformplayer.stores.db.ManagedDBIndex
-import java.time.OffsetDateTime
 import kotlin.reflect.KClass
 
-class DBChannelCache {
+class DBSubscriptionCache {
     companion object {
-        const val TABLE_NAME = "feed_cache";
+        const val TABLE_NAME = "subscription_cache";
     }
 
 
     //These classes solely exist for bounding generics for type erasure
     @Dao
     interface DBDAO: ManagedDBDAOBase<SerializedPlatformContent, Index> {}
-    @Database(entities = [Index::class], version = 4)
+    @Database(entities = [Index::class], version = 5)
     abstract class DB: ManagedDBDatabase<SerializedPlatformContent, Index, DBDAO>() {
         abstract override fun base(): DBDAO;
     }

app/src/main/java/com/futo/platformplayer/testing/DBTOs.kt

@@ -15,7 +15,7 @@ import java.util.UUID
 class DBTOs {
     @Dao
     interface DBDAO: ManagedDBDAOBase<TestObject, TestIndex> {}
-    @Database(entities = [TestIndex::class], version = 2)
+    @Database(entities = [TestIndex::class], version = 3)
     abstract class DB: ManagedDBDatabase<TestObject, TestIndex, DBDAO>() {
         abstract override fun base(): DBDAO;
     }

app/src/main/java/com/futo/platformplayer/views/adapters/HistoryListAdapter.kt

@@ -8,6 +8,7 @@ import com.futo.platformplayer.api.media.structures.IPager
 import com.futo.platformplayer.constructs.Event1
 import com.futo.platformplayer.models.HistoryVideo
 import com.futo.platformplayer.states.StateApp
+import com.futo.platformplayer.states.StateHistory
 import com.futo.platformplayer.states.StatePlaylists
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.coroutineScope
@@ -22,7 +23,7 @@ class HistoryListAdapter : RecyclerView.Adapter<HistoryListViewHolder> {
     constructor() : super() {
         updateFilteredVideos();
 
-        StatePlaylists.instance.onHistoricVideoChanged.subscribe(this) { video, position ->
+        StateHistory.instance.onHistoricVideoChanged.subscribe(this) { video, position ->
             StateApp.instance.scope.launch(Dispatchers.Main) {
                 val index = _filteredVideos.indexOfFirst { v -> v.video.url == video.url };
                 if (index == -1) {
@@ -45,7 +46,9 @@ class HistoryListAdapter : RecyclerView.Adapter<HistoryListViewHolder> {
     }
 
     fun updateFilteredVideos() {
-        val videos = StatePlaylists.instance.getHistory();
+        val videos = StateHistory.instance.getHistory();
+        val pager = StateHistory.instance.getHistoryPager();
+        //filtered val pager = StateHistory.instance.getHistorySearchPager("querrryyyyy");
 
         if (_query.isBlank()) {
             _filteredVideos = videos.toMutableList();
@@ -57,7 +60,7 @@ class HistoryListAdapter : RecyclerView.Adapter<HistoryListViewHolder> {
     }
 
     fun cleanup() {
-        StatePlaylists.instance.onHistoricVideoChanged.remove(this);
+        StateHistory.instance.onHistoricVideoChanged.remove(this);
     }
 
     override fun getItemCount() = _filteredVideos.size;
@@ -73,7 +76,7 @@ class HistoryListAdapter : RecyclerView.Adapter<HistoryListViewHolder> {
                 return@subscribe;
             }
 
-            StatePlaylists.instance.removeHistory(v.video.url);
+            StateHistory.instance.removeHistory(v.video.url);
             _filteredVideos.removeAt(index);
             notifyItemRemoved(index);
         };

app/src/main/java/com/futo/platformplayer/views/adapters/feedtypes/PreviewVideoView.kt

@@ -27,6 +27,7 @@ import com.futo.platformplayer.logging.Logger
 import com.futo.platformplayer.polycentric.PolycentricCache
 import com.futo.platformplayer.states.StateApp
 import com.futo.platformplayer.states.StateDownloads
+import com.futo.platformplayer.states.StateHistory
 import com.futo.platformplayer.states.StatePlaylists
 import com.futo.platformplayer.video.PlayerManager
 import com.futo.platformplayer.views.others.CreatorThumbnail
@@ -246,7 +247,7 @@ open class PreviewVideoView : LinearLayout {
             val timeBar = _timeBar
             if (timeBar != null) {
                 if (shouldShowTimeBar) {
-                    val historyPosition = StatePlaylists.instance.getHistoryPosition(video.url)
+                    val historyPosition = StateHistory.instance.getHistoryPosition(video.url)
                     timeBar.visibility = if (historyPosition > 0) VISIBLE else GONE
                     timeBar.progress = historyPosition.toFloat() / video.duration.toFloat()
                 } else {