From bdd50d70ca6dcc2ba630397ccfa8f2f20a406a98 Mon Sep 17 00:00:00 2001 From: Koen J Date: Thu, 29 Aug 2024 13:19:24 +0200 Subject: [PATCH 1/5] Download fixes. --- .../platformplayer/downloads/VideoDownload.kt | 78 +++++++++++++++---- .../services/DownloadService.kt | 9 ++- 2 files changed, 71 insertions(+), 16 deletions(-) diff --git a/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt b/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt index 574a2875..457d516d 100644 --- a/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt +++ b/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt @@ -58,6 +58,7 @@ import kotlinx.serialization.Transient import java.io.File import java.io.FileOutputStream import java.io.IOException +import java.lang.Thread.sleep import java.time.OffsetDateTime import java.util.UUID import java.util.concurrent.Executors @@ -156,6 +157,7 @@ class VideoDownload { this.targetBitrate = targetBitrate; this.hasVideoRequestExecutor = video is JSSource && video.hasRequestExecutor; this.requiresLiveVideoSource = false; + this.requiresLiveAudioSource = false; this.targetVideoName = videoSource?.name; } constructor(video: IPlatformVideoDetails, videoSource: IVideoSource?, audioSource: IAudioSource?, subtitleSource: SubtitleRawSource?) { @@ -320,8 +322,15 @@ class VideoDownload { throw DownloadException("Audio source is not supported for downloading (yet)", false); } - if(((!requiresLiveVideoSource && videoSource == null) || (requiresLiveVideoSource && !isLiveVideoSourceValid)) || ((!requiresLiveAudioSource && audioSource == null) || (requiresLiveAudioSource && !isLiveAudioSourceValid))) - throw DownloadException("No valid sources found for video/audio"); + if (requiresLiveVideoSource) { + if (!isLiveVideoSourceValid && !isLiveAudioSourceValid) { + throw DownloadException("No valid sources found for video and audio") + } + } else { + if (videoSource == null && audioSource == null) { + throw DownloadException("No valid sources found for video and audio") + } + } } } @@ -667,14 +676,29 @@ class VideoDownload { if(Settings.instance.downloads.byteRangeDownload && head?.containsKey("accept-ranges") == true && head.containsKey("content-length")) { val concurrency = Settings.instance.downloads.getByteRangeThreadCount(); - Logger.i(TAG, "Download $name ByteRange Parallel (${concurrency})"); + Logger.i(TAG, "Download $name ByteRange Parallel (${concurrency}): " + videoUrl); sourceLength = head["content-length"]!!.toLong(); onProgress(sourceLength, 0, 0); downloadSource_Ranges(name, client, fileStream, videoUrl, sourceLength, 1024*512, concurrency, onProgress); } else { - Logger.i(TAG, "Download $name Sequential"); - sourceLength = downloadSource_Sequential(client, fileStream, videoUrl, onProgress); + if (head == null) { + val rangeResp = client.get(videoUrl, mutableMapOf("Range" to "bytes=0-0")) + if(rangeResp.isOk && rangeResp.headers.containsKey("content-range")) + { + val concurrency = Settings.instance.downloads.getByteRangeThreadCount(); + Logger.i(TAG, "Download $name ByteRange Parallel (${concurrency}): " + videoUrl); + sourceLength = rangeResp.headers["content-range"]?.firstOrNull()?.split('/')?.get(1)?.toLong()!!; + onProgress(sourceLength, 0, 0); + downloadSource_Ranges(name, client, fileStream, videoUrl, sourceLength, 1024*512, concurrency, onProgress); + } else { + Logger.i(TAG, "Download $name Sequential: $videoUrl"); + sourceLength = downloadSource_Sequential(client, fileStream, videoUrl, onProgress); + } + } else { + Logger.i(TAG, "Download $name Sequential: $videoUrl"); + sourceLength = downloadSource_Sequential(client, fileStream, videoUrl, onProgress); + } } Logger.i(TAG, "$name downloadSource Finished"); @@ -823,18 +847,42 @@ class VideoDownload { return tasks.map { it.get() }; } private fun requestByteRange(client: ManagedHttpClient, url: String, rangeStart: Long, rangeEnd: Long): Triple { - val toRead = rangeEnd - rangeStart; - val req = client.get(url, mutableMapOf(Pair("Range", "bytes=${rangeStart}-${rangeEnd}"))); - if(!req.isOk) - throw IllegalStateException("Range request failed Code [${req.code}] due to: ${req.message}"); - if(req.body == null) - throw IllegalStateException("Range request failed, No body"); - val read = req.body.contentLength(); + var retryCount = 0 + var lastException: Throwable? = null - if(read < toRead) - throw IllegalStateException("Byte-Range request attempted to provide less (${read} < ${toRead})"); + while (retryCount <= 3) { + try { + val toRead = rangeEnd - rangeStart; + val req = client.get(url, mutableMapOf(Pair("Range", "bytes=${rangeStart}-${rangeEnd}"))); + if (!req.isOk) { + val bodyString = req.body?.string() + req.body?.close() + throw IllegalStateException("Range request failed Code [${req.code}] due to: ${req.message}"); + } + if (req.body == null) + throw IllegalStateException("Range request failed, No body"); + val read = req.body.contentLength(); - return Triple(req.body.bytes(), rangeStart, rangeEnd); + if (read < toRead) + throw IllegalStateException("Byte-Range request attempted to provide less (${read} < ${toRead})"); + + return Triple(req.body.bytes(), rangeStart, rangeEnd); + } catch (e: Throwable) { + Logger.w(TAG, "Failed to download range (url=${url} bytes=${rangeStart}-${rangeEnd})", e) + + retryCount++ + lastException = e + + sleep(when (retryCount) { + 1 -> 1000 + ((Math.random() * 300.0).toLong() - 150) + 2 -> 2000 + ((Math.random() * 300.0).toLong() - 150) + 3 -> 4000 + ((Math.random() * 300.0).toLong() - 150) + else -> 1000 + ((Math.random() * 300.0).toLong() - 150) + }) + } + } + + throw lastException!! } fun validate() { diff --git a/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt b/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt index 6955cbe9..a0181700 100644 --- a/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt +++ b/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt @@ -28,7 +28,9 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.cancel import kotlinx.coroutines.launch +import okhttp3.OkHttpClient import java.net.SocketException +import java.time.Duration import java.time.OffsetDateTime class DownloadService : Service() { @@ -44,7 +46,12 @@ class DownloadService : Service() { private var _notificationManager: NotificationManager? = null; private var _notificationChannel: NotificationChannel? = null; - private val _client = ManagedHttpClient(); + private val _client = ManagedHttpClient(OkHttpClient.Builder() + //.proxy(Proxy(Proxy.Type.HTTP, InetSocketAddress(InetAddress.getByName("192.168.1.175"), 8081))) + .readTimeout(Duration.ofSeconds(5)) + .writeTimeout(Duration.ofSeconds(5)) + .connectTimeout(Duration.ofSeconds(5)) + .callTimeout(Duration.ofSeconds(5))) private var _started = false; From 5713cf0508bbc774781ebdf9180efc553a6cec15 Mon Sep 17 00:00:00 2001 From: Koen J Date: Thu, 29 Aug 2024 15:41:53 +0200 Subject: [PATCH 2/5] Processed feedback. --- .../platformplayer/downloads/VideoDownload.kt | 100 +++++++++++++----- .../services/DownloadService.kt | 11 +- 2 files changed, 81 insertions(+), 30 deletions(-) diff --git a/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt b/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt index 457d516d..b5e96b11 100644 --- a/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt +++ b/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt @@ -322,15 +322,8 @@ class VideoDownload { throw DownloadException("Audio source is not supported for downloading (yet)", false); } - if (requiresLiveVideoSource) { - if (!isLiveVideoSourceValid && !isLiveAudioSourceValid) { - throw DownloadException("No valid sources found for video and audio") - } - } else { - if (videoSource == null && audioSource == null) { - throw DownloadException("No valid sources found for video and audio") - } - } + if(((!requiresLiveVideoSource && videoSource == null) || (requiresLiveVideoSource && !isLiveVideoSourceValid)) || ((!requiresLiveAudioSource && audioSource == null) || (requiresLiveAudioSource && !isLiveAudioSourceValid))) + throw DownloadException("No valid sources found for video/audio"); } } @@ -682,23 +675,8 @@ class VideoDownload { downloadSource_Ranges(name, client, fileStream, videoUrl, sourceLength, 1024*512, concurrency, onProgress); } else { - if (head == null) { - val rangeResp = client.get(videoUrl, mutableMapOf("Range" to "bytes=0-0")) - if(rangeResp.isOk && rangeResp.headers.containsKey("content-range")) - { - val concurrency = Settings.instance.downloads.getByteRangeThreadCount(); - Logger.i(TAG, "Download $name ByteRange Parallel (${concurrency}): " + videoUrl); - sourceLength = rangeResp.headers["content-range"]?.firstOrNull()?.split('/')?.get(1)?.toLong()!!; - onProgress(sourceLength, 0, 0); - downloadSource_Ranges(name, client, fileStream, videoUrl, sourceLength, 1024*512, concurrency, onProgress); - } else { - Logger.i(TAG, "Download $name Sequential: $videoUrl"); - sourceLength = downloadSource_Sequential(client, fileStream, videoUrl, onProgress); - } - } else { - Logger.i(TAG, "Download $name Sequential: $videoUrl"); - sourceLength = downloadSource_Sequential(client, fileStream, videoUrl, onProgress); - } + Logger.i(TAG, "Download $name Sequential"); + sourceLength = downloadSource_Sequential(client, fileStream, videoUrl, onProgress); } Logger.i(TAG, "$name downloadSource Finished"); @@ -775,6 +753,76 @@ class VideoDownload { onProgress(sourceLength, totalRead, 0); return sourceLength; } + /*private fun downloadSource_Sequential(client: ManagedHttpClient, fileStream: FileOutputStream, url: String, onProgress: (Long, Long, Long) -> Unit): Long { + val progressRate: Int = 4096 * 25 + var lastProgressCount: Int = 0 + val speedRate: Int = 4096 * 25 + var readSinceLastSpeedTest: Long = 0 + var timeSinceLastSpeedTest: Long = System.currentTimeMillis() + + var lastSpeed: Long = 0 + + var totalRead: Long = 0 + var sourceLength: Long + val buffer = ByteArray(4096) + + var isPartialDownload = false + var result: ManagedHttpClient.Response? = null + do { + result = client.get(url, if (isPartialDownload) hashMapOf("Range" to "bytes=$totalRead-") else hashMapOf()) + if (isPartialDownload) { + if (result.code != 206) + throw IllegalStateException("Failed to download source, byte range fallback failed. Web[${result.code}] Error") + } else { + if (!result.isOk) + throw IllegalStateException("Failed to download source. Web[${result.code}] Error") + } + if (result.body == null) + throw IllegalStateException("Failed to download source. Web[${result.code}] No response") + + isPartialDownload = true + sourceLength = result.body!!.contentLength() + val sourceStream = result.body!!.byteStream() + + try { + while (true) { + val read = sourceStream.read(buffer) + if (read <= 0) { + break + } + + fileStream.write(buffer, 0, read) + + totalRead += read + readSinceLastSpeedTest += read + + if (totalRead / progressRate > lastProgressCount) { + onProgress(sourceLength, totalRead, lastSpeed) + lastProgressCount++ + } + if (readSinceLastSpeedTest > speedRate) { + val lastSpeedTime = timeSinceLastSpeedTest + timeSinceLastSpeedTest = System.currentTimeMillis() + val timeSince = timeSinceLastSpeedTest - lastSpeedTime + if (timeSince > 0) + lastSpeed = (readSinceLastSpeedTest / (timeSince / 1000.0)).toLong() + readSinceLastSpeedTest = 0 + } + + if (isCancelled) + throw CancellationException("Cancelled") + } + } catch (e: Throwable) { + Logger.w(TAG, "Sequential download was interrupted, trying to fallback to byte ranges", e) + } finally { + sourceStream.close() + result.body?.close() + } + } while (totalRead < sourceLength) + + onProgress(sourceLength, totalRead, 0) + return sourceLength + }*/ private fun downloadSource_Ranges(name: String, client: ManagedHttpClient, fileStream: FileOutputStream, url: String, sourceLength: Long, rangeSize: Int, concurrency: Int = 1, onProgress: (Long, Long, Long) -> Unit) { val progressRate: Int = 4096 * 5; var lastProgressCount: Int = 0; diff --git a/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt b/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt index a0181700..86f8a07b 100644 --- a/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt +++ b/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt @@ -29,6 +29,9 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.cancel import kotlinx.coroutines.launch import okhttp3.OkHttpClient +import java.net.InetAddress +import java.net.InetSocketAddress +import java.net.Proxy import java.net.SocketException import java.time.Duration import java.time.OffsetDateTime @@ -48,10 +51,10 @@ class DownloadService : Service() { private val _client = ManagedHttpClient(OkHttpClient.Builder() //.proxy(Proxy(Proxy.Type.HTTP, InetSocketAddress(InetAddress.getByName("192.168.1.175"), 8081))) - .readTimeout(Duration.ofSeconds(5)) - .writeTimeout(Duration.ofSeconds(5)) - .connectTimeout(Duration.ofSeconds(5)) - .callTimeout(Duration.ofSeconds(5))) + .readTimeout(Duration.ofSeconds(30)) + .writeTimeout(Duration.ofSeconds(30)) + .connectTimeout(Duration.ofSeconds(30)) + .callTimeout(Duration.ofMinutes(30))) private var _started = false; From 0b4770188c2bce0a47e7b5ba5fa0cbaa96f2235b Mon Sep 17 00:00:00 2001 From: Koen J Date: Thu, 29 Aug 2024 17:57:55 +0200 Subject: [PATCH 3/5] Better error --- .../com/futo/platformplayer/downloads/VideoDownload.kt | 7 ++++++- .../com/futo/platformplayer/services/DownloadService.kt | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt b/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt index 7de33363..1b249a74 100644 --- a/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt +++ b/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt @@ -706,7 +706,12 @@ class VideoDownload { } else { Logger.i(TAG, "Download $name Sequential"); - sourceLength = downloadSource_Sequential(client, fileStream, videoUrl, onProgress); + try { + sourceLength = downloadSource_Sequential(client, fileStream, videoUrl, onProgress); + } catch (e: Throwable) { + Logger.w(TAG, "Failed to download sequentially (url = $videoUrl)") + throw e + } } Logger.i(TAG, "$name downloadSource Finished"); diff --git a/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt b/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt index 39fc5562..e943cecd 100644 --- a/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt +++ b/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt @@ -51,8 +51,8 @@ class DownloadService : Service() { private val _client = ManagedHttpClient(OkHttpClient.Builder() //.proxy(Proxy(Proxy.Type.HTTP, InetSocketAddress(InetAddress.getByName("192.168.1.175"), 8081))) - .readTimeout(Duration.ofSeconds(30)) - .writeTimeout(Duration.ofSeconds(30)) + .readTimeout(Duration.ofMinutes(30)) + .writeTimeout(Duration.ofMinutes(30)) .connectTimeout(Duration.ofSeconds(30)) .callTimeout(Duration.ofMinutes(30))) From 0281da1c5a65f57acd18aad7a86d5e4ff087dfd2 Mon Sep 17 00:00:00 2001 From: Koen J Date: Thu, 29 Aug 2024 18:06:53 +0200 Subject: [PATCH 4/5] Fixed progress for sequential downloads. --- .../java/com/futo/platformplayer/downloads/VideoDownload.kt | 2 +- .../java/com/futo/platformplayer/services/DownloadService.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt b/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt index 1b249a74..f961301e 100644 --- a/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt +++ b/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt @@ -767,7 +767,7 @@ class VideoDownload { totalRead += read; readSinceLastSpeedTest += read; - if (totalRead / progressRate > lastProgressCount) { + if (totalRead.toDouble() / progressRate > lastProgressCount) { onProgress(sourceLength, totalRead, lastSpeed); lastProgressCount++; } diff --git a/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt b/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt index e943cecd..04958a29 100644 --- a/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt +++ b/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt @@ -54,7 +54,7 @@ class DownloadService : Service() { .readTimeout(Duration.ofMinutes(30)) .writeTimeout(Duration.ofMinutes(30)) .connectTimeout(Duration.ofSeconds(30)) - .callTimeout(Duration.ofMinutes(30))) + .callTimeout(Duration.ofMinutes(0))) private var _started = false; From 9ce0be6450ee9053b150d650c78e161e9a5b8ca9 Mon Sep 17 00:00:00 2001 From: Koen J Date: Thu, 29 Aug 2024 18:30:10 +0200 Subject: [PATCH 5/5] Adjusted timeouts --- .../com/futo/platformplayer/services/DownloadService.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt b/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt index 04958a29..b39b4592 100644 --- a/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt +++ b/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt @@ -51,9 +51,9 @@ class DownloadService : Service() { private val _client = ManagedHttpClient(OkHttpClient.Builder() //.proxy(Proxy(Proxy.Type.HTTP, InetSocketAddress(InetAddress.getByName("192.168.1.175"), 8081))) - .readTimeout(Duration.ofMinutes(30)) - .writeTimeout(Duration.ofMinutes(30)) - .connectTimeout(Duration.ofSeconds(30)) + .readTimeout(Duration.ofMinutes(0)) + .writeTimeout(Duration.ofMinutes(0)) + .connectTimeout(Duration.ofSeconds(100)) .callTimeout(Duration.ofMinutes(0))) private var _started = false;