From f083d1b2767554a0f5e760341fff03e3ea9562c8 Mon Sep 17 00:00:00 2001 From: Gabriele Musco Date: Sat, 24 Aug 2024 08:49:11 +0200 Subject: [PATCH] feat: async downloader and file cacher --- Cargo.lock | 72 +++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/downloader.rs | 47 +++++++++++++++++++++++++++---- src/hash.rs | 7 +++++ src/main.rs | 1 + 5 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 src/hash.rs diff --git a/Cargo.lock b/Cargo.lock index e537679..15b56ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -115,6 +115,15 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.14.0" @@ -237,6 +246,35 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "cpufeatures" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dlopen2" version = "0.7.0" @@ -295,6 +333,7 @@ dependencies = [ "rusb", "serde", "serde_json", + "sha2", "tracker", "uuid", "vte4", @@ -544,6 +583,16 @@ dependencies = [ "system-deps", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.10" @@ -1898,6 +1947,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.2.0" @@ -2235,6 +2295,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-bidi" version = "0.3.13" @@ -2301,6 +2367,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "vte4" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index 3032c7d..24ec7b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,3 +28,4 @@ openxr = { git = "https://github.com/galister/openxrs", rev = "af4a55d", feature "linked", ] } ash = "0.38.0" +sha2 = "0.10.8" diff --git a/src/downloader.rs b/src/downloader.rs index 87cb093..90602a6 100644 --- a/src/downloader.rs +++ b/src/downloader.rs @@ -1,10 +1,10 @@ -use crate::{constants::APP_ID, file_utils::get_writer}; +use crate::{constants::APP_ID, file_utils::get_writer, hash::sha256, paths::get_cache_dir}; use reqwest::{ header::{HeaderMap, USER_AGENT}, Method, }; -use std::time::Duration; use std::{io::prelude::*, path::Path}; +use std::{path::PathBuf, time::Duration}; const TIMEOUT: Duration = Duration::from_secs(60); const CHUNK_SIZE: usize = 1024; @@ -15,16 +15,24 @@ fn headers() -> HeaderMap { headers } -fn client() -> reqwest::blocking::Client { +fn sync_client() -> reqwest::blocking::Client { reqwest::blocking::Client::builder() .timeout(TIMEOUT) .default_headers(headers()) .build() - .expect("Failed to build reqwest::Client") + .expect("Failed to build reqwest::blocking::Client") +} + +fn async_client() -> reqwest::Client { + reqwest::Client::builder() + .timeout(TIMEOUT) + .default_headers(headers()) + .build() + .expect("Failed to build request::Client") } pub fn download_file_sync(url: &str, path: &Path) -> Result<(), reqwest::Error> { - let client = client(); + let client = sync_client(); let res = client.request(Method::GET, url).send()?; let status = res.status(); if status.is_client_error() || status.is_server_error() { @@ -41,3 +49,32 @@ pub fn download_file_sync(url: &str, path: &Path) -> Result<(), reqwest::Error> writer.flush().expect("Failed to flush download writer"); Ok(()) } + +pub async fn download_file_async(url: &str, path: &Path) -> Result<(), reqwest::Error> { + let client = async_client(); + let res = client.request(Method::GET, url).send().await?; + let status = res.status(); + if status.is_client_error() || status.is_server_error() { + return Err(res.error_for_status().unwrap_err()); + } + let mut writer = get_writer(path).expect("Unable to write to path"); + for chunk in res + .bytes() + .await + .expect("Could not get HTTP response bytes") + .chunks(CHUNK_SIZE) + { + writer.write_all(chunk).expect("Failed to write chunk"); + } + writer.flush().expect("Failed to flush download writer"); + Ok(()) +} + +pub async fn cache_file(url: &str) -> Result { + let hash = sha256(url); + let dest = get_cache_dir().join(hash); + if !dest.is_file() { + download_file_async(url, &dest).await?; + } + Ok(dest) +} diff --git a/src/hash.rs b/src/hash.rs new file mode 100644 index 0000000..9221b86 --- /dev/null +++ b/src/hash.rs @@ -0,0 +1,7 @@ +use sha2::Digest; + +pub fn sha256(txt: &str) -> String { + let mut hasher = sha2::Sha256::new(); + hasher.update(txt); + format!("{:x}", hasher.finalize()) +} diff --git a/src/main.rs b/src/main.rs index 4f27ff6..602ee12 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,6 +30,7 @@ pub mod env_var_descriptions; pub mod file_builders; pub mod file_utils; pub mod gpu_profile; +pub mod hash; pub mod is_appimage; pub mod linux_distro; pub mod log_level;