From 63c16b817fae67109f2f2931bb9ab3962107e98f Mon Sep 17 00:00:00 2001 From: Andrzej Janik Date: Sun, 6 Apr 2025 21:32:56 +0000 Subject: [PATCH] Build tar.gz on Linux --- Cargo.lock | 254 ++++++++++++++++++++++++++++++++++++++++++++-- xtask/Cargo.toml | 4 + xtask/src/main.rs | 211 ++++++++++++++++++++++++-------------- 3 files changed, 384 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5726bb3..5eb237b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -21,6 +27,15 @@ version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "argh" version = "0.1.12" @@ -122,6 +137,12 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + [[package]] name = "byteorder" version = "1.5.0" @@ -220,6 +241,21 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "cuda-config" version = "0.1.0" @@ -256,6 +292,15 @@ dependencies = [ "hip_runtime-sys", ] +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", +] + [[package]] name = "derivative" version = "2.2.0" @@ -267,6 +312,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "derive_more" version = "1.0.0" @@ -342,12 +398,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -356,12 +412,35 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + [[package]] name = "fixedbitset" version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flate2" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +dependencies = [ + "crc32fast", + "libz-rs-sys", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -453,9 +532,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.159" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libloading" @@ -467,12 +546,38 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.6.0", + "libc", + "redox_syscall", +] + +[[package]] +name = "libz-rs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "linux-raw-sys" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" + [[package]] name = "llvm-sys" version = "170.2.0" @@ -497,6 +602,12 @@ dependencies = [ "llvm-sys", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.22" @@ -587,6 +698,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff70ce3e48ae43fa075863cef62e8b43b71a4f2382229920e0df362592919430" +dependencies = [ + "adler2", +] + [[package]] name = "ndarray" version = "0.16.1" @@ -621,6 +741,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.46" @@ -707,6 +833,12 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "pretty_assertions" version = "1.4.1" @@ -871,6 +1003,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" +[[package]] +name = "redox_syscall" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" +dependencies = [ + "bitflags 2.6.0", +] + [[package]] name = "regex" version = "1.11.0" @@ -927,10 +1068,23 @@ dependencies = [ "bitflags 2.6.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.4.14", "windows-sys 0.52.0", ] +[[package]] +name = "rustix" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys 0.9.3", + "windows-sys 0.59.0", +] + [[package]] name = "rustversion" version = "1.0.17" @@ -1010,6 +1164,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "smallvec" version = "1.13.2" @@ -1069,6 +1229,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.13.0" @@ -1078,7 +1249,7 @@ dependencies = [ "cfg-if", "fastrand", "once_cell", - "rustix", + "rustix 0.38.37", "windows-sys 0.59.0", ] @@ -1122,6 +1293,25 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "time" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" + [[package]] name = "toml" version = "0.5.11" @@ -1296,14 +1486,28 @@ dependencies = [ "memchr", ] +[[package]] +name = "xattr" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" +dependencies = [ + "libc", + "rustix 1.0.5", +] + [[package]] name = "xtask" version = "0.0.0" dependencies = [ "bpaf", "cargo_metadata", + "flate2", "serde", "serde_json", + "tar", + "time", + "zip", ] [[package]] @@ -1312,6 +1516,28 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +[[package]] +name = "zip" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dcb24d0152526ae49b9b96c1dcf71850ca1e0b882e4e28ed898a93c41334744" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "flate2", + "indexmap", + "memchr", + "time", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" + [[package]] name = "zluda" version = "0.0.0" @@ -1393,3 +1619,17 @@ dependencies = [ "wchar", "winapi", ] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index cc4d2d2..24da039 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -10,6 +10,10 @@ cargo_metadata = "0.19.1" serde = "1.0.217" serde_json = "1.0.137" +[target.'cfg(unix)'.dependencies] +flate2 = { version = "1.1.1", features = ["zlib-rs"], default-features = false } +tar = "0.4.44" + [target.'cfg(windows)'.dependencies] zip = { version = "2.6.1", features = ["deflate", "time"], default-features = false } time = "0.3.41" diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 134be4c..e32c463 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,8 +1,7 @@ use bpaf::{Args, Bpaf, Parser}; use cargo_metadata::{MetadataCommand, Package}; use serde::Deserialize; -use std::{env, ffi::OsString, fs::File, path::PathBuf, process::Command}; -use zip::{write::SimpleFileOptions, ZipWriter}; +use std::{env, ffi::OsString, path::PathBuf, process::Command}; #[derive(Debug, Clone, Bpaf)] #[bpaf(options)] @@ -79,9 +78,9 @@ impl Project { #[cfg(unix)] fn prefix(&self) -> &'static str { - match self.clib_name { - None => "", - Some(_) => "lib", + match self.target_kind { + ProjectTarget::Bin => "", + ProjectTarget::Cdylib => "lib", } } @@ -100,6 +99,40 @@ impl Project { ProjectTarget::Cdylib => ".dll", } } + + // Returns tuple: + // * symlink file path (relative to the root of build dir) + // * symlink absolute file path + // * target actual file (relative to symlink file) + fn symlinks<'a>( + &'a self, + target_dir: &'a PathBuf, + profile: &'a str, + libname: &'a str, + ) -> impl Iterator + 'a { + self.meta.linux_symlinks.iter().map(move |source| { + let mut link = target_dir.clone(); + link.extend([profile, source]); + let relative_link = PathBuf::from(source); + let ancestors = relative_link.as_path().ancestors().count(); + let mut target = std::iter::repeat_with(|| "../").take(ancestors - 2).fold( + PathBuf::new(), + |mut buff, segment| { + buff.push(segment); + buff + }, + ); + target.push(libname); + (&**source, link, target) + }) + } + + fn file_name(&self) -> String { + let target_name = &self.target_name; + let prefix = self.prefix(); + let suffix = self.suffix(); + format!("{prefix}{target_name}{suffix}") + } } #[derive(Clone, Copy)] @@ -152,18 +185,21 @@ fn compile(b: Build) -> (PathBuf, String, Vec) { .packages .into_iter() .filter_map(Project::try_new) + .filter(|project| { + if project.meta.windows_only && cfg!(not(windows)) { + return false; + } + if project.meta.debug_only && profile != "debug" { + return false; + } + true + }) .collect::>(); let cargo = env::var("CARGO").unwrap_or_else(|_| "cargo".to_string()); let mut command = Command::new(&cargo); command.arg("build"); command.arg("--locked"); for project in projects.iter() { - if project.meta.windows_only && cfg!(not(windows)) { - continue; - } - if project.meta.debug_only && profile != "debug" { - continue; - } command.arg("--package"); command.arg(&project.name); } @@ -189,6 +225,92 @@ fn sniff_out_profile_name(b: &[OsString]) -> String { } fn zip(zip: Build) { + let (target_dir, profile, projects) = compile(zip); + os::zip(target_dir, profile, projects) +} + +#[cfg(unix)] +mod os { + use flate2::write::GzEncoder; + use flate2::Compression; + use std::{ + fs::{self, File}, + path::PathBuf, + }; + use tar::Header; + + pub fn make_symlinks( + target_directory: &std::path::PathBuf, + projects: &[super::Project], + profile: &str, + ) { + use std::os::unix::fs as unix_fs; + for project in projects.iter() { + let libname = project.file_name(); + for (_, full_path, target) in project.symlinks(target_directory, profile, &libname) { + let mut dir = full_path.clone(); + assert!(dir.pop()); + fs::create_dir_all(dir).unwrap(); + fs::remove_file(&full_path).ok(); + unix_fs::symlink(&target, full_path).unwrap(); + } + } + } + + pub(crate) fn zip(target_dir: PathBuf, profile: String, projects: Vec) { + let tar_gz = + File::create(format!("{}/{profile}/zluda.tar.gz", target_dir.display())).unwrap(); + let enc = GzEncoder::new(tar_gz, Compression::default()); + let mut tar = tar::Builder::new(enc); + for project in projects.iter() { + let file_name = project.file_name(); + let mut file = + File::open(format!("{}/{profile}/{file_name}", target_dir.display())).unwrap(); + tar.append_file(format!("zluda/{file_name}"), &mut file) + .unwrap(); + for (source, full_path, target) in project.symlinks(&target_dir, &profile, &file_name) { + let mut header = Header::new_gnu(); + let meta = fs::symlink_metadata(&full_path).unwrap(); + header.set_metadata(&meta); + tar.append_link(&mut header, format!("zluda/{source}"), target) + .unwrap(); + } + } + tar.finish().unwrap(); + } +} + +#[cfg(not(unix))] +mod os { + use zip::{write::SimpleFileOptions, ZipWriter}; + + pub fn make_symlinks( + _target_directory: &std::path::PathBuf, + _projects: &[super::Project], + _profile: &str, + ) { + } + + pub(crate) fn zip(target_dir: PathBuf, profile: String, projects: Vec) { + let zip_file = + File::create(format!("{}/{profile}/zluda.zip", target_dir.display())).unwrap(); + let mut zip = ZipWriter::new(zip_file); + zip.add_directory("zluda", SimpleFileOptions::default()) + .unwrap(); + for project in projects.iter() { + let name = &project.target_name; + let ext = project.suffix(); + let mut file = + std::fs::File::open(format!("{}/{profile}/{name}{ext}", target_dir.display())) + .unwrap(); + let file_options = file_options_from_time(&file).unwrap_or_default(); + zip.start_file(format!("zluda/{name}{ext}"), file_options) + .unwrap(); + std::io::copy(&mut file, &mut zip).unwrap(); + } + zip.finish().unwrap(); + } + fn file_options_from_time(from: &File) -> std::io::Result { let metadata = from.metadata()?; let modified = metadata.modified()?; @@ -197,71 +319,4 @@ fn zip(zip: Build) { zip::DateTime::try_from(modified).map_err(|err| std::io::Error::other(err))?, )) } - - let (target_dir, profile, projects) = compile(zip); - let zip_file = File::create(format!("{}/{profile}/zluda.zip", target_dir.display())).unwrap(); - let mut zip = ZipWriter::new(zip_file); - zip.add_directory("zluda", SimpleFileOptions::default()) - .unwrap(); - for project in projects.iter() { - let name = &project.target_name; - let ext = project.suffix(); - let mut file = - std::fs::File::open(format!("{}/{profile}/{name}{ext}", target_dir.display())).unwrap(); - let file_options = file_options_from_time(&file).unwrap_or_default(); - zip.start_file(format!("zluda/{name}{ext}"), file_options) - .unwrap(); - std::io::copy(&mut file, &mut zip).unwrap(); - } - zip.finish().unwrap(); -} - -#[cfg(unix)] -mod os { - use std::path::PathBuf; - - pub fn make_symlinks( - target_directory: &std::path::PathBuf, - _projects: &[super::Project], - profile: &str, - ) { - use std::fs; - use std::os::unix::fs as unix_fs; - for project in projects.iter() { - let clib_name = match project.clib_name { - Some(ref l) => l, - None => continue, - }; - let libname = format!("lib{}.so", clib_name); - for source in project.meta.linux_symlinks.iter() { - let relative_link = PathBuf::from(source); - let ancestors = relative_link.as_path().ancestors().count(); - let mut target = std::iter::repeat_with(|| "../").take(ancestors - 2).fold( - PathBuf::new(), - |mut buff, segment| { - buff.push(segment); - buff - }, - ); - let mut link = target_directory.clone(); - link.extend([profile, source]); - let mut dir = link.clone(); - assert!(dir.pop()); - fs::create_dir_all(dir).unwrap(); - fs::remove_file(&link).ok(); - target.push(&*libname); - unix_fs::symlink(&target, link).unwrap(); - } - } - } -} - -#[cfg(not(unix))] -mod os { - pub fn make_symlinks( - _target_directory: &std::path::PathBuf, - _projects: &[super::Project], - _profile: &str, - ) { - } }