From 77477a86dd57435574e7747f254942bc4b1b7196 Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 13 Jun 2024 11:51:10 +0200 Subject: [PATCH] Implement downloading of flatc on Linux --- Cargo.lock | 345 +++++++++++++++++++++++++++++++ Cargo.toml | 4 + build.rs | 95 ++++++++- tests/common/mod.rs | 1 + tests/common/nearest_response.rs | 1 + 5 files changed, 443 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e3b74239c..2881d08cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -93,6 +104,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bitflags" version = "1.3.2" @@ -105,6 +122,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" version = "1.9.1" @@ -121,11 +147,43 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] [[package]] name = "cfg-if" @@ -179,6 +237,16 @@ dependencies = [ "chksum-hash-md5", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.4" @@ -249,6 +317,21 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -283,6 +366,16 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "cucumber" version = "0.21.0" @@ -356,6 +449,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "drain_filter_polyfill" version = "0.1.3" @@ -383,6 +487,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.9" @@ -520,6 +630,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "geo-types" version = "0.7.13" @@ -583,6 +703,12 @@ dependencies = [ "walkdir", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "heck" version = "0.4.1" @@ -601,6 +727,15 @@ version = "0.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5d2c6714f78261bbacbd07338eb21943fa5a25c67760287aecb6aefbec89062" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "humantime" version = "2.1.0" @@ -633,12 +768,31 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "inflections" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a257582fdcde896fd96463bf2d40eefea0580021c0712a0e2b028b60b47a837a" +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "inventory" version = "0.3.15" @@ -666,6 +820,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + [[package]] name = "lazy-regex" version = "3.1.0" @@ -810,8 +973,10 @@ dependencies = [ "log", "serde", "serde_json", + "toml", "ureq", "xml-builder", + "zip-extract", ] [[package]] @@ -820,6 +985,29 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", + "hmac", + "password-hash", + "sha2", +] + [[package]] name = "peg" version = "0.6.3" @@ -885,6 +1073,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + [[package]] name = "proc-macro2" version = "1.0.84" @@ -903,6 +1097,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + [[package]] name = "regex" version = "1.10.4" @@ -1070,6 +1270,37 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +dependencies = [ + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1250,6 +1481,40 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "toml" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tracing" version = "0.1.40" @@ -1327,6 +1592,12 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-bidi" version = "0.3.15" @@ -1407,6 +1678,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "walkdir" version = "2.5.0" @@ -1602,6 +1879,15 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winnow" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +dependencies = [ + "memchr", +] + [[package]] name = "xml-builder" version = "0.5.2" @@ -1613,3 +1899,62 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "aes", + "byteorder", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2", + "sha1", + "zstd", +] + +[[package]] +name = "zip-extract" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e109e5a291403b4c1e514d39f8a22d3f98d257e691a52bb1f16051bb1ffed63e" +dependencies = [ + "log", + "thiserror", + "zip", +] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.10+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 4496aa7c6..eb9356c51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,3 +28,7 @@ debug = true [build-dependencies] flatc-rust = "0.2.0" +serde = { version = "1.0.203", features = ["serde_derive"] } +toml = "0.8.14" +ureq = { version = "2.9.7", features = ["charset"] } +zip-extract = "0.1.3" diff --git a/build.rs b/build.rs index 48fcdd0ca..e3ad1976e 100644 --- a/build.rs +++ b/build.rs @@ -1,9 +1,98 @@ -extern crate flatc_rust; // or just `use flatc_rust;` with Rust 2018 edition. +use std::fmt::Display; +use std::io::{Cursor, Write}; +use std::path::PathBuf; +use std::time::Duration; +use std::{collections::HashMap, path::Path}; +use std::{env, fs}; -use std::path::Path; +use serde::{Deserialize, Serialize}; +use ureq::{Agent, AgentBuilder}; + +#[derive(Debug)] +enum OS { + Mac, + MacIntel, + Linux, + Windows, +} + +impl Display for OS { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} fn main() { - println!("cargo:rerun-if-changed=include/engine/api/flatbuffers/"); + #[derive(Debug, Serialize, Deserialize)] + #[serde(untagged)] + enum DependencyValue { + String(String), + Object { + version: String, + features: Vec, + }, + } + #[derive(Debug, Serialize, Deserialize)] + struct CargoToml { + dependencies: HashMap, + } + + let cargo_toml_raw = include_str!("Cargo.toml"); + let cargo_toml: CargoToml = toml::from_str(cargo_toml_raw).unwrap(); + + let version = match cargo_toml + .dependencies + .get("flatbuffers") + .expect("Must have dependency flatbuffers") + { + DependencyValue::String(s) => s, + DependencyValue::Object { + version, + features: _, + } => version, + }; + if let Some((platform, compiler)) = match env::consts::OS { + "linux" if env::consts::ARCH == "x86_64" => Some((OS::Linux, ".clang++-15")), + "macos" if env::consts::ARCH == "x86_64" => Some((OS::MacIntel, "")), // TODO: check literals + "macos" if env::consts::ARCH == "arm" => Some((OS::Mac, "")), // TODO: check literals + "windows" if env::consts::ARCH == "x86_64" => Some((OS::Windows, "")), + _ => { + println!("cargo:warning=unsupported platform: {} {}. 'flatc' binary supporting version {} of the library needs to be in system path", env::consts::OS, env::consts::ARCH, version); + None + } + } { + let url = format!("https://github.com/google/flatbuffers/releases/download/v{version}/{platform}.flatc.binary{compiler}.zip"); + + // download flatc compiler if it does not exist + if !Path::new("target/flatc").exists() { + println!("cargo:warning=downloading flatc from {url}"); + let agent = AgentBuilder::new() + .timeout_read(Duration::from_secs(5)) + .timeout_write(Duration::from_secs(5)) + .build(); + + let call = agent.get(&url).call(); + let mut reader = match call { + Ok(response) => response.into_reader(), + Err(e) => panic!("http error: {e}"), + }; + let mut archive = Vec::new(); + if let Err(e) = reader.read_to_end(&mut archive) { + panic!("cannot read from strem: {e}"); + }; + let target_dir = PathBuf::from("target"); // Doesn't need to exist + zip_extract::extract(Cursor::new(archive), &target_dir, true).expect("flatc cannot be unpacked") + } else { + println!("cargo:warning=cached zip file found, not downloading"); + } + // TODO: unpack binary + } + // Linux + x86: https://github.com/google/flatbuffers/releases/download/v24.3.25/Linux.flatc.binary.clang++-15.zip + // macOS + aarch: https://github.com/google/flatbuffers/releases/download/v24.3.25/Mac.flatc.binary.zip + // macOS + x86: https://github.com/google/flatbuffers/releases/download/v24.3.25/MacIntel.flatc.binary.zip + // Windows + x86: https://github.com/google/flatbuffers/releases/download/v24.3.25/Windows.flatc.binary.zip + + // TODO: check if file exists, and then run it or try global installation flatc_rust::run(flatc_rust::Args { extra: &["--gen-all"], inputs: &[ diff --git a/tests/common/mod.rs b/tests/common/mod.rs index f3741f8d3..59a15cac6 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -6,6 +6,7 @@ pub mod dot_writer; pub mod f64_utils; pub mod file_util; pub mod hash_util; +pub mod http_request; pub mod lexicographic_file_walker; pub mod location; pub mod nearest_response; diff --git a/tests/common/nearest_response.rs b/tests/common/nearest_response.rs index 8aa626b61..39513b120 100644 --- a/tests/common/nearest_response.rs +++ b/tests/common/nearest_response.rs @@ -25,6 +25,7 @@ pub struct NearestResponse { } impl NearestResponse { + // TODO: the from_* functions should be a) into_. or b) implement From<_> trait pub fn from_json_reader(reader: impl std::io::Read) -> Self { let response = match serde_json::from_reader::<_, NearestResponse>(reader) { Ok(response) => response,