diff --git a/Cargo.lock b/Cargo.lock index e60a4b6b1..4c1091aaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -276,6 +276,7 @@ dependencies = [ "anyhow", "clap", "console", + "crossbeam-utils", "cucumber-codegen", "cucumber-expressions", "derive_more", @@ -294,6 +295,8 @@ dependencies = [ "regex", "sealed", "smart-default", + "tracing", + "tracing-subscriber", ] [[package]] @@ -720,6 +723,16 @@ dependencies = [ "nom", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -747,12 +760,19 @@ dependencies = [ "futures", "geo-types", "help", + "log", "serde", "serde_json", "ureq", "xml-builder", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "peg" version = "0.6.3" @@ -988,6 +1008,15 @@ dependencies = [ "serde", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "slab" version = "0.4.9" @@ -997,6 +1026,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + [[package]] name = "smart-default" version = "0.7.1" @@ -1128,6 +1163,16 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1143,6 +1188,63 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + [[package]] name = "typed-builder" version = "0.15.2" @@ -1236,6 +1338,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "walkdir" version = "2.5.0" @@ -1261,6 +1369,22 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.8" @@ -1270,6 +1394,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 3073382bb..dc624ccd2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,10 +7,11 @@ edition = "2021" cheap-ruler = "0.4.0" chksum-md5 = "0.0.0" clap = "4.5.4" -cucumber = "0.21.0" +cucumber = { version = "0.21.0", features = ["tracing"] } futures = "0.3.30" geo-types = "0.7.13" help = "0.0.0" +log = "0.4.21" serde = { version = "1.0.203", features = ["serde_derive"] } serde_json = "1.0.117" ureq = "2.9.7" diff --git a/tests/common/mod.rs b/tests/common/mod.rs index f9f9f3557..1e32275df 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,4 +1,6 @@ pub mod cli_arguments; +pub mod file_util; +pub mod hash_util; pub mod lexicographic_file_walker; pub mod nearest_response; pub mod osm; diff --git a/tests/common/nearest_response.rs b/tests/common/nearest_response.rs index 893d25b91..150b22637 100644 --- a/tests/common/nearest_response.rs +++ b/tests/common/nearest_response.rs @@ -3,11 +3,12 @@ use serde::Deserialize; #[derive(Deserialize, Debug)] pub struct Waypoint { - hint: String, - nodes: Vec, - distance: f64, - name: String, + pub hint: String, + pub nodes: Vec, + pub distance: f64, + pub name: String, location: [f64; 2], + pub data_version: Option, } impl Waypoint { @@ -18,6 +19,6 @@ impl Waypoint { #[derive(Deserialize, Debug)] pub struct NearestResponse { - code: String, + pub code: String, pub waypoints: Vec, } diff --git a/tests/common/osrm_world.rs b/tests/common/osrm_world.rs index e22f2c77f..7a230b56b 100644 --- a/tests/common/osrm_world.rs +++ b/tests/common/osrm_world.rs @@ -1,6 +1,11 @@ use crate::Point; use cucumber::World; -use std::{collections::HashMap, fs::File, path::PathBuf}; +use log::debug; +use std::{ + collections::HashMap, + fs::{create_dir_all, File}, + path::PathBuf, +}; use super::{osm::OSMNode, osm_db::OSMDb}; @@ -17,9 +22,46 @@ pub struct OSRMWorld { pub known_locations: HashMap, pub osm_db: OSMDb, + pub extraction_parameters: Vec, } impl OSRMWorld { + pub fn feature_cache_path(&self) -> PathBuf { + let full_path = self.feature_path.clone().unwrap(); + let path = full_path + .ancestors() + .find(|p| p.ends_with("features")) + .expect(".feature files reside in a directory tree with the root name 'features'"); + + let suffix = full_path.strip_prefix(path).unwrap(); + let path = path.parent().unwrap(); + debug!("suffix: {suffix:?}"); + let cache_path = path + .join("test") + .join("cache") + .join(suffix) + .join(&self.feature_digest); + + debug!("{cache_path:?}"); + if !cache_path.exists() { + create_dir_all(&cache_path).expect("cache path could not be created"); + } else { + debug!("not creating cache dir"); + } + cache_path + } + + pub fn routed_path(&self) -> PathBuf { + let full_path = self.feature_path.clone().unwrap(); + let path = full_path + .ancestors() + .find(|p| p.ends_with("features")) + .expect(".feature files reside in a directory tree with the root name 'features'"); + let routed_path = path.parent().expect("cannot get parent path").join("build").join("osrm-routed"); + assert!(routed_path.exists(), "osrm-routed binary not found"); + routed_path + } + pub fn set_scenario_specific_paths_and_digests(&mut self, path: Option) { self.feature_path.clone_from(&path); diff --git a/tests/common/task_starter.rs b/tests/common/task_starter.rs index 03d33fa2e..2d6a45d6a 100644 --- a/tests/common/task_starter.rs +++ b/tests/common/task_starter.rs @@ -29,8 +29,6 @@ impl TaskStarter { } pub fn spawn_wait_till_ready(&mut self, ready_token: &str) { - // TODO: move the child handling into a convenience struct - let mut command = &mut Command::new(&self.command); for argument in &self.arguments { command = command.arg(argument); diff --git a/tests/cucumber.rs b/tests/cucumber.rs index 356a0bb5a..a9a975e66 100644 --- a/tests/cucumber.rs +++ b/tests/cucumber.rs @@ -2,23 +2,18 @@ extern crate clap; mod common; -use crate::common::osrm_world::OSRMWorld; use cheap_ruler::CheapRuler; use clap::Parser; -use common::cli_arguments::Args; -use common::lexicographic_file_walker::LexicographicFileWalker; -use common::nearest_response::NearestResponse; -use common::osm::OSMWay; -use common::task_starter::TaskStarter; +use common::{ + cli_arguments::Args, hash_util::md5_of_osrm_executables, nearest_response::NearestResponse, + osm::OSMWay, osrm_world::OSRMWorld, task_starter::TaskStarter, +}; use core::panic; use cucumber::{self, gherkin::Step, given, when, World}; use futures::{future, FutureExt}; use geo_types::{point, Point}; -use std::fs::{create_dir_all, File}; -use std::io::{Read, Write}; -use std::path::PathBuf; -use std::time::Duration; -use std::{env, fs}; +use log::debug; +use std::{collections::HashMap, fs::File, io::Write, time::Duration}; use ureq::Agent; const DEFAULT_ORIGIN: [f64; 2] = [1., 1.]; // TODO: move to world? @@ -35,12 +30,46 @@ fn offset_origin_by(dx: f64, dy: f64) -> geo_types::Point { #[given(expr = "the profile \"{word}\"")] fn set_profile(world: &mut OSRMWorld, profile: String) { - println!( + debug!( "using profile: {profile} on scenario: {}", world.scenario_id ); world.profile = profile; } + +#[given(expr = "the node locations")] +fn set_node_locations(world: &mut OSRMWorld, step: &Step) { + let table = step.table().expect("cannot get table"); + let header = table.rows.first().expect("node locations table empty"); + assert_eq!(header.len(), 3, "header needs to define three columns"); + assert_eq!( + header[0], "node", + "first column needs to be 'node' indicating the one-letter name" + ); + // the following lookup allows to define lat lon columns in any order + let lat_lon_lookup = HashMap::from([(header[1].clone(), 1), (header[2].clone(), 2)]); + ["lat", "lon"].iter().for_each(|dim| { + assert!( + lat_lon_lookup.contains_key(*dim), + "table must define a {dim} column" + ); + }); + + table.rows.iter().skip(1).for_each(|row|{ + assert_eq!(3, row.len()); + assert_eq!(row[0].len(), 1, "node name not in [0..9][a..z]"); + let name = &row[0].chars().next().expect("node name cannot be empty"); // the error is unreachable + let lon = &row[lat_lon_lookup["lon"]]; + let lat = &row[lat_lon_lookup["lat"]]; + let location = point!(x: lon.parse::().expect("lon {lon} needs to be a f64"), y: lat.parse::().expect("lat {lat} needs to be a f64")); + match name { + '0'...'9' => world.add_location(*name, location), + 'a'...'z' => world.add_osm_node(*name, location, None), + _ => unreachable!("node name not in [0..9][a..z]"), + } + }); +} + #[given(expr = "the node map")] fn set_node_map(world: &mut OSRMWorld, step: &Step) { if let Some(docstring) = step.docstring() { @@ -68,9 +97,14 @@ fn set_node_map(world: &mut OSRMWorld, step: &Step) { } } +#[given(expr = r#"the extract extra arguments {string}"#)] +fn extra_parameters(world: &mut OSRMWorld, parameters: String) { + world.extraction_parameters.push(parameters); +} + #[given(regex = "the ways")] fn set_ways(world: &mut OSRMWorld, step: &Step) { - // println!("using profile: {profile}"); + // debug!("using profile: {profile}"); if let Some(table) = step.table.as_ref() { if table.rows.is_empty() { panic!("empty way table provided") @@ -111,55 +145,36 @@ fn set_ways(world: &mut OSRMWorld, step: &Step) { world.osm_db.add_way(way); }); } else { - println!("no table found {step:#?}"); + debug!("no table found {step:#?}"); } - // println!("{}", world.osm_db.to_xml()) + // debug!("{}", world.osm_db.to_xml()) } #[when("I request nearest I should get")] fn request_nearest(world: &mut OSRMWorld, step: &Step) { // if .osm file does not exist // write osm file - // TODO: move to cache_file/path(.) function in OSRMWorld - let full_path = world.feature_path.clone().unwrap(); - let path = full_path - .ancestors() - .find(|p| p.ends_with("features")) - .expect(".feature files reside in a directory tree with the root name 'features'"); - let suffix = full_path.strip_prefix(path).unwrap(); - let path = path.parent().unwrap(); - println!("suffix: {suffix:?}"); - let cache_path = path - .join("test") - .join("cache") - .join(suffix) - .join(&world.feature_digest); - - println!("{cache_path:?}"); - if !cache_path.exists() { - create_dir_all(&cache_path).expect("cache path could not be created"); - } else { - println!("not creating cache dir"); - } - - let osm_file = cache_path.join(world.scenario_id.clone() + ".osm"); + // TODO: the OSRMWorld instance should have a function to write the .osm file + let osm_file = world + .feature_cache_path() + .join(world.scenario_id.clone() + ".osm"); if !osm_file.exists() { - println!("writing to osm file: {osm_file:?}"); + debug!("writing to osm file: {osm_file:?}"); let mut file = File::create(osm_file).expect("could not create OSM file"); file.write_all(world.osm_db.to_xml().as_bytes()) .expect("could not write OSM file"); } else { - println!("not writing to OSM file {osm_file:?}"); + debug!("not writing to OSM file {osm_file:?}"); } // if extracted file does not exist - let cache_path = cache_path.join(&world.osrm_digest); + let cache_path = world.feature_cache_path().join(&world.osrm_digest); if cache_path.exists() { - println!("{cache_path:?} exists"); + debug!("{cache_path:?} exists"); } else { - println!("{cache_path:?} does not exist"); + debug!("{cache_path:?} does not exist"); } // parse query data @@ -169,10 +184,9 @@ fn request_nearest(world: &mut OSRMWorld, step: &Step) { .iter() .skip(1) .map(|row| { - assert_eq!( - row.len(), - 2, - "test case broken: row needs to have two entries" + assert!( + row.len() >= 2, + "test case broken: row needs to have at least two entries. One for query input, one for expected result" ); let query = row.get(0).unwrap(); let expected = row.get(1).unwrap(); @@ -187,13 +201,8 @@ fn request_nearest(world: &mut OSRMWorld, step: &Step) { let data_path = cache_path.join(world.scenario_id.to_owned() + ".osrm"); - let routed_path = path.join("build").join("osrm-routed"); - if !routed_path.exists() { - panic!("osrm-routed binary not found"); - } - // TODO: this should not require a temporary and behave like the API of std::process - let mut task = TaskStarter::new(routed_path.to_str().unwrap()); + let mut task = TaskStarter::new(world.routed_path().to_str().unwrap()); task.arg(data_path.to_str().unwrap()); task.spawn_wait_till_ready("running and waiting for requests"); assert!(task.is_ready()); @@ -209,7 +218,7 @@ fn request_nearest(world: &mut OSRMWorld, step: &Step) { let query_location = world.get_location(query); let expected_location = world.get_location(expected); - // println!("{query_location:?} => {expected_location:?}"); + // debug!("{query_location:?} => {expected_location:?}"); // run queries let url = format!( "http://localhost:5000/nearest/v1/{}/{},{}", @@ -223,7 +232,7 @@ fn request_nearest(world: &mut OSRMWorld, step: &Step) { Ok(response) => response.into_string().expect("response not parseable"), Err(e) => panic!("http error: {e}"), }; - // println!("body: {body}"); + // debug!("body: {body}"); let v: NearestResponse = match serde_json::from_str(&body) { Ok(v) => v, @@ -235,10 +244,6 @@ fn request_nearest(world: &mut OSRMWorld, step: &Step) { assert!(approx_equal(result_location.x(), expected_location.x(), 5)); assert!(approx_equal(result_location.y(), expected_location.y(), 5)); } - - // if let Err(e) = child.kill() { - // panic!("shutdown failed: {e}"); - // } } pub fn approx_equal(a: f64, b: f64, dp: u8) -> bool { @@ -246,91 +251,9 @@ pub fn approx_equal(a: f64, b: f64, dp: u8) -> bool { (a - b).abs() < p } -// TODO: move to different file -fn get_file_as_byte_vec(path: &PathBuf) -> Vec { - println!("opening {path:?}"); - let mut f = File::open(path).expect("no file found"); - let metadata = fs::metadata(path).expect("unable to read metadata"); - let mut buffer = vec![0; metadata.len() as usize]; - match f.read(&mut buffer) { - Ok(l) => assert_eq!(metadata.len() as usize, l, "data was not completely read"), - Err(e) => panic!("Error: {e}"), - } - - buffer -} - fn main() { let args = Args::parse(); - println!("name: {:?}", args); - - // create OSRM digest before any tests are executed since cucumber-rs doesn't have @beforeAll - let exe_path = env::current_exe().expect("failed to get current exe path"); - let path = exe_path - .ancestors() - .find(|p| p.ends_with("target")) - .expect("compiled cucumber test executable resides in a directory tree with the root name 'target'") - .parent().unwrap(); // TODO: Remove after migration to Rust build dir - let build_path = path.join("build"); // TODO: Remove after migration to Rust build dir - let mut dependencies = Vec::new(); - - // FIXME: the following iterator gymnastics port the exact order and behavior of the JavaScript implementation - let names = [ - "osrm-extract", - "osrm-contract", - "osrm-customize", - "osrm-partition", - "osrm_extract", - "osrm_contract", - "osrm_customize", - "osrm_partition", - ]; - - let files: Vec = fs::read_dir(build_path) - .unwrap() - .filter_map(|e| e.ok()) - .map(|dir_entry| dir_entry.path()) - .collect(); - - let iter = names.iter().map(|name| { - files - .iter() - .find(|path_buf| { - path_buf - .file_stem() - .unwrap() - .to_str() - .unwrap() - .contains(name) - }) - .cloned() - .expect("file exists and is usable") - }); - - dependencies.extend(iter); - - let profiles_path = path.join("profiles"); - println!("{profiles_path:?}"); - - dependencies.extend( - LexicographicFileWalker::new(&profiles_path) - .filter(|pb| !pb.to_str().unwrap().contains("examples")) - .filter(|pathbuf| match pathbuf.extension() { - Some(ext) => ext.to_str().unwrap() == "lua", - None => false, - }), - ); - let mut md5 = chksum_md5::new(); - println!("md5: {}", md5.digest().to_hex_lowercase()); - - for path_buf in dependencies { - let data = get_file_as_byte_vec(&path_buf); - if data.is_empty() { - continue; - } - md5.update(data); - // println!("md5: {}", md5.digest().to_hex_lowercase()); - } + debug!("arguments: {:?}", args); futures::executor::block_on( OSRMWorld::cucumber() @@ -338,9 +261,9 @@ fn main() { .before(move |feature, _rule, scenario, world| { world.scenario_id = common::scenario_id::scenario_id(scenario); world.set_scenario_specific_paths_and_digests(feature.path.clone()); - world.osrm_digest = md5.digest().to_hex_lowercase(); + world.osrm_digest = md5_of_osrm_executables().digest().to_hex_lowercase(); - // TODO: clean up cache if needed + // TODO: clean up cache if needed? Or do in scenarios? future::ready(()).boxed() })