Implement parsing of node tables with locations

This commit is contained in:
Dennis 2024-06-05 10:31:07 +02:00
parent 96e90ce321
commit beaaa597d4
No known key found for this signature in database
GPG Key ID: 6937EAEA33A3FA5D
7 changed files with 250 additions and 153 deletions

130
Cargo.lock generated
View File

@ -276,6 +276,7 @@ dependencies = [
"anyhow",
"clap",
"console",
"crossbeam-utils",
"cucumber-codegen",
"cucumber-expressions",
"derive_more",
@ -294,6 +295,8 @@ dependencies = [
"regex",
"sealed",
"smart-default",
"tracing",
"tracing-subscriber",
]
[[package]]
@ -720,6 +723,16 @@ dependencies = [
"nom",
]
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
dependencies = [
"overload",
"winapi",
]
[[package]]
name = "num-traits"
version = "0.2.19"
@ -747,12 +760,19 @@ dependencies = [
"futures",
"geo-types",
"help",
"log",
"serde",
"serde_json",
"ureq",
"xml-builder",
]
[[package]]
name = "overload"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]]
name = "peg"
version = "0.6.3"
@ -988,6 +1008,15 @@ dependencies = [
"serde",
]
[[package]]
name = "sharded-slab"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
dependencies = [
"lazy_static",
]
[[package]]
name = "slab"
version = "0.4.9"
@ -997,6 +1026,12 @@ dependencies = [
"autocfg",
]
[[package]]
name = "smallvec"
version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
[[package]]
name = "smart-default"
version = "0.7.1"
@ -1128,6 +1163,16 @@ dependencies = [
"syn 2.0.66",
]
[[package]]
name = "thread_local"
version = "1.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
dependencies = [
"cfg-if",
"once_cell",
]
[[package]]
name = "tinyvec"
version = "1.6.0"
@ -1143,6 +1188,63 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tracing"
version = "0.1.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
dependencies = [
"pin-project-lite",
"tracing-attributes",
"tracing-core",
]
[[package]]
name = "tracing-attributes"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
]
[[package]]
name = "tracing-core"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
dependencies = [
"once_cell",
"valuable",
]
[[package]]
name = "tracing-log"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
dependencies = [
"log",
"once_cell",
"tracing-core",
]
[[package]]
name = "tracing-subscriber"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
dependencies = [
"nu-ansi-term",
"sharded-slab",
"smallvec",
"thread_local",
"tracing-core",
"tracing-log",
]
[[package]]
name = "typed-builder"
version = "0.15.2"
@ -1236,6 +1338,12 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
[[package]]
name = "valuable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]]
name = "walkdir"
version = "2.5.0"
@ -1261,6 +1369,22 @@ dependencies = [
"rustls-pki-types",
]
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.8"
@ -1270,6 +1394,12 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-sys"
version = "0.48.0"

View File

@ -7,10 +7,11 @@ edition = "2021"
cheap-ruler = "0.4.0"
chksum-md5 = "0.0.0"
clap = "4.5.4"
cucumber = "0.21.0"
cucumber = { version = "0.21.0", features = ["tracing"] }
futures = "0.3.30"
geo-types = "0.7.13"
help = "0.0.0"
log = "0.4.21"
serde = { version = "1.0.203", features = ["serde_derive"] }
serde_json = "1.0.117"
ureq = "2.9.7"

View File

@ -1,4 +1,6 @@
pub mod cli_arguments;
pub mod file_util;
pub mod hash_util;
pub mod lexicographic_file_walker;
pub mod nearest_response;
pub mod osm;

View File

@ -3,11 +3,12 @@ use serde::Deserialize;
#[derive(Deserialize, Debug)]
pub struct Waypoint {
hint: String,
nodes: Vec<u64>,
distance: f64,
name: String,
pub hint: String,
pub nodes: Vec<u64>,
pub distance: f64,
pub name: String,
location: [f64; 2],
pub data_version: Option<String>,
}
impl Waypoint {
@ -18,6 +19,6 @@ impl Waypoint {
#[derive(Deserialize, Debug)]
pub struct NearestResponse {
code: String,
pub code: String,
pub waypoints: Vec<Waypoint>,
}

View File

@ -1,6 +1,11 @@
use crate::Point;
use cucumber::World;
use std::{collections::HashMap, fs::File, path::PathBuf};
use log::debug;
use std::{
collections::HashMap,
fs::{create_dir_all, File},
path::PathBuf,
};
use super::{osm::OSMNode, osm_db::OSMDb};
@ -17,9 +22,46 @@ pub struct OSRMWorld {
pub known_locations: HashMap<char, Point>,
pub osm_db: OSMDb,
pub extraction_parameters: Vec<String>,
}
impl OSRMWorld {
pub fn feature_cache_path(&self) -> PathBuf {
let full_path = self.feature_path.clone().unwrap();
let path = full_path
.ancestors()
.find(|p| p.ends_with("features"))
.expect(".feature files reside in a directory tree with the root name 'features'");
let suffix = full_path.strip_prefix(path).unwrap();
let path = path.parent().unwrap();
debug!("suffix: {suffix:?}");
let cache_path = path
.join("test")
.join("cache")
.join(suffix)
.join(&self.feature_digest);
debug!("{cache_path:?}");
if !cache_path.exists() {
create_dir_all(&cache_path).expect("cache path could not be created");
} else {
debug!("not creating cache dir");
}
cache_path
}
pub fn routed_path(&self) -> PathBuf {
let full_path = self.feature_path.clone().unwrap();
let path = full_path
.ancestors()
.find(|p| p.ends_with("features"))
.expect(".feature files reside in a directory tree with the root name 'features'");
let routed_path = path.parent().expect("cannot get parent path").join("build").join("osrm-routed");
assert!(routed_path.exists(), "osrm-routed binary not found");
routed_path
}
pub fn set_scenario_specific_paths_and_digests(&mut self, path: Option<PathBuf>) {
self.feature_path.clone_from(&path);

View File

@ -29,8 +29,6 @@ impl TaskStarter {
}
pub fn spawn_wait_till_ready(&mut self, ready_token: &str) {
// TODO: move the child handling into a convenience struct
let mut command = &mut Command::new(&self.command);
for argument in &self.arguments {
command = command.arg(argument);

View File

@ -2,23 +2,18 @@ extern crate clap;
mod common;
use crate::common::osrm_world::OSRMWorld;
use cheap_ruler::CheapRuler;
use clap::Parser;
use common::cli_arguments::Args;
use common::lexicographic_file_walker::LexicographicFileWalker;
use common::nearest_response::NearestResponse;
use common::osm::OSMWay;
use common::task_starter::TaskStarter;
use common::{
cli_arguments::Args, hash_util::md5_of_osrm_executables, nearest_response::NearestResponse,
osm::OSMWay, osrm_world::OSRMWorld, task_starter::TaskStarter,
};
use core::panic;
use cucumber::{self, gherkin::Step, given, when, World};
use futures::{future, FutureExt};
use geo_types::{point, Point};
use std::fs::{create_dir_all, File};
use std::io::{Read, Write};
use std::path::PathBuf;
use std::time::Duration;
use std::{env, fs};
use log::debug;
use std::{collections::HashMap, fs::File, io::Write, time::Duration};
use ureq::Agent;
const DEFAULT_ORIGIN: [f64; 2] = [1., 1.]; // TODO: move to world?
@ -35,12 +30,46 @@ fn offset_origin_by(dx: f64, dy: f64) -> geo_types::Point {
#[given(expr = "the profile \"{word}\"")]
fn set_profile(world: &mut OSRMWorld, profile: String) {
println!(
debug!(
"using profile: {profile} on scenario: {}",
world.scenario_id
);
world.profile = profile;
}
#[given(expr = "the node locations")]
fn set_node_locations(world: &mut OSRMWorld, step: &Step) {
let table = step.table().expect("cannot get table");
let header = table.rows.first().expect("node locations table empty");
assert_eq!(header.len(), 3, "header needs to define three columns");
assert_eq!(
header[0], "node",
"first column needs to be 'node' indicating the one-letter name"
);
// the following lookup allows to define lat lon columns in any order
let lat_lon_lookup = HashMap::from([(header[1].clone(), 1), (header[2].clone(), 2)]);
["lat", "lon"].iter().for_each(|dim| {
assert!(
lat_lon_lookup.contains_key(*dim),
"table must define a {dim} column"
);
});
table.rows.iter().skip(1).for_each(|row|{
assert_eq!(3, row.len());
assert_eq!(row[0].len(), 1, "node name not in [0..9][a..z]");
let name = &row[0].chars().next().expect("node name cannot be empty"); // the error is unreachable
let lon = &row[lat_lon_lookup["lon"]];
let lat = &row[lat_lon_lookup["lat"]];
let location = point!(x: lon.parse::<f64>().expect("lon {lon} needs to be a f64"), y: lat.parse::<f64>().expect("lat {lat} needs to be a f64"));
match name {
'0'...'9' => world.add_location(*name, location),
'a'...'z' => world.add_osm_node(*name, location, None),
_ => unreachable!("node name not in [0..9][a..z]"),
}
});
}
#[given(expr = "the node map")]
fn set_node_map(world: &mut OSRMWorld, step: &Step) {
if let Some(docstring) = step.docstring() {
@ -68,9 +97,14 @@ fn set_node_map(world: &mut OSRMWorld, step: &Step) {
}
}
#[given(expr = r#"the extract extra arguments {string}"#)]
fn extra_parameters(world: &mut OSRMWorld, parameters: String) {
world.extraction_parameters.push(parameters);
}
#[given(regex = "the ways")]
fn set_ways(world: &mut OSRMWorld, step: &Step) {
// println!("using profile: {profile}");
// debug!("using profile: {profile}");
if let Some(table) = step.table.as_ref() {
if table.rows.is_empty() {
panic!("empty way table provided")
@ -111,55 +145,36 @@ fn set_ways(world: &mut OSRMWorld, step: &Step) {
world.osm_db.add_way(way);
});
} else {
println!("no table found {step:#?}");
debug!("no table found {step:#?}");
}
// println!("{}", world.osm_db.to_xml())
// debug!("{}", world.osm_db.to_xml())
}
#[when("I request nearest I should get")]
fn request_nearest(world: &mut OSRMWorld, step: &Step) {
// if .osm file does not exist
// write osm file
// TODO: move to cache_file/path(.) function in OSRMWorld
let full_path = world.feature_path.clone().unwrap();
let path = full_path
.ancestors()
.find(|p| p.ends_with("features"))
.expect(".feature files reside in a directory tree with the root name 'features'");
let suffix = full_path.strip_prefix(path).unwrap();
let path = path.parent().unwrap();
println!("suffix: {suffix:?}");
let cache_path = path
.join("test")
.join("cache")
.join(suffix)
.join(&world.feature_digest);
println!("{cache_path:?}");
if !cache_path.exists() {
create_dir_all(&cache_path).expect("cache path could not be created");
} else {
println!("not creating cache dir");
}
let osm_file = cache_path.join(world.scenario_id.clone() + ".osm");
// TODO: the OSRMWorld instance should have a function to write the .osm file
let osm_file = world
.feature_cache_path()
.join(world.scenario_id.clone() + ".osm");
if !osm_file.exists() {
println!("writing to osm file: {osm_file:?}");
debug!("writing to osm file: {osm_file:?}");
let mut file = File::create(osm_file).expect("could not create OSM file");
file.write_all(world.osm_db.to_xml().as_bytes())
.expect("could not write OSM file");
} else {
println!("not writing to OSM file {osm_file:?}");
debug!("not writing to OSM file {osm_file:?}");
}
// if extracted file does not exist
let cache_path = cache_path.join(&world.osrm_digest);
let cache_path = world.feature_cache_path().join(&world.osrm_digest);
if cache_path.exists() {
println!("{cache_path:?} exists");
debug!("{cache_path:?} exists");
} else {
println!("{cache_path:?} does not exist");
debug!("{cache_path:?} does not exist");
}
// parse query data
@ -169,10 +184,9 @@ fn request_nearest(world: &mut OSRMWorld, step: &Step) {
.iter()
.skip(1)
.map(|row| {
assert_eq!(
row.len(),
2,
"test case broken: row needs to have two entries"
assert!(
row.len() >= 2,
"test case broken: row needs to have at least two entries. One for query input, one for expected result"
);
let query = row.get(0).unwrap();
let expected = row.get(1).unwrap();
@ -187,13 +201,8 @@ fn request_nearest(world: &mut OSRMWorld, step: &Step) {
let data_path = cache_path.join(world.scenario_id.to_owned() + ".osrm");
let routed_path = path.join("build").join("osrm-routed");
if !routed_path.exists() {
panic!("osrm-routed binary not found");
}
// TODO: this should not require a temporary and behave like the API of std::process
let mut task = TaskStarter::new(routed_path.to_str().unwrap());
let mut task = TaskStarter::new(world.routed_path().to_str().unwrap());
task.arg(data_path.to_str().unwrap());
task.spawn_wait_till_ready("running and waiting for requests");
assert!(task.is_ready());
@ -209,7 +218,7 @@ fn request_nearest(world: &mut OSRMWorld, step: &Step) {
let query_location = world.get_location(query);
let expected_location = world.get_location(expected);
// println!("{query_location:?} => {expected_location:?}");
// debug!("{query_location:?} => {expected_location:?}");
// run queries
let url = format!(
"http://localhost:5000/nearest/v1/{}/{},{}",
@ -223,7 +232,7 @@ fn request_nearest(world: &mut OSRMWorld, step: &Step) {
Ok(response) => response.into_string().expect("response not parseable"),
Err(e) => panic!("http error: {e}"),
};
// println!("body: {body}");
// debug!("body: {body}");
let v: NearestResponse = match serde_json::from_str(&body) {
Ok(v) => v,
@ -235,10 +244,6 @@ fn request_nearest(world: &mut OSRMWorld, step: &Step) {
assert!(approx_equal(result_location.x(), expected_location.x(), 5));
assert!(approx_equal(result_location.y(), expected_location.y(), 5));
}
// if let Err(e) = child.kill() {
// panic!("shutdown failed: {e}");
// }
}
pub fn approx_equal(a: f64, b: f64, dp: u8) -> bool {
@ -246,91 +251,9 @@ pub fn approx_equal(a: f64, b: f64, dp: u8) -> bool {
(a - b).abs() < p
}
// TODO: move to different file
fn get_file_as_byte_vec(path: &PathBuf) -> Vec<u8> {
println!("opening {path:?}");
let mut f = File::open(path).expect("no file found");
let metadata = fs::metadata(path).expect("unable to read metadata");
let mut buffer = vec![0; metadata.len() as usize];
match f.read(&mut buffer) {
Ok(l) => assert_eq!(metadata.len() as usize, l, "data was not completely read"),
Err(e) => panic!("Error: {e}"),
}
buffer
}
fn main() {
let args = Args::parse();
println!("name: {:?}", args);
// create OSRM digest before any tests are executed since cucumber-rs doesn't have @beforeAll
let exe_path = env::current_exe().expect("failed to get current exe path");
let path = exe_path
.ancestors()
.find(|p| p.ends_with("target"))
.expect("compiled cucumber test executable resides in a directory tree with the root name 'target'")
.parent().unwrap(); // TODO: Remove after migration to Rust build dir
let build_path = path.join("build"); // TODO: Remove after migration to Rust build dir
let mut dependencies = Vec::new();
// FIXME: the following iterator gymnastics port the exact order and behavior of the JavaScript implementation
let names = [
"osrm-extract",
"osrm-contract",
"osrm-customize",
"osrm-partition",
"osrm_extract",
"osrm_contract",
"osrm_customize",
"osrm_partition",
];
let files: Vec<PathBuf> = fs::read_dir(build_path)
.unwrap()
.filter_map(|e| e.ok())
.map(|dir_entry| dir_entry.path())
.collect();
let iter = names.iter().map(|name| {
files
.iter()
.find(|path_buf| {
path_buf
.file_stem()
.unwrap()
.to_str()
.unwrap()
.contains(name)
})
.cloned()
.expect("file exists and is usable")
});
dependencies.extend(iter);
let profiles_path = path.join("profiles");
println!("{profiles_path:?}");
dependencies.extend(
LexicographicFileWalker::new(&profiles_path)
.filter(|pb| !pb.to_str().unwrap().contains("examples"))
.filter(|pathbuf| match pathbuf.extension() {
Some(ext) => ext.to_str().unwrap() == "lua",
None => false,
}),
);
let mut md5 = chksum_md5::new();
println!("md5: {}", md5.digest().to_hex_lowercase());
for path_buf in dependencies {
let data = get_file_as_byte_vec(&path_buf);
if data.is_empty() {
continue;
}
md5.update(data);
// println!("md5: {}", md5.digest().to_hex_lowercase());
}
debug!("arguments: {:?}", args);
futures::executor::block_on(
OSRMWorld::cucumber()
@ -338,9 +261,9 @@ fn main() {
.before(move |feature, _rule, scenario, world| {
world.scenario_id = common::scenario_id::scenario_id(scenario);
world.set_scenario_specific_paths_and_digests(feature.path.clone());
world.osrm_digest = md5.digest().to_hex_lowercase();
world.osrm_digest = md5_of_osrm_executables().digest().to_hex_lowercase();
// TODO: clean up cache if needed
// TODO: clean up cache if needed? Or do in scenarios?
future::ready(()).boxed()
})