Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Implement parsing of flatbuffer response

Browse Source
This commit is contained in:
Dennis 2024-06-08 17:21:54 +02:00
parent c9b8462754
commit eba856af17
No known key found for this signature in database
GPG Key ID: 6937EAEA33A3FA5D
10 changed files with 242 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
Cargo.lock generated
View File

@ -93,6 +93,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.5.0"
@ -368,6 +374,15 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "encoding_rs"
version = "0.8.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"
dependencies = [
"cfg-if",
]
[[package]]
name = "errno"
version = "0.3.9"
@ -378,6 +393,25 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "flatbuffers"
version = "24.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f"
dependencies = [
"bitflags 1.3.2",
"rustc_version",
]
[[package]]
name = "flatc-rust"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57e61227926ef5b237af48bee74394cc4a5a221ebd10c5147a98e612f207851d"
dependencies = [
"log",
]
[[package]]
name = "flate2"
version = "1.0.30"
@ -544,7 +578,7 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757"
dependencies = [
"bitflags",
"bitflags 2.5.0",
"ignore",
"walkdir",
]
@ -768,6 +802,8 @@ dependencies = [
"clap",
"colored",
"cucumber",
"flatbuffers",
"flatc-rust",
"futures",
"geo-types",
"help",
@ -917,13 +953,22 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rustc_version"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
dependencies = [
"semver",
]
[[package]]
name = "rustix"
version = "0.38.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
dependencies = [
"bitflags",
"bitflags 2.5.0",
"errno",
"libc",
"linux-raw-sys",
@ -988,6 +1033,12 @@ dependencies = [
"syn 2.0.66",
]
[[package]]
name = "semver"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
[[package]]
name = "serde"
version = "1.0.203"
@ -1322,6 +1373,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d11a831e3c0b56e438a28308e7c810799e3c118417f342d30ecec080105395cd"
dependencies = [
"base64",
"encoding_rs",
"flate2",
"log",
"once_cell",

6
Cargo.toml
View File

@ -9,13 +9,14 @@ chksum-md5 = "0.0.0"
clap = "4.5.4"
colored = "2.1.0"
cucumber = { version = "0.21.0", features = ["tracing"] }
flatbuffers = "24.3.25"
futures = "0.3.30"
geo-types = "0.7.13"
help = "0.0.0"
log = "0.4.21"
serde = { version = "1.0.203", features = ["serde_derive"] }
serde_json = "1.0.117"
ureq = "2.9.7"
ureq = { version = "2.9.7", features = ["charset"] }
xml-builder = "0.5.2"
[[test]]
@ -24,3 +25,6 @@ harness = false
[profile.bench]
debug = true
[build-dependencies]
flatc-rust = "0.2.0"

20
build.rs Normal file
View File

@ -0,0 +1,20 @@
extern crate flatc_rust; // or just `use flatc_rust;` with Rust 2018 edition.
use std::path::Path;
fn main() {
println!("cargo:rerun-if-changed=include/engine/api/flatbuffers/");
flatc_rust::run(flatc_rust::Args {
extra: &["--gen-all"],
inputs: &[
Path::new("include/engine/api/flatbuffers/position.fbs"),
Path::new("include/engine/api/flatbuffers/waypoint.fbs"),
Path::new("include/engine/api/flatbuffers/route.fbs"),
Path::new("include/engine/api/flatbuffers/table.fbs"),
Path::new("include/engine/api/flatbuffers/fbresult.fbs"),
],
out_dir: Path::new("target/flatbuffers/"),
..Default::default()
})
.expect("flatc");
}

4
tests/common/f64_utils.rs
View File

@ -1,4 +1,4 @@
pub fn approx_equal(a: f64, b: f64, dp: u8) -> bool {
let p = 10f64.powi(-(dp as i32));
pub fn approx_equal(a: f32, b: f32, dp: u8) -> bool {
let p = 10f32.powi(-(dp as i32));
(a - b).abs() < p
}

8
tests/common/location.rs Normal file
View File

@ -0,0 +1,8 @@
use serde::Deserialize;
#[derive(Clone, Copy, Debug, Default, Deserialize)]
pub struct Location {
// Note: The order is important since we derive Deserialize
pub longitude: f32,
pub latitude: f32,
}

23
tests/common/mod.rs
View File

@ -1,12 +1,33 @@
#![allow(clippy::derivable_impls, clippy::all)]
extern crate flatbuffers;
pub mod cli_arguments;
pub mod dot_writer;
pub mod f64_utils;
pub mod file_util;
pub mod hash_util;
pub mod lexicographic_file_walker;
pub mod location;
pub mod nearest_response;
pub mod osm;
pub mod osm_db;
pub mod osrm_world;
pub mod scenario_id;
pub mod task_starter;
pub mod task_starter;
// flatbuffer
#[allow(dead_code, unused_imports)]
#[path = "../../target/flatbuffers/position_generated.rs"]
pub mod position_flatbuffers;
#[allow(dead_code, unused_imports)]
#[path = "../../target/flatbuffers/waypoint_generated.rs"]
pub mod waypoint_flatbuffers;
#[allow(dead_code, unused_imports)]
#[path = "../../target/flatbuffers/table_generated.rs"]
pub mod table_flatbuffers;
#[allow(dead_code, unused_imports)]
#[path = "../../target/flatbuffers/route_generated.rs"]
pub mod route_flatbuffers;
#[allow(dead_code, unused_imports)]
#[path = "../../target/flatbuffers/fbresult_generated.rs"]
pub mod fbresult_flatbuffers;

73
tests/common/nearest_response.rs
View File

@ -1,18 +1,19 @@
use geo_types::{point, Point};
use crate::common::fbresult_flatbuffers::osrm::engine::api::fbresult::FBResult;
use super::location::Location;
use serde::Deserialize;
#[derive(Deserialize, Debug)]
pub struct Waypoint {
pub hint: String,
pub nodes: Vec<u64>,
pub distance: f64,
pub distance: f32,
pub name: String,
location: [f64; 2],
location: Location,
}
impl Waypoint {
pub fn location(&self) -> Point {
point!(self.location)
pub fn location(&self) -> &Location {
&self.location
}
}
@ -22,3 +23,65 @@ pub struct NearestResponse {
pub waypoints: Vec<Waypoint>,
pub data_version: Option<String>,
}
impl NearestResponse {
pub fn from_json_reader(reader: impl std::io::Read) -> Self {
let response = match serde_json::from_reader::<_, NearestResponse>(reader) {
Ok(response) => response,
Err(e) => panic!("parsing error {e}"),
};
response
}
pub fn from_flatbuffer(mut reader: impl std::io::Read) -> Self {
let mut buffer = Vec::new();
if let Err(e) = reader.read_to_end(&mut buffer) {
panic!("cannot read from strem: {e}");
};
let decoded: Result<FBResult, flatbuffers::InvalidFlatbuffer> =
flatbuffers::root::<FBResult>(&buffer);
let decoded: FBResult = match decoded {
Ok(d) => d,
Err(e) => panic!("Error during parsing: {e} {:?}", buffer),
};
let code = match decoded.code() {
Some(e) => e.message().expect("code exists but is not unwrappable"),
None => "",
};
let data_version = match decoded.data_version() {
Some(s) => s,
None => "",
};
let waypoints = decoded
.waypoints()
.expect("waypoints should be at least an empty list")
.iter()
.map(|wp| {
let hint = wp.hint().expect("hint is missing").to_string();
let location = wp.location().expect("waypoint must have a location");
let location = Location {
latitude: location.latitude(),
longitude: location.longitude(),
};
let nodes = wp.nodes().expect("waypoint mus have nodes");
let nodes = vec![nodes.first(), nodes.second()];
let distance = wp.distance();
Waypoint {
hint,
nodes,
distance,
name: "".into(),
location,
}
})
.collect();
Self {
code: code.into(),
waypoints,
data_version: Some(data_version.into()),
}
}
}

13
tests/common/osm.rs
View File

@ -2,6 +2,8 @@ use std::collections::HashMap;
use xml_builder::XMLElement;
use super::location::Location;
static OSM_USER: &str = "osrm";
static OSM_TIMESTAMP: &str = "2000-01-01T00:00:00Z";
static OSM_UID: &str = "1";
@ -9,8 +11,7 @@ static OSM_UID: &str = "1";
#[derive(Clone, Debug, Default)]
pub struct OSMNode {
pub id: u64,
pub lat: f64,
pub lon: f64,
pub location: Location,
pub tags: HashMap<String, String>,
}
@ -34,8 +35,8 @@ impl OSMNode {
node.add_attribute("uid", OSM_UID);
node.add_attribute("user", OSM_USER);
node.add_attribute("timestamp", OSM_TIMESTAMP);
node.add_attribute("lon", &format!("{:?}", self.lon));
node.add_attribute("lat", &format!("{:?}", self.lat));
node.add_attribute("lon", &format!("{:?}", self.location.longitude));
node.add_attribute("lat", &format!("{:?}", self.location.latitude));
if !self.tags.is_empty() {
for (key, value) in &self.tags {
@ -81,8 +82,8 @@ impl OSMWay {
let mut nd = XMLElement::new("nd");
nd.add_attribute("ref", &node.id.to_string());
if self.add_locations {
nd.add_attribute("lon", &format!("{:?}", node.lon));
nd.add_attribute("lat", &format!("{:?}", node.lat));
nd.add_attribute("lon", &format!("{:?}", node.location.longitude));
nd.add_attribute("lat", &format!("{:?}", node.location.latitude));
}
way.add_child(nd).unwrap();
}

17
tests/common/osrm_world.rs
View File

@ -1,4 +1,4 @@
use crate::Point;
use crate::Location;
use cucumber::World;
use log::debug;
use std::{
@ -18,11 +18,13 @@ pub struct OSRMWorld {
pub osm_id: u64,
pub profile: String,
pub known_osm_nodes: HashMap<char, Point>,
pub known_locations: HashMap<char, Point>,
pub known_osm_nodes: HashMap<char, Location>,
pub known_locations: HashMap<char, Location>,
pub osm_db: OSMDb,
pub extraction_parameters: Vec<String>,
pub request_with_flatbuffers: bool,
}
impl OSRMWorld {
@ -78,7 +80,7 @@ impl OSRMWorld {
self.osm_id
}
pub fn add_osm_node(&mut self, name: char, location: Point, id: Option<u64>) {
pub fn add_osm_node(&mut self, name: char, location: Location, id: Option<u64>) {
if self.known_osm_nodes.contains_key(&name) {
panic!("duplicate node: {name}");
}
@ -88,8 +90,7 @@ impl OSRMWorld {
};
let node = OSMNode {
id,
lat: location.y(),
lon: location.x(),
location,
tags: HashMap::from([("name".to_string(), name.to_string())]),
};
@ -97,7 +98,7 @@ impl OSRMWorld {
self.osm_db.add_node(node);
}
pub fn get_location(&self, name: char) -> Point {
pub fn get_location(&self, name: char) -> Location {
*match name {
// TODO: move lookup to world
'0'..='9' => self
@ -112,7 +113,7 @@ impl OSRMWorld {
}
}
pub fn add_location(&mut self, name: char, location: Point) {
pub fn add_location(&mut self, name: char, location: Location) {
if self.known_locations.contains_key(&name) {
panic!("duplicate location: {name}")
}

74
tests/cucumber.rs
View File

@ -1,4 +1,4 @@
extern crate clap;
// extern crate clap;
mod common;
@ -6,13 +6,13 @@ use cheap_ruler::CheapRuler;
use clap::Parser;
use common::{
cli_arguments::Args, dot_writer::DotWriter, f64_utils::approx_equal,
hash_util::md5_of_osrm_executables, nearest_response::NearestResponse, osm::OSMWay,
osrm_world::OSRMWorld, task_starter::TaskStarter,
hash_util::md5_of_osrm_executables, location::Location, nearest_response::NearestResponse,
osm::OSMWay, osrm_world::OSRMWorld, task_starter::TaskStarter,
};
use core::panic;
use cucumber::{gherkin::Step, given, when, World, WriterExt};
use futures::{future, FutureExt};
use geo_types::{point, Point};
use geo_types::point;
use log::debug;
use std::{collections::HashMap, fs::File, io::Write, time::Duration};
use ureq::Agent;
@ -20,13 +20,17 @@ use ureq::Agent;
const DEFAULT_ORIGIN: [f64; 2] = [1., 1.]; // TODO: move to world?
const DEFAULT_GRID_SIZE: f64 = 100.; // TODO: move to world?
fn offset_origin_by(dx: f64, dy: f64) -> geo_types::Point {
fn offset_origin_by(dx: f64, dy: f64) -> Location {
let ruler = CheapRuler::new(DEFAULT_ORIGIN[1], cheap_ruler::DistanceUnit::Meters);
ruler.offset(
let loc = ruler.offset(
&point!(DEFAULT_ORIGIN),
dx * DEFAULT_GRID_SIZE,
dy * DEFAULT_GRID_SIZE,
) //TODO: needs to be world's gridSize, not the local one
); //TODO: needs to be world's gridSize, not the local one
Location {
latitude: loc.y() as f32,
longitude: loc.x() as f32,
}
}
#[given(expr = "the profile \"{word}\"")]
@ -60,13 +64,16 @@ fn set_node_locations(world: &mut OSRMWorld, step: &Step) {
);
});
table.rows.iter().skip(1).for_each(|row|{
table.rows.iter().skip(1).for_each(|row| {
assert_eq!(3, row.len());
assert_eq!(row[0].len(), 1, "node name not in [0..9][a..z]");
let name = &row[0].chars().next().expect("node name cannot be empty"); // the error is unreachable
let lon = &row[header_lookup["lon"]];
let lat = &row[header_lookup["lat"]];
let location = point!(x: lon.parse::<f64>().expect("lon {lon} needs to be a f64"), y: lat.parse::<f64>().expect("lat {lat} needs to be a f64"));
let location = Location {
latitude: lat.parse::<f32>().expect("lat {lat} needs to be a f64"),
longitude: lon.parse::<f32>().expect("lon {lon} needs to be a f64"),
};
match name {
'0'...'9' => world.add_location(*name, location),
'a'...'z' => world.add_osm_node(*name, location, None),
@ -115,7 +122,7 @@ fn set_ways(world: &mut OSRMWorld, step: &Step) {
panic!("empty way table provided")
}
// store a reference to the headers for convenient lookup
let headers = table.rows.first().unwrap();
let headers = table.rows.first().expect("table has a first row");
// iterate over the following rows and build ways one by one
table.rows.iter().skip(1).for_each(|row| {
@ -156,8 +163,10 @@ fn set_ways(world: &mut OSRMWorld, step: &Step) {
// debug!("{}", world.osm_db.to_xml())
}
#[when("I request nearest I should get")]
fn request_nearest(world: &mut OSRMWorld, step: &Step) {
// #[when("I request nearest I should get")]
#[when(regex = r"^I request nearest( with flatbuffers|) I should get$")]
fn request_nearest(world: &mut OSRMWorld, step: &Step, state: String) {
let request_with_flatbuffers = state == " with flatbuffers";
// if .osm file does not exist
// write osm file
@ -207,8 +216,8 @@ fn request_nearest(world: &mut OSRMWorld, step: &Step) {
let data_path = cache_path.join(world.scenario_id.to_owned() + ".osrm");
// TODO: this should not require a temporary and behave like the API of std::process
let mut task = TaskStarter::new(world.routed_path().to_str().unwrap());
task.arg(data_path.to_str().unwrap());
let mut task = TaskStarter::new(world.routed_path().to_str().expect("task can be started"));
task.arg(data_path.to_str().expect("data path unwrappable"));
task.spawn_wait_till_ready("running and waiting for requests");
assert!(task.is_ready());
@ -239,33 +248,44 @@ fn request_nearest(world: &mut OSRMWorld, step: &Step) {
// debug!("{query_location:?} => {expected_location:?}");
// run queries
let url = format!(
"http://localhost:5000/nearest/v1/{}/{},{}",
world.profile,
query_location.x(),
query_location.y()
let mut url = format!(
"http://localhost:5000/nearest/v1/{}/{:?},{:?}",
world.profile, query_location.longitude, query_location.latitude
);
if request_with_flatbuffers {
url += ".flatbuffers";
}
let call = agent.get(&url).call();
let body = match call {
Ok(response) => response.into_string().expect("response not parseable"),
Ok(response) => response.into_reader(),
Err(e) => panic!("http error: {e}"),
};
// debug!("body: {body}");
let response: NearestResponse = match serde_json::from_str(&body) {
Ok(response) => response,
Err(e) => panic!("parsing error {e}"),
let response = match request_with_flatbuffers {
true => NearestResponse::from_flatbuffer(body),
false => NearestResponse::from_json_reader(body),
};
if test_case.contains_key("out") {
// check that result node is (approximately) equivalent
let result_location = response.waypoints[0].location();
assert!(approx_equal(result_location.x(), expected_location.x(), 5));
assert!(approx_equal(result_location.y(), expected_location.y(), 5));
assert!(approx_equal(
result_location.longitude,
expected_location.longitude,
5
));
assert!(approx_equal(
result_location.latitude,
expected_location.latitude,
5
));
}
if test_case.contains_key("data_version") {
assert_eq!(test_case.get("data_version"), response.data_version.as_ref());
assert_eq!(
test_case.get("data_version"),
response.data_version.as_ref()
);
}
}
}