From 12e1bd80b303fc637b9a15d8f6e6956f59ff6817 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 29 Apr 2015 14:00:17 +0200 Subject: [PATCH 01/87] Send travis mails to me --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ed023e4e9..f42e7485d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,7 +58,7 @@ notifications: skip_join: false recipients: - - dennis@mapbox.com + - patrick@mapbox.com email: on_success: change on_failure: always From 1ed2c16a51bcadbfc69e03df22be984b2f11ade7 Mon Sep 17 00:00:00 2001 From: Emil Tin Date: Thu, 30 Apr 2015 14:54:14 +0200 Subject: [PATCH 02/87] upgrade to cucumber 2.0 --- Gemfile.lock | 27 ++++++++++++-------- features/step_definitions/distance_matrix.rb | 2 +- features/step_definitions/locate.rb | 2 +- features/step_definitions/nearest.rb | 2 +- features/step_definitions/routability.rb | 2 +- features/step_definitions/routing.rb | 2 +- features/support/cucumber.rb | 4 +-- features/support/hooks.rb | 16 ++++-------- 8 files changed, 28 insertions(+), 29 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index ac056a0a2..3363e9218 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,22 +2,27 @@ GEM remote: http://rubygems.org/ specs: builder (3.2.2) - cucumber (1.3.8) + cucumber (2.0.0) builder (>= 2.1.2) + cucumber-core (~> 1.1.3) diff-lcs (>= 1.1.3) - gherkin (~> 2.12.1) + gherkin (~> 2.12) multi_json (>= 1.7.5, < 2.0) - multi_test (>= 0.0.2) - diff-lcs (1.2.4) - gherkin (2.12.1) + multi_test (>= 0.1.2) + cucumber-core (1.1.3) + gherkin (~> 2.12.0) + diff-lcs (1.2.5) + gherkin (2.12.2) multi_json (~> 1.3) - multi_json (1.8.0) - multi_test (0.0.2) + multi_json (1.11.0) + multi_test (0.1.2) osmlib-base (0.1.4) - rake (10.1.0) - rspec-expectations (2.14.3) - diff-lcs (>= 1.1.3, < 2.0) - sys-proctable (0.9.3) + rake (10.4.2) + rspec-expectations (3.2.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.2.0) + rspec-support (3.2.2) + sys-proctable (0.9.8) PLATFORMS ruby diff --git a/features/step_definitions/distance_matrix.rb b/features/step_definitions/distance_matrix.rb index f85bdcc07..8c4e927de 100644 --- a/features/step_definitions/distance_matrix.rb +++ b/features/step_definitions/distance_matrix.rb @@ -52,5 +52,5 @@ When /^I request a travel time matrix I should get$/ do |table| actual << r end end - table.routing_diff! actual + table.diff! actual end diff --git a/features/step_definitions/locate.rb b/features/step_definitions/locate.rb index 3e67be3c9..190795c26 100644 --- a/features/step_definitions/locate.rb +++ b/features/step_definitions/locate.rb @@ -39,7 +39,7 @@ When /^I request locate I should get$/ do |table| actual << got end end - table.routing_diff! actual + table.diff! actual end When /^I request locate (\d+) times I should get$/ do |n,table| diff --git a/features/step_definitions/nearest.rb b/features/step_definitions/nearest.rb index 532c1f0d7..099f4e06f 100644 --- a/features/step_definitions/nearest.rb +++ b/features/step_definitions/nearest.rb @@ -39,7 +39,7 @@ When /^I request nearest I should get$/ do |table| actual << got end end - table.routing_diff! actual + table.diff! actual end When /^I request nearest (\d+) times I should get$/ do |n,table| diff --git a/features/step_definitions/routability.rb b/features/step_definitions/routability.rb index dcbde5afa..a9241bef7 100644 --- a/features/step_definitions/routability.rb +++ b/features/step_definitions/routability.rb @@ -74,5 +74,5 @@ Then /^routability should be$/ do |table| actual << output_row end end - table.routing_diff! actual + table.diff! actual end diff --git a/features/step_definitions/routing.rb b/features/step_definitions/routing.rb index c25f5b7c8..597ea65d6 100644 --- a/features/step_definitions/routing.rb +++ b/features/step_definitions/routing.rb @@ -140,7 +140,7 @@ When /^I route I should get$/ do |table| actual << got end end - table.routing_diff! actual + table.diff! actual end When /^I route (\d+) times I should get$/ do |n,table| diff --git a/features/support/cucumber.rb b/features/support/cucumber.rb index a43082a7a..09df5498d 100644 --- a/features/support/cucumber.rb +++ b/features/support/cucumber.rb @@ -1,8 +1,8 @@ #monkey patch cucumber table class to reorder output. #we always want failed rows to be shown right below the expected row. -class Cucumber::Ast::Table - def routing_diff!(other_table, options={}) +class Cucumber::MultilineArgument::DataTable + def diff!(other_table, options={}) options = {:missing_row => true, :surplus_row => true, :missing_col => true, :surplus_col => false}.merge(options) other_table = ensure_table(other_table) diff --git a/features/support/hooks.rb b/features/support/hooks.rb index 337014246..2ad49dcae 100644 --- a/features/support/hooks.rb +++ b/features/support/hooks.rb @@ -4,22 +4,16 @@ STRESS_TIMEOUT = 300 Before do |scenario| - # feature name + # fetch scenario and feature name, so we can use it in log files if needed case scenario - when Cucumber::Ast::Scenario + when Cucumber::RunningTestCase::Scenario @feature_name = scenario.feature.name - when Cucumber::Ast::OutlineTable::ExampleRow - @feature_name = scenario.scenario_outline.feature.name - end - - # scenario name - case scenario - when Cucumber::Ast::Scenario @scenario_title = scenario.name - when Cucumber::Ast::OutlineTable::ExampleRow + when Cucumber::RunningTestCase::ExampleRow + @feature_name = scenario.scenario_outline.feature.name @scenario_title = scenario.scenario_outline.name end - + @load_method = DEFAULT_LOAD_METHOD @query_params = {} @scenario_time = Time.now.strftime("%Y-%m-%dT%H:%m:%SZ") From e1a13f5ce80358727b1afeb1287455232d5a6ea3 Mon Sep 17 00:00:00 2001 From: Emil Tin Date: Thu, 30 Apr 2015 15:07:28 +0200 Subject: [PATCH 03/87] remove obsolete cucumber patch --- features/support/cucumber.rb | 76 ------------------------------------ 1 file changed, 76 deletions(-) delete mode 100644 features/support/cucumber.rb diff --git a/features/support/cucumber.rb b/features/support/cucumber.rb deleted file mode 100644 index 09df5498d..000000000 --- a/features/support/cucumber.rb +++ /dev/null @@ -1,76 +0,0 @@ -#monkey patch cucumber table class to reorder output. -#we always want failed rows to be shown right below the expected row. - -class Cucumber::MultilineArgument::DataTable - def diff!(other_table, options={}) - options = {:missing_row => true, :surplus_row => true, :missing_col => true, :surplus_col => false}.merge(options) - - other_table = ensure_table(other_table) - other_table.convert_headers! - other_table.convert_columns! - ensure_green! - - convert_headers! - convert_columns! - - original_width = cell_matrix[0].length - other_table_cell_matrix = pad!(other_table.cell_matrix) - padded_width = cell_matrix[0].length - - missing_col = cell_matrix[0].detect{|cell| cell.status == :undefined} - surplus_col = padded_width > original_width - - require_diff_lcs - cell_matrix.extend(Diff::LCS) - changes = cell_matrix.diff(other_table_cell_matrix).flatten - - inserted = 0 - missing = 0 - - row_indices = Array.new(other_table_cell_matrix.length) {|n| n} - - last_change = nil - missing_row_pos = nil - insert_row_pos = nil - - changes.each do |change| - if(change.action == '-') - missing_row_pos = change.position + inserted - cell_matrix[missing_row_pos].each{|cell| cell.status = :undefined} - row_indices.insert(missing_row_pos, nil) - missing += 1 - else # '+' - #change index so we interleave instead - insert_row_pos = change.position + inserted + 1 - #insert_row_pos = change.position + missing #original - - inserted_row = change.element - inserted_row.each{|cell| cell.status = :comment} - cell_matrix.insert(insert_row_pos, inserted_row) - row_indices[insert_row_pos] = nil - inspect_rows(cell_matrix[missing_row_pos], inserted_row) if last_change && last_change.action == '-' - inserted += 1 - end - last_change = change - end - - other_table_cell_matrix.each_with_index do |other_row, i| - row_index = row_indices.index(i) - row = cell_matrix[row_index] if row_index - if row - (original_width..padded_width).each do |col_index| - surplus_cell = other_row[col_index] - row[col_index].value = surplus_cell.value if row[col_index] - end - end - end - - clear_cache! - should_raise = - missing_row_pos && options[:missing_row] || - insert_row_pos && options[:surplus_row] || - missing_col && options[:missing_col] || - surplus_col && options[:surplus_col] - raise Different.new(self) if should_raise - end -end \ No newline at end of file From 71197e1c890d320126722116e2df2dfee1aaf5dd Mon Sep 17 00:00:00 2001 From: Emil Tin Date: Thu, 30 Apr 2015 18:29:29 +0200 Subject: [PATCH 04/87] remove last use of routing_diff --- features/step_definitions/matching.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/step_definitions/matching.rb b/features/step_definitions/matching.rb index 3f3181f64..e9b723460 100644 --- a/features/step_definitions/matching.rb +++ b/features/step_definitions/matching.rb @@ -91,6 +91,6 @@ When /^I match I should get$/ do |table| actual << got end end - table.routing_diff! actual + table.diff! actual end From e2954211ecacb0b265820daae654b114f58a3298 Mon Sep 17 00:00:00 2001 From: Will White Date: Fri, 1 May 2015 11:53:08 -0400 Subject: [PATCH 05/87] Fix travis links. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 624cc1da7..77305fbb9 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ When using the code in a (scientific) publication, please cite | build config | branch | status | |:-------------|:--------|:------------| -| Linux | master | [![Build Status](https://travis-ci.org/Project-OSRM/osrm-backend.png?branch=master)](https://travis-ci.org/DennisOSRM/Project-OSRM) | -| Linux | develop | [![Build Status](https://travis-ci.org/Project-OSRM/osrm-backend.png?branch=develop)](https://travis-ci.org/DennisOSRM/Project-OSRM) | +| Linux | master | [![Build Status](https://travis-ci.org/Project-OSRM/osrm-backend.png?branch=master)](https://travis-ci.org/Project-OSRM/osrm-backend) | +| Linux | develop | [![Build Status](https://travis-ci.org/Project-OSRM/osrm-backend.png?branch=develop)](https://travis-ci.org/Project-OSRM/osrm-backend) | | Windows | master/develop | [![Build status](https://ci.appveyor.com/api/projects/status/4iuo3s9gxprmcjjh)](https://ci.appveyor.com/project/DennisOSRM/osrm-backend) | | LUAbind fork | master | [![Build Status](https://travis-ci.org/DennisOSRM/luabind.png?branch=master)](https://travis-ci.org/DennisOSRM/luabind) | From 80e07943eea141e13a56b4372010aa6afc468040 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 2 May 2015 01:30:44 +0200 Subject: [PATCH 06/87] Make README more meaningful --- README.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 77305fbb9..cd9737374 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,21 @@ -# Readme +## About -For instructions on how to compile and run OSRM, please consult the Wiki at +The Open Source Routing Machine is a high performance routing engine written in C++11 designed to run on OpenStreetMap data. -https://github.com/Project-OSRM/osrm-backend/wiki +## Current build status -or use our free and daily updated online service at +| build config | branch | status | +|:-------------|:--------|:------------| +| Linux | master | [![Build Status](https://travis-ci.org/Project-OSRM/osrm-backend.png?branch=master)](https://travis-ci.org/Project-OSRM/osrm-backend) | +| Linux | develop | [![Build Status](https://travis-ci.org/Project-OSRM/osrm-backend.png?branch=develop)](https://travis-ci.org/Project-OSRM/osrm-backend) | +| Windows | master/develop | [![Build status](https://ci.appveyor.com/api/projects/status/4iuo3s9gxprmcjjh)](https://ci.appveyor.com/project/DennisOSRM/osrm-backend) | +| LUAbind fork | master | [![Build Status](https://travis-ci.org/DennisOSRM/luabind.png?branch=master)](https://travis-ci.org/DennisOSRM/luabind) | -http://map.project-osrm.org +## Building + +For instructions on how to [build](https://github.com/Project-OSRM/osrm-backend/wiki/Building-OSRM) and [run OSRM](https://github.com/Project-OSRM/osrm-backend/wiki/Running-OSRM), please consult [the Wiki](https://github.com/Project-OSRM/osrm-backend/wiki). + +To quickly try OSRM use our [free and daily updated online service](http://map.project-osrm.org) ## References in publications @@ -31,11 +40,3 @@ When using the code in a (scientific) publication, please cite } ``` -## Current build status - -| build config | branch | status | -|:-------------|:--------|:------------| -| Linux | master | [![Build Status](https://travis-ci.org/Project-OSRM/osrm-backend.png?branch=master)](https://travis-ci.org/Project-OSRM/osrm-backend) | -| Linux | develop | [![Build Status](https://travis-ci.org/Project-OSRM/osrm-backend.png?branch=develop)](https://travis-ci.org/Project-OSRM/osrm-backend) | -| Windows | master/develop | [![Build status](https://ci.appveyor.com/api/projects/status/4iuo3s9gxprmcjjh)](https://ci.appveyor.com/project/DennisOSRM/osrm-backend) | -| LUAbind fork | master | [![Build Status](https://travis-ci.org/DennisOSRM/luabind.png?branch=master)](https://travis-ci.org/DennisOSRM/luabind) | From d0175798bf48b1f663fced1cfa50b56edb877a6b Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 2 May 2015 15:18:13 +0200 Subject: [PATCH 07/87] Add link to documentation in the wiki --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index cd9737374..d319d26bc 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,10 @@ For instructions on how to [build](https://github.com/Project-OSRM/osrm-backend/ To quickly try OSRM use our [free and daily updated online service](http://map.project-osrm.org) +## Documentation + +See the Wiki's [server API documentation](https://github.com/Project-OSRM/osrm-backend/wiki/Server-api) as well as the [library API documentation](https://github.com/Project-OSRM/osrm-backend/wiki/Library-api) + ## References in publications When using the code in a (scientific) publication, please cite From a457d69034a63880c2a28fe1048198d7b175e68d Mon Sep 17 00:00:00 2001 From: alex85k Date: Sat, 9 May 2015 19:22:16 +0500 Subject: [PATCH 08/87] fix cucumber tests running on Windows --- features/support/env.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/features/support/env.rb b/features/support/env.rb index f9a4fe436..ea74959ed 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -22,10 +22,10 @@ SHUTDOWN_TIMEOUT = 10 DEFAULT_LOAD_METHOD = 'datastore' OSRM_ROUTED_LOG_FILE = 'osrm-routed.log' -if ENV['OS']==/Windows.*/ then - TERMSIGNAL='TERM' -else +if ENV['OS']=~/Windows.*/ then TERMSIGNAL=9 +else + TERMSIGNAL='TERM' end @@ -72,8 +72,8 @@ end def verify_existance_of_binaries ["osrm-extract", "osrm-prepare", "osrm-routed"].each do |bin| - unless File.exists? "#{BIN_PATH}/#{bin}" - raise "*** #{BIN_PATH}/#{bin} is missing. Build failed?" + unless File.exists? "#{BIN_PATH}/#{bin}#{EXE}" + raise "*** #{BIN_PATH}/#{bin}#{EXE} is missing. Build failed?" end end end From 079eea3f2bc8c5726cf668cee1c5898ec769c7da Mon Sep 17 00:00:00 2001 From: Emil Tin Date: Sun, 10 May 2015 20:04:14 +0200 Subject: [PATCH 09/87] add test for route duration formats --- features/testbot/ferry.feature | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/features/testbot/ferry.feature b/features/testbot/ferry.feature index 55160614b..21cefcb6f 100644 --- a/features/testbot/ferry.feature +++ b/features/testbot/ferry.feature @@ -173,3 +173,33 @@ Feature: Testbot - Handle ferry routes | from | to | route | time | | a | g | abcdefg | 23400s +-2 | | g | a | abcdefg | 23400s +-2 | + + @todo + Scenario: Testbot - Ferry duration formats + Given the node map + | a | c | e | g | i | k | m | o | q | s | + | b | d | f | h | j | l | n | p | r | t | + + And the ways + | nodes | route | duration | + | ab | ferry | 0:01 | + | cd | ferry | 00:01 | + | ef | ferry | 1:00 | + | gh | ferry | 01:00 | + | ij | ferry | 02:20 | + | kl | ferry | 10:00 | + | mn | ferry | 100:00 | + | op | ferry | 1000:00 | + | qr | ferry | 10000:00 | + + When I route I should get + | from | to | route | time | + | a | b | ab | 60s +-1 | + | c | d | cd | 60s +-1 | + | e | f | ef | 3600s +-1 | + | g | h | gh | 3600s +-1 | + | i | j | ij | 8400s +-1 | + | k | l | kl | 36000s +-1 | + | m | n | mn | 360000s +-1 | + | o | p | mn | 3600000s +-1 | + | q | r | mn | 36000000s +-1 | From c4c6ab24946be8355ffc7ea528dce537b88ab8c6 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 14 May 2015 22:24:07 +0200 Subject: [PATCH 10/87] Use distance based search radius This limits the nearest neighbour search to a maximum distance of 1000 meters, but will also work in dense areas. --- data_structures/static_rtree.hpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/data_structures/static_rtree.hpp b/data_structures/static_rtree.hpp index f219a6475..929ceb8bd 100644 --- a/data_structures/static_rtree.hpp +++ b/data_structures/static_rtree.hpp @@ -646,7 +646,7 @@ class StaticRTree const FixedPointCoordinate &input_coordinate, std::vector &result_phantom_node_vector, const unsigned max_number_of_phantom_nodes, - const unsigned max_checked_elements = 4 * LEAF_NODE_SIZE) + const float max_distance = 1000) { unsigned inspected_elements = 0; unsigned number_of_elements_from_big_cc = 0; @@ -669,6 +669,10 @@ class StaticRTree while (!traversal_queue.empty()) { const IncrementalQueryCandidate current_query_node = traversal_queue.top(); + if (current_query_node.min_dist > max_distance) + { + break; + } traversal_queue.pop(); if (current_query_node.node.template is()) @@ -771,9 +775,8 @@ class StaticRTree } // stop the search by flushing the queue - if ((result_phantom_node_vector.size() >= max_number_of_phantom_nodes && - number_of_elements_from_big_cc > 0) || - inspected_elements >= max_checked_elements) + if (result_phantom_node_vector.size() >= max_number_of_phantom_nodes && + number_of_elements_from_big_cc > 0) { traversal_queue = std::priority_queue{}; } From 1b0d8739c1486d6a8c77108f057332ca17429e3b Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 15 May 2015 00:34:34 +0200 Subject: [PATCH 11/87] Increase max distance to 1100 to fix unrelated test cases --- data_structures/static_rtree.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_structures/static_rtree.hpp b/data_structures/static_rtree.hpp index 929ceb8bd..f7a959a20 100644 --- a/data_structures/static_rtree.hpp +++ b/data_structures/static_rtree.hpp @@ -646,7 +646,7 @@ class StaticRTree const FixedPointCoordinate &input_coordinate, std::vector &result_phantom_node_vector, const unsigned max_number_of_phantom_nodes, - const float max_distance = 1000) + const float max_distance = 1100) { unsigned inspected_elements = 0; unsigned number_of_elements_from_big_cc = 0; From 7b1a5566fbff0c19279e8cc8e6f959ab0d981904 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 15 May 2015 00:34:53 +0200 Subject: [PATCH 12/87] Move nodes inside the search radius --- features/nearest/pick.feature | 20 ++++++++--------- features/testbot/snap.feature | 42 ++++++++++++++++------------------- 2 files changed, 28 insertions(+), 34 deletions(-) diff --git a/features/nearest/pick.feature b/features/nearest/pick.feature index 08e665243..75dbc6ea1 100644 --- a/features/nearest/pick.feature +++ b/features/nearest/pick.feature @@ -57,14 +57,13 @@ Feature: Locating Nearest node on a Way - pick closest way Scenario: Nearest - High lat/lon Given the node locations - | node | lat | lon | - | a | -85 | -180 | - | b | -85 | -160 | - | c | -85 | -140 | - | x | 75 | -180 | - | y | 75 | -180 | - | v | 1 | 1 | - | w | -1 | -1 | + | node | lat | lon | + | a | -85 | -180 | + | b | -85 | -160 | + | c | -85 | -140 | + | x | -84.999 | -180 | + | y | -84.999 | -160 | + | z | -84.999 | -140 | And the ways | nodes | @@ -73,6 +72,5 @@ Feature: Locating Nearest node on a Way - pick closest way When I request nearest I should get | in | out | | x | a | - | y | a | - | v | c | - | w | c | + | y | b | + | z | c | diff --git a/features/testbot/snap.feature b/features/testbot/snap.feature index bee75bc44..01c590be5 100644 --- a/features/testbot/snap.feature +++ b/features/testbot/snap.feature @@ -98,30 +98,26 @@ Feature: Snap start/end point to the nearest way | b | x | xb | | c | x | xc | - Scenario: Find edges within 1km, and the same from 10km - Given a grid size of 1000 meters + Scenario: Find edges within 100m, and the same from 1km + Given a grid size of 100 meters Given the node map - | p | | | | | | | | | | | i | | | | | | | | | | | j | - | | | | | | | | | | | | | | | | | | | | | | | | - | | | | | | | | | | | | | | | | | | | | | | | | - | | | | | | | | | | | | | | | | | | | | | | | | - | | | | | | | | | | | | | | | | | | | | | | | | - | | | | | | | | | | | | | | | | | | | | | | | | - | | | | | | | | | | | | | | | | | | | | | | | | - | | | | | | | | | | 8 | | 1 | | 2 | | | | | | | | | | - | | | | | | | | | | | h | a | b | | | | | | | | | | | - | o | | | | | | | | | 7 | g | x | c | 3 | | | | | | | | | k | - | | | | | | | | | | | f | e | d | | | | | | | | | | | - | | | | | | | | | | 6 | | 5 | | 4 | | | | | | | | | | - | | | | | | | | | | | | | | | | | | | | | | | | - | | | | | | | | | | | | | | | | | | | | | | | | - | | | | | | | | | | | | | | | | | | | | | | | | - | | | | | | | | | | | | | | | | | | | | | | | | - | | | | | | | | | | | | | | | | | | | | | | | | - | | | | | | | | | | | | | | | | | | | | | | | | - | | | | | | | | | | | | | | | | | | | | | | | | - | | | | | | | | | | | | | | | | | | | | | | | | - | n | | | | | | | | | | | m | | | | | | | | | | | l | + | p | | | | | | | | i | | | | | | | | j | + | | | | | | | | | | | | | | | | | | + | | | | | | | | | | | | | | | | | | + | | | | | | | | | | | | | | | | | | + | | | | | | | | | | | | | | | | | | + | | | | | | | | | | | | | | | | | | + | | | | | | | 8 | | 1 | | 2 | | | | | | | + | | | | | | | | h | a | b | | | | | | | | + | o | | | | | | 7 | g | x | c | 3 | | | | | | k | + | | | | | | | | f | e | d | | | | | | | | + | | | | | | | 6 | | 5 | | 4 | | | | | | | + | | | | | | | | | | | | | | | | | | + | | | | | | | | | | | | | | | | | | + | | | | | | | | | | | | | | | | | | + | | | | | | | | | | | | | | | | | | + | | | | | | | | | | | | | | | | | | + | n | | | | | | | | m | | | | | | | | l | Given the ways | nodes | From 566ab993df4e5e05920ceff6dab1da2448bfc68b Mon Sep 17 00:00:00 2001 From: Lauren Budorick Date: Fri, 1 May 2015 14:06:45 -0400 Subject: [PATCH 13/87] Use lua 5.2+ without needing compatibility flags. --- profiles/bicycle.lua | 13 +++++++------ profiles/car.lua | 16 ++++------------ profiles/foot.lua | 7 ++++--- profiles/lib/access.lua | 8 +++++--- profiles/lib/maxspeed.lua | 6 ++++-- 5 files changed, 24 insertions(+), 26 deletions(-) diff --git a/profiles/bicycle.lua b/profiles/bicycle.lua index 497321ccf..dc290e11a 100644 --- a/profiles/bicycle.lua +++ b/profiles/bicycle.lua @@ -1,5 +1,7 @@ -require("lib/access") -require("lib/maxspeed") +-- Bicycle profile + +local find_access_tag = require("lib/access").find_access_tag +local limit = require("lib/maxspeed").limit -- Begin of globals barrier_whitelist = { [""] = true, ["cycle_barrier"] = true, ["bollard"] = true, ["entrance"] = true, ["cattle_grid"] = true, ["border_control"] = true, ["toll_booth"] = true, ["sally_port"] = true, ["gate"] = true, ["no"] = true } @@ -122,7 +124,6 @@ local function parse_maxspeed(source) return n end - function get_exceptions(vector) for i,v in ipairs(restriction_exception_tags) do vector:Add(v) @@ -131,7 +132,7 @@ end function node_function (node, result) local barrier = node:get_value_by_key("barrier") - local access = Access.find_access_tag(node, access_tags_hierachy) + local access = find_access_tag(node, access_tags_hierachy) local traffic_signal = node:get_value_by_key("highway") -- flag node if it carries a traffic light @@ -181,7 +182,7 @@ function way_function (way, result) end -- access - local access = Access.find_access_tag(way, access_tags_hierachy) + local access = find_access_tag(way, access_tags_hierachy) if access and access_tag_blacklist[access] then return end @@ -391,7 +392,7 @@ function way_function (way, result) end -- maxspeed - MaxSpeed.limit( result, maxspeed, maxspeed_forward, maxspeed_backward ) + limit( result, maxspeed, maxspeed_forward, maxspeed_backward ) end function turn_function (angle) diff --git a/profiles/car.lua b/profiles/car.lua index 813afd19c..6b77575ed 100644 --- a/profiles/car.lua +++ b/profiles/car.lua @@ -1,6 +1,8 @@ --- Begin of globals ---require("lib/access") --function temporarily inlined +-- Car profile +local find_access_tag = require("lib/access").find_access_tag + +-- Begin of globals barrier_whitelist = { ["cattle_grid"] = true, ["border_control"] = true, ["checkpoint"] = true, ["toll_booth"] = true, ["sally_port"] = true, ["gate"] = true, ["lift_gate"] = true, ["no"] = true, ["entrance"] = true } access_tag_whitelist = { ["yes"] = true, ["motorcar"] = true, ["motor_vehicle"] = true, ["vehicle"] = true, ["permissive"] = true, ["designated"] = true } access_tag_blacklist = { ["no"] = true, ["private"] = true, ["agricultural"] = true, ["forestry"] = true, ["emergency"] = true, ["psv"] = true } @@ -147,16 +149,6 @@ local mode_normal = 1 local mode_ferry = 2 local mode_movable_bridge = 3 -local function find_access_tag(source, access_tags_hierachy) - for i,v in ipairs(access_tags_hierachy) do - local access_tag = source:get_value_by_key(v) - if access_tag and "" ~= access_tag then - return access_tag - end - end - return "" -end - function get_exceptions(vector) for i,v in ipairs(restriction_exception_tags) do vector:Add(v) diff --git a/profiles/foot.lua b/profiles/foot.lua index 5a370971a..3133cb843 100644 --- a/profiles/foot.lua +++ b/profiles/foot.lua @@ -1,7 +1,8 @@ -- Foot profile -require("lib/access") +local find_access_tag = require("lib/access").find_access_tag +-- Begin of globals barrier_whitelist = { [""] = true, ["cycle_barrier"] = true, ["bollard"] = true, ["entrance"] = true, ["cattle_grid"] = true, ["border_control"] = true, ["toll_booth"] = true, ["sally_port"] = true, ["gate"] = true, ["no"] = true} access_tag_whitelist = { ["yes"] = true, ["foot"] = true, ["permissive"] = true, ["designated"] = true } access_tag_blacklist = { ["no"] = true, ["private"] = true, ["agricultural"] = true, ["forestery"] = true } @@ -75,7 +76,7 @@ end function node_function (node, result) local barrier = node:get_value_by_key("barrier") - local access = Access.find_access_tag(node, access_tags_hierachy) + local access = find_access_tag(node, access_tags_hierachy) local traffic_signal = node:get_value_by_key("highway") -- flag node if it carries a traffic light @@ -125,7 +126,7 @@ function way_function (way, result) end -- access - local access = Access.find_access_tag(way, access_tags_hierachy) + local access = find_access_tag(way, access_tags_hierachy) if access_tag_blacklist[access] then return end diff --git a/profiles/lib/access.lua b/profiles/lib/access.lua index 76d2e2c89..3fd4ff605 100644 --- a/profiles/lib/access.lua +++ b/profiles/lib/access.lua @@ -1,13 +1,15 @@ local ipairs = ipairs -module "Access" +local Access = {} -function find_access_tag(source,access_tags_hierachy) +function Access.find_access_tag(source,access_tags_hierachy) for i,v in ipairs(access_tags_hierachy) do local tag = source:get_value_by_key(v) if tag and tag ~= '' then return tag end end - return nil + return "" end + +return Access diff --git a/profiles/lib/maxspeed.lua b/profiles/lib/maxspeed.lua index aca344a79..0dd9b822d 100644 --- a/profiles/lib/maxspeed.lua +++ b/profiles/lib/maxspeed.lua @@ -1,8 +1,8 @@ local math = math -module "MaxSpeed" +local MaxSpeed = {} -function limit(way,max,maxf,maxb) +function MaxSpeed.limit(way,max,maxf,maxb) if maxf and maxf>0 then way.forward_speed = math.min(way.forward_speed, maxf) elseif max and max>0 then @@ -15,3 +15,5 @@ function limit(way,max,maxf,maxb) way.backward_speed = math.min(way.backward_speed, max) end end + +return MaxSpeed From f46b600ec0e883a37303c45fbe071afbd6d00e22 Mon Sep 17 00:00:00 2001 From: Lauren Budorick Date: Fri, 1 May 2015 14:24:07 -0400 Subject: [PATCH 14/87] Upgrade lua to 5.2 on travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f42e7485d..bff8cfc54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ install: - sudo apt-add-repository -y ppa:ubuntu-toolchain-r/test - sudo add-apt-repository -y ppa:boost-latest/ppa - sudo apt-get update >/dev/null - - sudo apt-get -q install protobuf-compiler libprotoc-dev libprotobuf7 libprotobuf-dev libbz2-dev libstxxl-dev libstxxl1 libxml2-dev libzip-dev lua5.1 liblua5.1-0-dev rubygems libtbb-dev + - sudo apt-get -q install protobuf-compiler libprotoc-dev libprotobuf7 libprotobuf-dev libbz2-dev libstxxl-dev libstxxl1 libxml2-dev libzip-dev lua5.2 liblua5.2-dev rubygems libtbb-dev - sudo apt-get -q install g++-4.8 - sudo apt-get install libboost1.54-all-dev - sudo apt-get install libgdal-dev From 95088a785ab66ac21baf41f5e17e6c6e3014c566 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 1 May 2015 21:01:21 +0200 Subject: [PATCH 15/87] Make using profile.lua work again. Symbolic links are a _bad_ idea with lua script. Lua will search at the place of the symbolic link for modules _not_ at the actual location of the script. --- util/lua_util.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/lua_util.hpp b/util/lua_util.hpp index af6277c59..c68c427cf 100644 --- a/util/lua_util.hpp +++ b/util/lua_util.hpp @@ -59,7 +59,7 @@ inline void luaAddScriptFolderToLoadPath(lua_State *lua_state, const char *file_ std::string folder = profile_path.parent_path().string(); // TODO: This code is most probably not Windows safe since it uses UNIX'ish path delimiters const std::string lua_code = - "package.path = \"" + folder + "/?.lua;profiles/?.lua;\" .. package.path"; + "package.path = \"" + folder + "/?.lua;" + folder + "/profiles/?.lua;profiles/?.lua;\" .. package.path"; luaL_dostring(lua_state, lua_code.c_str()); } From fd76fba235c3deacdd8327d9198fee57f23e7f15 Mon Sep 17 00:00:00 2001 From: Lauren Budorick Date: Fri, 1 May 2015 16:14:07 -0400 Subject: [PATCH 16/87] Keep apt-get 5.1 for now --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bff8cfc54..f42e7485d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ install: - sudo apt-add-repository -y ppa:ubuntu-toolchain-r/test - sudo add-apt-repository -y ppa:boost-latest/ppa - sudo apt-get update >/dev/null - - sudo apt-get -q install protobuf-compiler libprotoc-dev libprotobuf7 libprotobuf-dev libbz2-dev libstxxl-dev libstxxl1 libxml2-dev libzip-dev lua5.2 liblua5.2-dev rubygems libtbb-dev + - sudo apt-get -q install protobuf-compiler libprotoc-dev libprotobuf7 libprotobuf-dev libbz2-dev libstxxl-dev libstxxl1 libxml2-dev libzip-dev lua5.1 liblua5.1-0-dev rubygems libtbb-dev - sudo apt-get -q install g++-4.8 - sudo apt-get install libboost1.54-all-dev - sudo apt-get install libgdal-dev From ed53888fce5dc510cf57193e090ce1f5cb85c108 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 15 May 2015 15:30:41 +0200 Subject: [PATCH 17/87] Follow symlinks --- util/lua_util.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/util/lua_util.hpp b/util/lua_util.hpp index c68c427cf..6dbdc8bc1 100644 --- a/util/lua_util.hpp +++ b/util/lua_util.hpp @@ -55,11 +55,10 @@ inline bool lua_function_exists(lua_State *lua_state, const char *name) // See http://lua-users.org/wiki/PackagePath for details on the package.path syntax. inline void luaAddScriptFolderToLoadPath(lua_State *lua_state, const char *file_name) { - const boost::filesystem::path profile_path(file_name); + boost::filesystem::path profile_path = boost::filesystem::canonical(file_name); std::string folder = profile_path.parent_path().string(); // TODO: This code is most probably not Windows safe since it uses UNIX'ish path delimiters - const std::string lua_code = - "package.path = \"" + folder + "/?.lua;" + folder + "/profiles/?.lua;profiles/?.lua;\" .. package.path"; + const std::string lua_code = "package.path = \"" + folder + "/?.lua;\" .. package.path"; luaL_dostring(lua_state, lua_code.c_str()); } From 7ad52de2b1b2e2c0ec89fb20ecfb7eaf27347fde Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 15 May 2015 14:45:50 +0200 Subject: [PATCH 18/87] Cleanup the profiles --- profiles/bicycle.lua | 99 ++++++++++++++++++++++---------------------- profiles/car.lua | 10 +---- 2 files changed, 52 insertions(+), 57 deletions(-) diff --git a/profiles/bicycle.lua b/profiles/bicycle.lua index dc290e11a..1e8e97928 100644 --- a/profiles/bicycle.lua +++ b/profiles/bicycle.lua @@ -12,10 +12,10 @@ access_tags_hierachy = { "bicycle", "vehicle", "access" } cycleway_tags = {["track"]=true,["lane"]=true,["opposite"]=true,["opposite_lane"]=true,["opposite_track"]=true,["share_busway"]=true,["sharrow"]=true,["shared"]=true } service_tag_restricted = { ["parking_aisle"] = true } restriction_exception_tags = { "bicycle", "vehicle", "access" } +unsafe_highway_list = { ["primary"] = true, ["secondary"] = true, ["tertiary"] = true, ["primary_link"] = true, ["secondary_link"] = true, ["tertiary_link"] = true} -default_speed = 15 - -walking_speed = 6 +local default_speed = 15 +local walking_speed = 6 bicycle_speeds = { ["cycleway"] = default_speed, @@ -91,24 +91,22 @@ surface_speeds = { ["sand"] = 3 } -take_minimum_of_speeds = true -obey_oneway = true -obey_bollards = false -use_restrictions = true -ignore_areas = true -- future feature -traffic_signal_penalty = 5 -u_turn_penalty = 20 -use_turn_restrictions = false -turn_penalty = 60 -turn_bias = 1.4 - +local obey_oneway = true +local obey_bollards = false +local ignore_areas = true +local u_turn_penalty = 20 +local turn_penalty = 60 +local turn_bias = 1.4 +-- reduce the driving speed by 30% for unsafe roads +local safety_penalty = 0.7 +local use_public_transport = false --modes -mode_normal = 1 -mode_pushing = 2 -mode_ferry = 3 -mode_train = 4 -mode_movable_bridge = 5 +local mode_normal = 1 +local mode_pushing = 2 +local mode_ferry = 3 +local mode_train = 4 +local mode_movable_bridge = 5 local function parse_maxspeed(source) if not source then @@ -131,29 +129,26 @@ function get_exceptions(vector) end function node_function (node, result) - local barrier = node:get_value_by_key("barrier") + -- parse access and barrier tags local access = find_access_tag(node, access_tags_hierachy) - local traffic_signal = node:get_value_by_key("highway") + if access ~= "" then + if access_tag_blacklist[access] then + result.barrier = true + end + else + local barrier = node:get_value_by_key("barrier") + if barrier and "" ~= barrier then + if not barrier_whitelist[barrier] then + result.barrier = true + end + end + end - -- flag node if it carries a traffic light - if traffic_signal and traffic_signal == "traffic_signals" then - result.traffic_lights = true - end - - -- parse access and barrier tags - if access and access ~= "" then - if access_tag_blacklist[access] then - result.barrier = true - else - result.barrier = false - end - elseif barrier and barrier ~= "" then - if barrier_whitelist[barrier] then - result.barrier = false - else - result.barrier = true - end - end + -- check if node is a traffic light + local tag = node:get_value_by_key("highway") + if tag and "traffic_signals" == tag then + result.traffic_lights = true; + end end function way_function (way, result) @@ -166,8 +161,8 @@ function way_function (way, result) local public_transport = way:get_value_by_key("public_transport") local bridge = way:get_value_by_key("bridge") if (not highway or highway == '') and - (not route or route == '') and - (not railway or railway=='') and + (not use_public_transport or not route or route == '') and + (not use_public_transport or not railway or railway=='') and (not amenity or amenity=='') and (not man_made or man_made=='') and (not public_transport or public_transport=='') and @@ -214,10 +209,11 @@ function way_function (way, result) result.name = ref elseif name and "" ~= name then result.name = name - elseif highway then - -- if no name exists, use way type - -- this encoding scheme is excepted to be a temporary solution - result.name = "{highway:"..highway.."}" +-- TODO find a better solution for encoding way type +-- elseif highway then +-- -- if no name exists, use way type +-- -- this encoding scheme is excepted to be a temporary solution +-- result.name = "{highway:"..highway.."}" end -- roundabout handling @@ -247,15 +243,16 @@ function way_function (way, result) result.forward_speed = route_speeds[route] result.backward_speed = route_speeds[route] end - elseif railway and platform_speeds[railway] then + -- public transport + if use_public_transport and railway and platform_speeds[railway] then -- railway platforms (old tagging scheme) result.forward_speed = platform_speeds[railway] result.backward_speed = platform_speeds[railway] - elseif platform_speeds[public_transport] then + elseif use_public_transport and platform_speeds[public_transport] then -- public_transport platforms (new tagging platform) result.forward_speed = platform_speeds[public_transport] result.backward_speed = platform_speeds[public_transport] - elseif railway and railway_speeds[railway] then + elseif use_public_transport and railway and railway_speeds[railway] then result.forward_mode = mode_train result.backward_mode = mode_train -- railways @@ -271,6 +268,10 @@ function way_function (way, result) -- regular ways result.forward_speed = bicycle_speeds[highway] result.backward_speed = bicycle_speeds[highway] + if unsafe_highway_list[highway] then + result.forward_speed *= safety_penalty + result.backward_speed *= safety_penalty + end elseif access and access_tag_whitelist[access] then -- unknown way, but valid access tag result.forward_speed = default_speed diff --git a/profiles/car.lua b/profiles/car.lua index 6b77575ed..4f4e081d9 100644 --- a/profiles/car.lua +++ b/profiles/car.lua @@ -132,10 +132,9 @@ maxspeed_table = { traffic_signal_penalty = 2 use_turn_restrictions = true -local take_minimum_of_speeds = false local obey_oneway = true local obey_bollards = true -local ignore_areas = true -- future feature +local ignore_areas = true local u_turn_penalty = 20 local abs = math.abs @@ -179,6 +178,7 @@ local function parse_maxspeed(source) return n end +-- FIXME Why was this commented out? -- function turn_function (angle) -- -- print ("called at angle " .. angle ) -- local index = math.abs(math.floor(angle/10+0.5))+1 -- +1 'coz LUA starts as idx 1 @@ -420,9 +420,3 @@ function way_function (way, result) end end --- These are wrappers to parse vectors of nodes and ways and thus to speed up any tracing JIT -function node_vector_function(vector) - for v in vector.nodes do - node_function(v) - end -end From f04a3e3d2e34dbfba15b30090c330118a55befa4 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 15 May 2015 15:35:48 +0200 Subject: [PATCH 19/87] Fix bicycle profile syntax --- profiles/bicycle.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/profiles/bicycle.lua b/profiles/bicycle.lua index 1e8e97928..9efcf7070 100644 --- a/profiles/bicycle.lua +++ b/profiles/bicycle.lua @@ -269,8 +269,8 @@ function way_function (way, result) result.forward_speed = bicycle_speeds[highway] result.backward_speed = bicycle_speeds[highway] if unsafe_highway_list[highway] then - result.forward_speed *= safety_penalty - result.backward_speed *= safety_penalty + result.forward_speed = result.forward_speed * safety_penalty + result.backward_speed = result.backward_speed * safety_penalty end elseif access and access_tag_whitelist[access] then -- unknown way, but valid access tag From aad846b968098f00ee70ab996db84b65a8a9658f Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 15 May 2015 16:35:35 +0200 Subject: [PATCH 20/87] Fix call to function and transportation if clause --- profiles/bicycle.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/profiles/bicycle.lua b/profiles/bicycle.lua index 9efcf7070..7b7e54177 100644 --- a/profiles/bicycle.lua +++ b/profiles/bicycle.lua @@ -130,7 +130,7 @@ end function node_function (node, result) -- parse access and barrier tags - local access = find_access_tag(node, access_tags_hierachy) + local access = Access.find_access_tag(node, access_tags_hierachy) if access ~= "" then if access_tag_blacklist[access] then result.barrier = true @@ -244,7 +244,7 @@ function way_function (way, result) result.backward_speed = route_speeds[route] end -- public transport - if use_public_transport and railway and platform_speeds[railway] then + elseif use_public_transport and railway and platform_speeds[railway] then -- railway platforms (old tagging scheme) result.forward_speed = platform_speeds[railway] result.backward_speed = platform_speeds[railway] From c778ab96226ba72e779ec55e9ec2f06c2d904f1d Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 15 May 2015 17:01:03 +0200 Subject: [PATCH 21/87] Make bicycle profile backwards compatible --- profiles/bicycle.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/profiles/bicycle.lua b/profiles/bicycle.lua index 7b7e54177..aa688f12e 100644 --- a/profiles/bicycle.lua +++ b/profiles/bicycle.lua @@ -98,8 +98,9 @@ local u_turn_penalty = 20 local turn_penalty = 60 local turn_bias = 1.4 -- reduce the driving speed by 30% for unsafe roads -local safety_penalty = 0.7 -local use_public_transport = false +-- local safety_penalty = 0.7 +local safety_penalty = 1.0 +local use_public_transport = true --modes local mode_normal = 1 @@ -268,7 +269,7 @@ function way_function (way, result) -- regular ways result.forward_speed = bicycle_speeds[highway] result.backward_speed = bicycle_speeds[highway] - if unsafe_highway_list[highway] then + if safety_penalty < 1 and unsafe_highway_list[highway] then result.forward_speed = result.forward_speed * safety_penalty result.backward_speed = result.backward_speed * safety_penalty end From 6166d946f7c89915d63a41cec37a2c5068654998 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 15 May 2015 17:01:21 +0200 Subject: [PATCH 22/87] Fix access tag check --- profiles/bicycle.lua | 2 +- profiles/car.lua | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/profiles/bicycle.lua b/profiles/bicycle.lua index aa688f12e..2781cc404 100644 --- a/profiles/bicycle.lua +++ b/profiles/bicycle.lua @@ -132,7 +132,7 @@ end function node_function (node, result) -- parse access and barrier tags local access = Access.find_access_tag(node, access_tags_hierachy) - if access ~= "" then + if access and access ~= "" then if access_tag_blacklist[access] then result.barrier = true end diff --git a/profiles/car.lua b/profiles/car.lua index 4f4e081d9..3e22afe04 100644 --- a/profiles/car.lua +++ b/profiles/car.lua @@ -190,7 +190,7 @@ end function node_function (node, result) -- parse access and barrier tags local access = find_access_tag(node, access_tags_hierachy) - if access ~= "" then + if access and access ~= "" then if access_tag_blacklist[access] then result.barrier = true end From de2f06970d291536a2a9b779a4c4db8bc1fb715f Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 16 May 2015 14:36:32 +0200 Subject: [PATCH 23/87] Fix missing values and activate fallback names by default --- profiles/bicycle.lua | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/profiles/bicycle.lua b/profiles/bicycle.lua index 2781cc404..aaf29d28c 100644 --- a/profiles/bicycle.lua +++ b/profiles/bicycle.lua @@ -91,6 +91,9 @@ surface_speeds = { ["sand"] = 3 } +traffic_signal_penalty = 2 +use_turn_restrictions = false + local obey_oneway = true local obey_bollards = false local ignore_areas = true @@ -101,6 +104,7 @@ local turn_bias = 1.4 -- local safety_penalty = 0.7 local safety_penalty = 1.0 local use_public_transport = true +local fallback_names = true --modes local mode_normal = 1 @@ -210,11 +214,11 @@ function way_function (way, result) result.name = ref elseif name and "" ~= name then result.name = name --- TODO find a better solution for encoding way type --- elseif highway then --- -- if no name exists, use way type --- -- this encoding scheme is excepted to be a temporary solution --- result.name = "{highway:"..highway.."}" + -- TODO find a better solution for encoding way type + elseif fallback_names and highway then + -- if no name exists, use way type + -- this encoding scheme is excepted to be a temporary solution + result.name = "{highway:"..highway.."}" end -- roundabout handling From 074c7a9c403e22a655c9ef0d05eb928aeb3d9fda Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sun, 17 May 2015 17:26:10 +0200 Subject: [PATCH 24/87] Fix access module --- profiles/bicycle.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/bicycle.lua b/profiles/bicycle.lua index aaf29d28c..9630a6e4c 100644 --- a/profiles/bicycle.lua +++ b/profiles/bicycle.lua @@ -135,7 +135,7 @@ end function node_function (node, result) -- parse access and barrier tags - local access = Access.find_access_tag(node, access_tags_hierachy) + local access = find_access_tag(node, access_tags_hierachy) if access and access ~= "" then if access_tag_blacklist[access] then result.barrier = true From dd33a45644dc86e39f1718bc3e37403d90f90e1d Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sun, 17 May 2015 22:59:05 +0200 Subject: [PATCH 25/87] Revert "fix incorrect behavior when via point was on same one-way street as destination but should have been reached before, closes #1424" This reopens #1424 but potentially fixes #1429. This reverts commit 11c671354be8c791ef107e6ce6c4a38435417edb. --- routing_algorithms/shortest_path.hpp | 58 +++++++++++++++++++++------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/routing_algorithms/shortest_path.hpp b/routing_algorithms/shortest_path.hpp index 3dedc2b76..4b3cc49a4 100644 --- a/routing_algorithms/shortest_path.hpp +++ b/routing_algorithms/shortest_path.hpp @@ -92,9 +92,7 @@ class ShortestPathRouting final const bool allow_u_turn = current_leg > 0 && uturn_indicators.size() > current_leg && uturn_indicators[current_leg - 1]; - const EdgeWeight min_edge_offset = - std::min(phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), - phantom_node_pair.source_phantom.GetReverseWeightPlusOffset()); + EdgeWeight min_edge_offset = 0; // insert new starting nodes into forward heap, adjusted by previous distances. if ((allow_u_turn || search_from_1st_node) && @@ -102,32 +100,58 @@ class ShortestPathRouting final { forward_heap1.Insert( phantom_node_pair.source_phantom.forward_node_id, - -phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), + (allow_u_turn ? 0 : distance1) - + phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), phantom_node_pair.source_phantom.forward_node_id); + min_edge_offset = + std::min(min_edge_offset, + (allow_u_turn ? 0 : distance1) - + phantom_node_pair.source_phantom.GetForwardWeightPlusOffset()); // SimpleLogger().Write(logDEBUG) << "fwd-a2 insert: " << - // phantom_node_pair.source_phantom.forward_node_id << ", w: " << -phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(); + // phantom_node_pair.source_phantom.forward_node_id << ", w: " << (allow_u_turn ? 0 + // : distance1) - phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(); forward_heap2.Insert( phantom_node_pair.source_phantom.forward_node_id, - -phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), + (allow_u_turn ? 0 : distance1) - + phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), phantom_node_pair.source_phantom.forward_node_id); + min_edge_offset = + std::min(min_edge_offset, + (allow_u_turn ? 0 : distance1) - + phantom_node_pair.source_phantom.GetForwardWeightPlusOffset()); // SimpleLogger().Write(logDEBUG) << "fwd-b2 insert: " << - // phantom_node_pair.source_phantom.forward_node_id << ", w: " << -phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(); + // phantom_node_pair.source_phantom.forward_node_id << ", w: " << (allow_u_turn ? 0 + // : distance1) - phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(); } if ((allow_u_turn || search_from_2nd_node) && phantom_node_pair.source_phantom.reverse_node_id != SPECIAL_NODEID) { forward_heap1.Insert( phantom_node_pair.source_phantom.reverse_node_id, - -phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(), + (allow_u_turn ? 0 : distance2) - + phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(), phantom_node_pair.source_phantom.reverse_node_id); + min_edge_offset = + std::min(min_edge_offset, + (allow_u_turn ? 0 : distance2) - + phantom_node_pair.source_phantom.GetReverseWeightPlusOffset()); // SimpleLogger().Write(logDEBUG) << "fwd-a2 insert: " << - // phantom_node_pair.source_phantom.reverse_node_id << ", w: " << -phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(); + // phantom_node_pair.source_phantom.reverse_node_id << + // ", w: " << (allow_u_turn ? 0 : distance2) - + // phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(); forward_heap2.Insert( phantom_node_pair.source_phantom.reverse_node_id, - -phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(), + (allow_u_turn ? 0 : distance2) - + phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(), phantom_node_pair.source_phantom.reverse_node_id); + min_edge_offset = + std::min(min_edge_offset, + (allow_u_turn ? 0 : distance2) - + phantom_node_pair.source_phantom.GetReverseWeightPlusOffset()); // SimpleLogger().Write(logDEBUG) << "fwd-b2 insert: " << - // phantom_node_pair.source_phantom.reverse_node_id << ", w: " << -phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(); + // phantom_node_pair.source_phantom.reverse_node_id << + // ", w: " << (allow_u_turn ? 0 : distance2) - + // phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(); } // insert new backward nodes into backward heap, unadjusted. @@ -137,7 +161,9 @@ class ShortestPathRouting final phantom_node_pair.target_phantom.GetForwardWeightPlusOffset(), phantom_node_pair.target_phantom.forward_node_id); // SimpleLogger().Write(logDEBUG) << "rev-a insert: " << - // phantom_node_pair.target_phantom.forward_node_id << ", w: " << phantom_node_pair.target_phantom.GetForwardWeightPlusOffset(); + // phantom_node_pair.target_phantom.forward_node_id << + // ", w: " << + // phantom_node_pair.target_phantom.GetForwardWeightPlusOffset(); } if (phantom_node_pair.target_phantom.reverse_node_id != SPECIAL_NODEID) @@ -146,7 +172,9 @@ class ShortestPathRouting final phantom_node_pair.target_phantom.GetReverseWeightPlusOffset(), phantom_node_pair.target_phantom.reverse_node_id); // SimpleLogger().Write(logDEBUG) << "rev-a insert: " << - // phantom_node_pair.target_phantom.reverse_node_id << ", w: " << phantom_node_pair.target_phantom.GetReverseWeightPlusOffset(); + // phantom_node_pair.target_phantom.reverse_node_id << + // ", w: " << + // phantom_node_pair.target_phantom.GetReverseWeightPlusOffset(); } // run two-Target Dijkstra routing step. @@ -304,8 +332,8 @@ class ShortestPathRouting final BOOST_ASSERT(search_from_1st_node != search_from_2nd_node); } - distance1 += local_upper_bound1; - distance2 += local_upper_bound2; + distance1 = local_upper_bound1; + distance2 = local_upper_bound2; ++current_leg; } From fd9bb3ac43c0f6051a21657825cb7006cfabb7bc Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 18 May 2015 09:41:41 +0200 Subject: [PATCH 26/87] Only activate pruning for big cc after one was found --- data_structures/static_rtree.hpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/data_structures/static_rtree.hpp b/data_structures/static_rtree.hpp index f7a959a20..8f1a2389d 100644 --- a/data_structures/static_rtree.hpp +++ b/data_structures/static_rtree.hpp @@ -651,6 +651,9 @@ class StaticRTree unsigned inspected_elements = 0; unsigned number_of_elements_from_big_cc = 0; unsigned number_of_elements_from_tiny_cc = 0; + // is true if a big cc was added to the queue to we also have a lower bound + // for them. it actives pruning for big components + bool has_big_cc = false; #ifdef NDEBUG unsigned pruned_elements = 0; @@ -697,10 +700,11 @@ class StaticRTree BOOST_ASSERT(0.f <= current_perpendicular_distance); if (pruning_bound.get() >= current_perpendicular_distance || - current_edge.is_in_tiny_cc()) + (!has_big_cc && !current_edge.is_in_tiny_cc())) { pruning_bound.insert(current_perpendicular_distance); traversal_queue.emplace(current_perpendicular_distance, current_edge); + has_big_cc = has_big_cc || !current_edge.is_in_tiny_cc(); } #ifdef NDEBUG else @@ -812,6 +816,10 @@ class StaticRTree unsigned number_of_elements_from_big_cc = 0; unsigned number_of_elements_from_tiny_cc = 0; + // is true if a big cc was added to the queue to we also have a lower bound + // for them. it actives pruning for big components + bool has_big_cc = false; + unsigned pruned_elements = 0; std::pair projected_coordinate = { @@ -852,10 +860,11 @@ class StaticRTree BOOST_ASSERT(0.f <= current_perpendicular_distance); if (pruning_bound.get() >= current_perpendicular_distance || - current_edge.is_in_tiny_cc()) + (!has_big_cc && !current_edge.is_in_tiny_cc())) { pruning_bound.insert(current_perpendicular_distance); traversal_queue.emplace(current_perpendicular_distance, current_edge); + has_big_cc = has_big_cc || !current_edge.is_in_tiny_cc(); } else { From eab87c0827bac8c9bdd6b67c7c7dcb7157d62abf Mon Sep 17 00:00:00 2001 From: Daniel Patterson Date: Thu, 21 May 2015 15:39:23 -0700 Subject: [PATCH 27/87] Return error message when lua error occurs. The error may not be the first item in the stack while we're inside the error handler. ::from_stack() works OK outside the error callback, but not inside. --- extractor/restriction_parser.cpp | 2 +- extractor/scripting_environment.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/restriction_parser.cpp b/extractor/restriction_parser.cpp index 77dae9ce3..9b209b719 100644 --- a/extractor/restriction_parser.cpp +++ b/extractor/restriction_parser.cpp @@ -44,7 +44,7 @@ namespace { int lua_error_callback(lua_State *lua_state) { - luabind::object error_msg(luabind::from_stack(lua_state, -1)); + std::string error_msg = lua_tostring(lua_state, -1); std::ostringstream error_stream; error_stream << error_msg; throw osrm::exception("ERROR occured in profile script:\n" + error_stream.str()); diff --git a/extractor/scripting_environment.cpp b/extractor/scripting_environment.cpp index 972772246..b32ab4b25 100644 --- a/extractor/scripting_environment.cpp +++ b/extractor/scripting_environment.cpp @@ -53,7 +53,7 @@ auto get_value_by_key(T const &object, const char *key) -> decltype(object.get_v int lua_error_callback(lua_State *L) // This is so I can use my own function as an // exception handler, pcall_log() { - luabind::object error_msg(luabind::from_stack(L, -1)); + std::string error_msg = lua_tostring(L, -1); std::ostringstream error_stream; error_stream << error_msg; throw osrm::exception("ERROR occured in profile script:\n" + error_stream.str()); From 4c03ace9ebc91d5e2b00ed844d2e4e72be9e00bb Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 25 May 2015 02:07:01 +0200 Subject: [PATCH 28/87] Remove pruning in IncrementalFindPhantomNode --- data_structures/static_rtree.hpp | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/data_structures/static_rtree.hpp b/data_structures/static_rtree.hpp index 8f1a2389d..48ba20120 100644 --- a/data_structures/static_rtree.hpp +++ b/data_structures/static_rtree.hpp @@ -651,20 +651,11 @@ class StaticRTree unsigned inspected_elements = 0; unsigned number_of_elements_from_big_cc = 0; unsigned number_of_elements_from_tiny_cc = 0; - // is true if a big cc was added to the queue to we also have a lower bound - // for them. it actives pruning for big components - bool has_big_cc = false; -#ifdef NDEBUG - unsigned pruned_elements = 0; -#endif std::pair projected_coordinate = { mercator::lat2y(input_coordinate.lat / COORDINATE_PRECISION), input_coordinate.lon / COORDINATE_PRECISION}; - // upper bound pruning technique - upper_bound pruning_bound(max_number_of_phantom_nodes); - // initialize queue with root element std::priority_queue traversal_queue; traversal_queue.emplace(0.f, m_search_tree[0]); @@ -699,19 +690,7 @@ class StaticRTree // distance must be non-negative BOOST_ASSERT(0.f <= current_perpendicular_distance); - if (pruning_bound.get() >= current_perpendicular_distance || - (!has_big_cc && !current_edge.is_in_tiny_cc())) - { - pruning_bound.insert(current_perpendicular_distance); - traversal_queue.emplace(current_perpendicular_distance, current_edge); - has_big_cc = has_big_cc || !current_edge.is_in_tiny_cc(); - } -#ifdef NDEBUG - else - { - ++pruned_elements; - } -#endif + traversal_queue.emplace(current_perpendicular_distance, current_edge); } } else From 6a08d93e2c9993b87fc43c1bc2d7ea19f845fab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Wed, 27 May 2015 15:40:10 +0200 Subject: [PATCH 29/87] http post requests implemented --- server/request_parser.cpp | 50 +++++++++++++++++++++++++++++++++++++-- server/request_parser.hpp | 7 +++++- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/server/request_parser.cpp b/server/request_parser.cpp index 584dcbeeb..d57dd305b 100644 --- a/server/request_parser.cpp +++ b/server/request_parser.cpp @@ -42,7 +42,7 @@ namespace http RequestParser::RequestParser() : state(internal_state::method_start), current_header({"", ""}), - selected_compression(no_compression) + selected_compression(no_compression), is_post_header(false) { } @@ -58,6 +58,11 @@ RequestParser::parse(request ¤t_request, char *begin, char *end) } } osrm::tribool result = osrm::tribool::indeterminate; + + if(is_post_header && (begin == end)) + { + result = osrm::tribool::yes; + } return std::make_tuple(result, selected_compression); } @@ -70,8 +75,38 @@ osrm::tribool RequestParser::consume(request ¤t_request, const char input) { return osrm::tribool::no; } + if(input == 'P') + { + state = internal_state::post_O; + return osrm::tribool::indeterminate; + } state = internal_state::method; return osrm::tribool::indeterminate; + case internal_state::post_O: + if(input == 'O') + { + state = internal_state::post_S; + return osrm::tribool::indeterminate; + } + return osrm::tribool::no; + case internal_state::post_S: + if(input == 'S') + { + state = internal_state::post_T; + return osrm::tribool::indeterminate; + } + return osrm::tribool::no; + case internal_state::post_T: + if(input == 'T') + { + is_post_header = true; + state = internal_state::method; + return osrm::tribool::indeterminate; + } + return osrm::tribool::no; + case internal_state::post_request: + current_request.uri.push_back(input); + return osrm::tribool::indeterminate; case internal_state::method: if (input == ' ') { @@ -272,7 +307,18 @@ osrm::tribool RequestParser::consume(request ¤t_request, const char input) return osrm::tribool::indeterminate; } return osrm::tribool::no; - default: // expecting_newline_3 + case internal_state::expecting_newline_3: + if(input == '\n') + { + if(is_post_header) + { + state = internal_state::post_request; + return osrm::tribool::indeterminate; + } + return osrm::tribool::yes; + } + return osrm::tribool::no; + default: // should never be reached return input == '\n' ? osrm::tribool::yes : osrm::tribool::no; } } diff --git a/server/request_parser.hpp b/server/request_parser.hpp index 2b6bf6944..86b08c4cc 100644 --- a/server/request_parser.hpp +++ b/server/request_parser.hpp @@ -80,11 +80,16 @@ class RequestParser space_before_header_value, header_value, expecting_newline_2, - expecting_newline_3 + expecting_newline_3, + post_O, + post_S, + post_T, + post_request } state; header current_header; compression_type selected_compression; + bool is_post_header; }; } // namespace http From 1c7397fb213e585c72794dc3492e45307e5b2475 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 28 May 2015 12:31:59 +0200 Subject: [PATCH 30/87] Enable barrier check for TarjanSCC Re-enabling turn restrictions as well requires some further work to extend the algorithm. --- algorithms/tiny_components.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algorithms/tiny_components.hpp b/algorithms/tiny_components.hpp index 8eb61a912..8abc1cbc6 100644 --- a/algorithms/tiny_components.hpp +++ b/algorithms/tiny_components.hpp @@ -150,7 +150,7 @@ template class TarjanSCC // Traverse outgoing edges if (barrier_node_set.find(v) != barrier_node_set.end() && u != vprime) { - // continue; + continue; } if (to_node_of_only_restriction != std::numeric_limits::max() && From cf3b8d09d94197ab618d9ccf7aa858d858ae218b Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 12 May 2015 23:43:41 +0200 Subject: [PATCH 31/87] Also run the algorithm tests on travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index f42e7485d..55e719035 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,7 @@ script: - make - make tests - make benchmarks + - ./algorithm-tests - ./datastructure-tests - cd .. - cucumber -p verify From 5c0a964321deb05aec10a5ca29b5afeacad000d4 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 20 Apr 2015 16:24:55 +0200 Subject: [PATCH 32/87] Remove unused code SimpleNodeBasedDynamicGraph --- data_structures/node_based_graph.hpp | 85 ---------------------------- 1 file changed, 85 deletions(-) diff --git a/data_structures/node_based_graph.hpp b/data_structures/node_based_graph.hpp index 54e07a7ec..bd71362c1 100644 --- a/data_structures/node_based_graph.hpp +++ b/data_structures/node_based_graph.hpp @@ -72,14 +72,7 @@ struct NodeBasedEdgeData } }; -struct SimpleEdgeData -{ - SimpleEdgeData() : capacity(0) {} - EdgeWeight capacity; -}; - using NodeBasedDynamicGraph = DynamicGraph; -using SimpleNodeBasedDynamicGraph = DynamicGraph; // Factory method to create NodeBasedDynamicGraph from ImportEdges inline std::shared_ptr @@ -198,82 +191,4 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector -inline std::shared_ptr -SimpleNodeBasedDynamicGraphFromEdges(int number_of_nodes, std::vector &input_edge_list) -{ - static_assert(sizeof(NodeBasedEdgeData) == 16, - "changing node based edge data size changes memory consumption"); - tbb::parallel_sort(input_edge_list.begin(), input_edge_list.end()); - - DeallocatingVector edges_list; - SimpleNodeBasedDynamicGraph::InputEdge edge; - edge.data.capacity = 1; - for (const SimpleEdgeT &import_edge : input_edge_list) - { - if (import_edge.source == import_edge.target) - { - continue; - } - edge.source = import_edge.source; - edge.target = import_edge.target; - edges_list.push_back(edge); - std::swap(edge.source, edge.target); - edges_list.push_back(edge); - } - - // remove duplicate edges - tbb::parallel_sort(edges_list.begin(), edges_list.end()); - NodeID edge_count = 0; - for (NodeID i = 0; i < edges_list.size();) - { - const NodeID source = edges_list[i].source; - const NodeID target = edges_list[i].target; - // remove eigenloops - if (source == target) - { - i++; - continue; - } - SimpleNodeBasedDynamicGraph::InputEdge forward_edge; - SimpleNodeBasedDynamicGraph::InputEdge reverse_edge; - forward_edge = reverse_edge = edges_list[i]; - forward_edge.data.capacity = reverse_edge.data.capacity = INVALID_EDGE_WEIGHT; - // remove parallel edges - while (i < edges_list.size() && edges_list[i].source == source && - edges_list[i].target == target) - { - forward_edge.data.capacity = - std::min(edges_list[i].data.capacity, forward_edge.data.capacity); - reverse_edge.data.capacity = - std::min(edges_list[i].data.capacity, reverse_edge.data.capacity); - ++i; - } - // merge edges (s,t) and (t,s) into bidirectional edge - if (forward_edge.data.capacity == reverse_edge.data.capacity) - { - if (static_cast(forward_edge.data.capacity) != INVALID_EDGE_WEIGHT) - { - edges_list[edge_count++] = forward_edge; - } - } - else - { // insert seperate edges - if (static_cast(forward_edge.data.capacity) != INVALID_EDGE_WEIGHT) - { - edges_list[edge_count++] = forward_edge; - } - if (static_cast(reverse_edge.data.capacity) != INVALID_EDGE_WEIGHT) - { - edges_list[edge_count++] = reverse_edge; - } - } - } - SimpleLogger().Write() << "merged " << edges_list.size() - edge_count << " edges out of " - << edges_list.size(); - - auto graph = std::make_shared(number_of_nodes, edges_list); - return graph; -} - #endif // NODE_BASED_GRAPH_HPP From 1164a65df8eaca19fc5aecef03ed25c013cdc7cf Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 21 Apr 2015 10:43:02 +0200 Subject: [PATCH 33/87] Refactor processing_chain by splitting into sub functions --- CMakeLists.txt | 2 +- contractor/edge_based_graph_factory.cpp | 25 +- contractor/edge_based_graph_factory.hpp | 17 +- contractor/processing_chain.cpp | 269 ++++++++++--------- contractor/processing_chain.hpp | 44 +-- data_structures/node_based_graph.hpp | 3 +- data_structures/restriction_map.hpp | 7 +- data_structures/static_rtree.hpp | 2 +- util/graph_loader.hpp | 340 ++++++------------------ 9 files changed, 272 insertions(+), 437 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 503171a59..dd862d8dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,7 +90,7 @@ set( add_library(COORDINATE OBJECT ${CoordinateGlob}) add_library(GITDESCRIPTION OBJECT util/git_sha.cpp) -add_library(OSRM ${OSRMSources} $ $ $ $ $ $ $ $ $) +add_library(OSRM ${OSRMSources} $ $ $ $ $ $ $ $ $ $) add_library(FINGERPRINT OBJECT util/fingerprint.cpp) add_dependencies(FINGERPRINT FingerPrintConfigure) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index 268bb9f8d..241d43ec9 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -40,21 +40,21 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -EdgeBasedGraphFactory::EdgeBasedGraphFactory( - const std::shared_ptr &node_based_graph, - std::unique_ptr restriction_map, - std::vector &barrier_node_list, - std::vector &traffic_light_node_list, - std::vector &node_info_list, - SpeedProfileProperties &speed_profile) +EdgeBasedGraphFactory::EdgeBasedGraphFactory(std::shared_ptr node_based_graph, + std::shared_ptr restriction_map, + std::unique_ptr> barrier_node_list, + std::unique_ptr> traffic_light_node_list, + const std::vector &node_info_list, + const SpeedProfileProperties &speed_profile) : speed_profile(speed_profile), m_number_of_edge_based_nodes(std::numeric_limits::max()), - m_node_info_list(node_info_list), m_node_based_graph(node_based_graph), + m_node_info_list(node_info_list), + m_node_based_graph(std::move(node_based_graph)), m_restriction_map(std::move(restriction_map)), max_id(0), removed_node_count(0) { // insert into unordered sets for fast lookup - m_barrier_nodes.insert(barrier_node_list.begin(), barrier_node_list.end()); - m_traffic_lights.insert(traffic_light_node_list.begin(), traffic_light_node_list.end()); + m_barrier_nodes.insert(barrier_node_list->begin(), barrier_node_list->end()); + m_traffic_lights.insert(traffic_light_node_list->begin(), traffic_light_node_list->end()); } void EdgeBasedGraphFactory::GetEdgeBasedEdges(DeallocatingVector &output_edge_list) @@ -362,11 +362,11 @@ void EdgeBasedGraphFactory::CompressGeometry() // update any involved turn restrictions m_restriction_map->FixupStartingTurnRestriction(node_u, node_v, node_w); m_restriction_map->FixupArrivingTurnRestriction(node_u, node_v, node_w, - m_node_based_graph); + *m_node_based_graph); m_restriction_map->FixupStartingTurnRestriction(node_w, node_v, node_u); m_restriction_map->FixupArrivingTurnRestriction(node_w, node_v, node_u, - m_node_based_graph); + *m_node_based_graph); // store compressed geometry in container m_geometry_compressor.CompressEdge( @@ -415,6 +415,7 @@ void EdgeBasedGraphFactory::RenumberEdges() for (const auto current_edge : m_node_based_graph->GetAdjacentEdgeRange(current_node)) { EdgeData &edge_data = m_node_based_graph->GetEdgeData(current_edge); + // FIXME when does that happen? why can we skip here? if (!edge_data.forward) { continue; diff --git a/contractor/edge_based_graph_factory.hpp b/contractor/edge_based_graph_factory.hpp index 95b65babc..a5274396f 100644 --- a/contractor/edge_based_graph_factory.hpp +++ b/contractor/edge_based_graph_factory.hpp @@ -59,12 +59,12 @@ class EdgeBasedGraphFactory struct SpeedProfileProperties; - explicit EdgeBasedGraphFactory(const std::shared_ptr &node_based_graph, - std::unique_ptr restricion_map, - std::vector &barrier_node_list, - std::vector &traffic_light_node_list, - std::vector &node_info_list, - SpeedProfileProperties &speed_profile); + explicit EdgeBasedGraphFactory(std::shared_ptr node_based_graph, + std::shared_ptr restricion_map, + std::unique_ptr> barrier_node_list, + std::unique_ptr> traffic_light_node_list, + const std::vector &node_info_list, + const SpeedProfileProperties &speed_profile); void Run(const std::string &original_edge_data_filename, const std::string &geometry_filename, @@ -97,15 +97,16 @@ class EdgeBasedGraphFactory unsigned m_number_of_edge_based_nodes; - std::vector m_node_info_list; std::vector m_edge_based_node_list; DeallocatingVector m_edge_based_edge_list; + const std::vector& m_node_info_list; std::shared_ptr m_node_based_graph; + std::shared_ptr m_restriction_map; + std::unordered_set m_barrier_nodes; std::unordered_set m_traffic_lights; - std::unique_ptr m_restriction_map; GeometryCompressor m_geometry_compressor; diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index ccff7fd09..7ad682944 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -63,17 +63,25 @@ Prepare::~Prepare() {} int Prepare::Process(int argc, char *argv[]) { +#ifdef WIN32 +#pragma message("Memory consumption on Windows can be higher due to different bit packing") +#else + static_assert(sizeof(ImportEdge) == 20, + "changing ImportEdge type has influence on memory consumption!"); + static_assert(sizeof(EdgeBasedEdge) == 16, + "changing EdgeBasedEdge type has influence on memory consumption!"); +#endif + LogPolicy::GetInstance().Unmute(); TIMER_START(preparing); - TIMER_START(expansion); if (!ParseArguments(argc, argv)) { return 0; } - if (!boost::filesystem::is_regular_file(input_path)) + if (!boost::filesystem::is_regular_file(osrm_input_path)) { - SimpleLogger().Write(logWARNING) << "Input file " << input_path.string() << " not found!"; + SimpleLogger().Write(logWARNING) << "Input file " << osrm_input_path.string() << " not found!"; return 1; } @@ -91,7 +99,7 @@ int Prepare::Process(int argc, char *argv[]) const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); - SimpleLogger().Write() << "Input file: " << input_path.filename().string(); + SimpleLogger().Write() << "Input file: " << osrm_input_path.filename().string(); SimpleLogger().Write() << "Restrictions file: " << restrictions_path.filename().string(); SimpleLogger().Write() << "Profile: " << profile_path.filename().string(); SimpleLogger().Write() << "Threads: " << requested_num_threads; @@ -107,65 +115,40 @@ int Prepare::Process(int argc, char *argv[]) LogPolicy::GetInstance().Unmute(); FingerPrint fingerprint_orig; - CheckRestrictionsFile(fingerprint_orig); - boost::filesystem::ifstream input_stream(input_path, std::ios::in | std::ios::binary); - - node_filename = input_path.string() + ".nodes"; - edge_out = input_path.string() + ".edges"; - geometry_filename = input_path.string() + ".geometry"; - graph_out = input_path.string() + ".hsgr"; - rtree_nodes_path = input_path.string() + ".ramIndex"; - rtree_leafs_path = input_path.string() + ".fileIndex"; + node_output_path = osrm_input_path.string() + ".nodes"; + edge_output_path = osrm_input_path.string() + ".edges"; + geometry_output_path = osrm_input_path.string() + ".geometry"; + graph_output_path = osrm_input_path.string() + ".hsgr"; + rtree_nodes_output_path = osrm_input_path.string() + ".ramIndex"; + rtree_leafs_output_path = osrm_input_path.string() + ".fileIndex"; /*** Setup Scripting Environment ***/ // Create a new lua state - lua_State *lua_state = luaL_newstate(); - // Connect LuaBind to this lua state - luabind::open(lua_state); - EdgeBasedGraphFactory::SpeedProfileProperties speed_profile; + SimpleLogger().Write() << "Generating edge-expanded graph representation"; - if (!SetupScriptingEnvironment(lua_state, speed_profile)) - { - return 1; - } - -#ifdef WIN32 -#pragma message("Memory consumption on Windows can be higher due to different bit packing") -#else - static_assert(sizeof(ImportEdge) == 20, - "changing ImportEdge type has influence on memory consumption!"); -#endif - NodeID number_of_node_based_nodes = readBinaryOSRMGraphFromStream( - input_stream, edge_list, barrier_node_list, traffic_light_list, - &internal_to_external_node_map, restriction_list); - input_stream.close(); - - if (edge_list.empty()) - { - SimpleLogger().Write(logWARNING) << "The input data is empty, exiting."; - return 1; - } - - SimpleLogger().Write() << restriction_list.size() << " restrictions, " - << barrier_node_list.size() << " bollard nodes, " - << traffic_light_list.size() << " traffic lights"; + TIMER_START(expansion); std::vector node_based_edge_list; - unsigned number_of_edge_based_nodes = 0; DeallocatingVector edge_based_edge_list; + auto internal_to_external_node_map = osrm::make_unique>(); + auto graph_size = + BuildEdgeExpandedGraph(*internal_to_external_node_map, + node_based_edge_list, edge_based_edge_list); - // init node_based_edge_list, edge_based_edge_list by edgeList - number_of_edge_based_nodes = - BuildEdgeExpandedGraph(lua_state, number_of_node_based_nodes, node_based_edge_list, - edge_based_edge_list, speed_profile); - lua_close(lua_state); + auto number_of_node_based_nodes = graph_size.first; + auto number_of_edge_based_nodes = graph_size.second; TIMER_STOP(expansion); - BuildRTree(node_based_edge_list); + SimpleLogger().Write() << "building r-tree ..."; + TIMER_START(rtree); + + BuildRTree(node_based_edge_list, *internal_to_external_node_map); + + TIMER_STOP(rtree); RangebasedCRC32 crc32; if (crc32.using_hardware()) @@ -182,7 +165,8 @@ int Prepare::Process(int argc, char *argv[]) node_based_edge_list.shrink_to_fit(); SimpleLogger().Write() << "CRC32: " << crc32_value; - WriteNodeMapping(); + SimpleLogger().Write() << "writing node map ..."; + WriteNodeMapping(std::move(internal_to_external_node_map)); /*** * Contracting the edge-expanded graph @@ -211,7 +195,7 @@ int Prepare::Process(int argc, char *argv[]) SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count << " edges"; - boost::filesystem::ofstream hsgr_output_stream(graph_out, std::ios::binary); + boost::filesystem::ofstream hsgr_output_stream(graph_output_path, std::ios::binary); hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint)); const unsigned max_used_node_id = 1 + [&contracted_edge_list] { @@ -358,7 +342,7 @@ bool Prepare::ParseArguments(int argc, char *argv[]) // shown to the user boost::program_options::options_description hidden_options("Hidden options"); hidden_options.add_options()( - "input,i", boost::program_options::value(&input_path), + "input,i", boost::program_options::value(&osrm_input_path), "Input file in .osm, .osm.bz2 or .osm.pbf format"); // positional option @@ -408,7 +392,7 @@ bool Prepare::ParseArguments(int argc, char *argv[]) if (!option_variables.count("restrictions")) { - restrictions_path = std::string(input_path.string() + ".restrictions"); + restrictions_path = std::string(osrm_input_path.string() + ".restrictions"); } if (!option_variables.count("input")) @@ -420,36 +404,11 @@ bool Prepare::ParseArguments(int argc, char *argv[]) return true; } -/** - \brief Loads and checks file UUIDs -*/ -void Prepare::CheckRestrictionsFile(FingerPrint &fingerprint_orig) -{ - boost::filesystem::ifstream restriction_stream(restrictions_path, std::ios::binary); - FingerPrint fingerprint_loaded; - unsigned number_of_usable_restrictions = 0; - restriction_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint)); - if (!fingerprint_loaded.TestPrepare(fingerprint_orig)) - { - SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n" - "Reprocess to get rid of this warning."; - } - - restriction_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned)); - restriction_list.resize(number_of_usable_restrictions); - if (number_of_usable_restrictions > 0) - { - restriction_stream.read((char *)&(restriction_list[0]), - number_of_usable_restrictions * sizeof(TurnRestriction)); - } - restriction_stream.close(); -} - /** \brief Setups scripting environment (lua-scripting) Also initializes speed profile. */ -bool Prepare::SetupScriptingEnvironment( +void Prepare::SetupScriptingEnvironment( lua_State *lua_state, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile) { // open utility libraries string library; @@ -461,14 +420,16 @@ bool Prepare::SetupScriptingEnvironment( // Now call our function in a lua script if (0 != luaL_dofile(lua_state, profile_path.string().c_str())) { - std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; - return false; + std::stringstream msg; + msg << lua_tostring(lua_state, -1) << " occured in scripting block"; + throw osrm::exception(msg.str()); } if (0 != luaL_dostring(lua_state, "return traffic_signal_penalty\n")) { - std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; - return false; + std::stringstream msg; + msg << lua_tostring(lua_state, -1) << " occured in scripting block"; + throw osrm::exception(msg.str()); } speed_profile.traffic_signal_penalty = 10 * lua_tointeger(lua_state, -1); SimpleLogger().Write(logDEBUG) @@ -476,92 +437,134 @@ bool Prepare::SetupScriptingEnvironment( if (0 != luaL_dostring(lua_state, "return u_turn_penalty\n")) { - std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; - return false; + std::stringstream msg; + msg << lua_tostring(lua_state, -1) << " occured in scripting block"; + throw osrm::exception(msg.str()); } speed_profile.u_turn_penalty = 10 * lua_tointeger(lua_state, -1); speed_profile.has_turn_penalty_function = lua_function_exists(lua_state, "turn_function"); - - return true; } +/** + \brief Build load restrictions from .restriction file + */ +void Prepare::LoadRestrictionMap(const std::unordered_map &external_to_internal_node_map, + RestrictionMap &restriction_map) +{ + boost::filesystem::ifstream input_stream(restrictions_path, std::ios::in | std::ios::binary); + + std::vector restriction_list; + loadRestrictionsFromFile(input_stream, external_to_internal_node_map, restriction_list); + + SimpleLogger().Write() << " - " << restriction_list.size() << " restrictions."; + + restriction_map = RestrictionMap(restriction_list); +} + +/** + \brief Build load node based graph from .osrm file and restrictions from .restrictions file + */ +std::shared_ptr +Prepare::LoadNodeBasedGraph(std::vector &barrier_node_list, + std::vector &traffic_light_list, + RestrictionMap &restriction_map, + std::vector& internal_to_external_node_map) +{ + std::vector edge_list; + std::unordered_map external_to_internal_node_map; + + boost::filesystem::ifstream input_stream(osrm_input_path, std::ios::in | std::ios::binary); + + NodeID number_of_node_based_nodes = loadNodesFromFile(input_stream, + barrier_node_list, traffic_light_list, + internal_to_external_node_map, + external_to_internal_node_map); + + SimpleLogger().Write() << " - " << barrier_node_list.size() << " bollard nodes, " + << traffic_light_list.size() << " traffic lights"; + + loadEdgesFromFile(input_stream, external_to_internal_node_map, edge_list); + + LoadRestrictionMap(external_to_internal_node_map, restriction_map); + + if (edge_list.empty()) + { + SimpleLogger().Write(logWARNING) << "The input data is empty, exiting."; + return std::shared_ptr(); + } + + return NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list); +} + + /** \brief Building an edge-expanded graph from node-based input and turn restrictions */ -std::size_t -Prepare::BuildEdgeExpandedGraph(lua_State *lua_state, - NodeID number_of_node_based_nodes, +std::pair +Prepare::BuildEdgeExpandedGraph(std::vector &internal_to_external_node_map, std::vector &node_based_edge_list, - DeallocatingVector &edge_based_edge_list, - EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile) + DeallocatingVector &edge_based_edge_list) { - SimpleLogger().Write() << "Generating edge-expanded graph representation"; - std::shared_ptr node_based_graph = - NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list); - std::unique_ptr restriction_map = - osrm::make_unique(restriction_list); - std::shared_ptr edge_based_graph_factory = - std::make_shared(node_based_graph, std::move(restriction_map), - barrier_node_list, traffic_light_list, - internal_to_external_node_map, speed_profile); - edge_list.clear(); - edge_list.shrink_to_fit(); + lua_State *lua_state = luaL_newstate(); + luabind::open(lua_state); - edge_based_graph_factory->Run(edge_out, geometry_filename, lua_state); + EdgeBasedGraphFactory::SpeedProfileProperties speed_profile; - restriction_list.clear(); - restriction_list.shrink_to_fit(); - barrier_node_list.clear(); - barrier_node_list.shrink_to_fit(); - traffic_light_list.clear(); - traffic_light_list.shrink_to_fit(); + SetupScriptingEnvironment(lua_state, speed_profile); + + auto barrier_node_list = osrm::make_unique>(); + auto traffic_light_list = osrm::make_unique>(); + auto restriction_map = std::make_shared(); + + auto node_based_graph = LoadNodeBasedGraph(*barrier_node_list, *traffic_light_list, *restriction_map, internal_to_external_node_map); + + const std::size_t number_of_node_based_nodes = node_based_graph->GetNumberOfNodes(); + + EdgeBasedGraphFactory edge_based_graph_factory(node_based_graph, + restriction_map, + std::move(barrier_node_list), + std::move(traffic_light_list), + internal_to_external_node_map, + speed_profile); + + edge_based_graph_factory.Run(edge_output_path, geometry_output_path, lua_state); + lua_close(lua_state); const std::size_t number_of_edge_based_nodes = - edge_based_graph_factory->GetNumberOfEdgeBasedNodes(); + edge_based_graph_factory.GetNumberOfEdgeBasedNodes(); BOOST_ASSERT(number_of_edge_based_nodes != std::numeric_limits::max()); -#ifndef WIN32 - static_assert(sizeof(EdgeBasedEdge) == 16, - "changing ImportEdge type has influence on memory consumption!"); -#endif - edge_based_graph_factory->GetEdgeBasedEdges(edge_based_edge_list); - edge_based_graph_factory->GetEdgeBasedNodes(node_based_edge_list); + edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list); + edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list); - edge_based_graph_factory.reset(); - node_based_graph.reset(); - - return number_of_edge_based_nodes; + return std::make_pair(number_of_node_based_nodes, number_of_edge_based_nodes); } /** \brief Writing info on original (node-based) nodes */ -void Prepare::WriteNodeMapping() +void Prepare::WriteNodeMapping(std::unique_ptr> internal_to_external_node_map) { - SimpleLogger().Write() << "writing node map ..."; - boost::filesystem::ofstream node_stream(node_filename, std::ios::binary); - const unsigned size_of_mapping = internal_to_external_node_map.size(); + boost::filesystem::ofstream node_stream(node_output_path, std::ios::binary); + const unsigned size_of_mapping = internal_to_external_node_map->size(); node_stream.write((char *)&size_of_mapping, sizeof(unsigned)); if (size_of_mapping > 0) { - node_stream.write((char *)&(internal_to_external_node_map[0]), + node_stream.write((char *) internal_to_external_node_map->data(), size_of_mapping * sizeof(QueryNode)); } node_stream.close(); - internal_to_external_node_map.clear(); - internal_to_external_node_map.shrink_to_fit(); } /** \brief Building rtree-based nearest-neighbor data structure - Saves info to files: '.ramIndex' and '.fileIndex'. + Saves tree into '.ramIndex' and leaves into '.fileIndex'. */ -void Prepare::BuildRTree(std::vector &node_based_edge_list) +void Prepare::BuildRTree(const std::vector &node_based_edge_list, const std::vector& internal_to_external_node_map) { - SimpleLogger().Write() << "building r-tree ..."; - StaticRTree(node_based_edge_list, rtree_nodes_path.c_str(), - rtree_leafs_path.c_str(), internal_to_external_node_map); + StaticRTree(node_based_edge_list, rtree_nodes_output_path.c_str(), + rtree_leafs_output_path.c_str(), internal_to_external_node_map); } diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index 933213a4c..9d8f08f9a 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -58,38 +58,38 @@ class Prepare protected: bool ParseArguments(int argc, char *argv[]); - void CheckRestrictionsFile(FingerPrint &fingerprint_orig); - bool SetupScriptingEnvironment(lua_State *myLuaState, + void SetupScriptingEnvironment(lua_State *myLuaState, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile); - std::size_t BuildEdgeExpandedGraph(lua_State *myLuaState, - NodeID nodeBasedNodeNumber, - std::vector &nodeBasedEdgeList, - DeallocatingVector &edgeBasedEdgeList, - EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile); - void WriteNodeMapping(); - void BuildRTree(std::vector &node_based_edge_list); + void LoadRestrictionMap(const std::unordered_map &external_to_internal_node_map, + RestrictionMap &restriction_map); + std::shared_ptr + LoadNodeBasedGraph(std::vector &barrier_node_list, + std::vector &traffic_light_list, + RestrictionMap &restriction_map, + std::vector& internal_to_external_node_map); + std::pair + BuildEdgeExpandedGraph(std::vector &internal_to_external_node_map, + std::vector &node_based_edge_list, + DeallocatingVector &edge_based_edge_list); + void WriteNodeMapping(std::unique_ptr> internal_to_external_node_map); + void BuildRTree(const std::vector &node_based_edge_list, + const std::vector &internal_to_external_node_map); private: - std::vector internal_to_external_node_map; - std::vector restriction_list; - std::vector barrier_node_list; - std::vector traffic_light_list; - std::vector edge_list; unsigned requested_num_threads; boost::filesystem::path config_file_path; - boost::filesystem::path input_path; + boost::filesystem::path osrm_input_path; boost::filesystem::path restrictions_path; boost::filesystem::path preinfo_path; boost::filesystem::path profile_path; - std::string node_filename; - std::string edge_out; - std::string info_out; - std::string geometry_filename; - std::string graph_out; - std::string rtree_nodes_path; - std::string rtree_leafs_path; + std::string node_output_path; + std::string edge_output_path; + std::string geometry_output_path; + std::string graph_output_path; + std::string rtree_nodes_output_path; + std::string rtree_leafs_output_path; }; #endif // PROCESSING_CHAIN_HPP diff --git a/data_structures/node_based_graph.hpp b/data_structures/node_based_graph.hpp index bd71362c1..e3a0b594f 100644 --- a/data_structures/node_based_graph.hpp +++ b/data_structures/node_based_graph.hpp @@ -186,9 +186,8 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector( + return std::make_shared( static_cast(number_of_nodes), edges_list); - return graph; } #endif // NODE_BASED_GRAPH_HPP diff --git a/data_structures/restriction_map.hpp b/data_structures/restriction_map.hpp index 27a6698db..e2a89a008 100644 --- a/data_structures/restriction_map.hpp +++ b/data_structures/restriction_map.hpp @@ -91,6 +91,7 @@ template <> struct hash class RestrictionMap { public: + RestrictionMap() = default; RestrictionMap(const std::vector &restriction_list); // Replace end v with w in each turn restriction containing u as via node @@ -98,7 +99,7 @@ class RestrictionMap void FixupArrivingTurnRestriction(const NodeID node_u, const NodeID node_v, const NodeID node_w, - const std::shared_ptr &graph) + const GraphT &graph) { BOOST_ASSERT(node_u != SPECIAL_NODEID); BOOST_ASSERT(node_v != SPECIAL_NODEID); @@ -112,9 +113,9 @@ class RestrictionMap // find all potential start edges. It is more efficent to get a (small) list // of potential start edges than iterating over all buckets std::vector predecessors; - for (const EdgeID current_edge_id : graph->GetAdjacentEdgeRange(node_u)) + for (const EdgeID current_edge_id : graph.GetAdjacentEdgeRange(node_u)) { - const NodeID target = graph->GetTarget(current_edge_id); + const NodeID target = graph.GetTarget(current_edge_id); if (node_v != target) { predecessors.push_back(target); diff --git a/data_structures/static_rtree.hpp b/data_structures/static_rtree.hpp index 48ba20120..fa5607198 100644 --- a/data_structures/static_rtree.hpp +++ b/data_structures/static_rtree.hpp @@ -344,7 +344,7 @@ class StaticRTree StaticRTree(const StaticRTree &) = delete; // Construct a packed Hilbert-R-Tree with Kamel-Faloutsos algorithm [1] - explicit StaticRTree(std::vector &input_data_vector, + explicit StaticRTree(const std::vector &input_data_vector, const std::string tree_node_filename, const std::string leaf_node_filename, const std::vector &coordinate_list) diff --git a/util/graph_loader.hpp b/util/graph_loader.hpp index d741334ba..3779f0b9e 100644 --- a/util/graph_loader.hpp +++ b/util/graph_loader.hpp @@ -52,269 +52,34 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -template -NodeID read_undirected_osrm_stream(std::istream &input_stream, - std::vector &edge_list, - std::vector &coordinate_list) +/** + * Reads the .restrictions file and loads it to a vector. + * The since the restrictions reference nodes using their external node id, + * we need to renumber it to the new internal id. +*/ +unsigned loadRestrictionsFromFile(std::istream& input_stream, + const std::unordered_map& ext_to_int_id_map, + std::vector& restriction_list) { const FingerPrint fingerprint_orig; FingerPrint fingerprint_loaded; - input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(FingerPrint)); - - if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) + unsigned number_of_usable_restrictions = 0; + input_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint)); + if (!fingerprint_loaded.TestPrepare(fingerprint_orig)) { - SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n" + SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n" "Reprocess to get rid of this warning."; } - std::unordered_map ext_to_int_id_map; - - NodeID n; - input_stream.read(reinterpret_cast(&n), sizeof(NodeID)); - SimpleLogger().Write() << "Importing n = " << n << " nodes "; - - ExternalMemoryNode current_node; - for (NodeID i = 0; i < n; ++i) + input_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned)); + restriction_list.resize(number_of_usable_restrictions); + if (number_of_usable_restrictions > 0) { - input_stream.read(reinterpret_cast(¤t_node), sizeof(ExternalMemoryNode)); - coordinate_list.emplace_back(current_node.lat, current_node.lon); - ext_to_int_id_map.emplace(current_node.node_id, i); - // if (current_node.barrier) - // { - // barrier_node_list.emplace_back(i); - // } - // if (current_node.traffic_lights) - // { - // traffic_light_node_list.emplace_back(i); - // } + input_stream.read((char *) restriction_list.data(), + number_of_usable_restrictions * sizeof(TurnRestriction)); } - // tighten vector sizes - // barrier_node_list.shrink_to_fit(); - // traffic_light_node_list.shrink_to_fit(); - - // renumber nodes in turn restrictions - // for (TurnRestriction ¤t_restriction : restriction_list) - // { - // auto internal_id_iter = ext_to_int_id_map.find(current_restriction.from.node); - // if (internal_id_iter == ext_to_int_id_map.end()) - // { - // SimpleLogger().Write(logDEBUG) << "Unmapped from node " << - // current_restriction.from.node - // << " of restriction"; - // continue; - // } - // current_restriction.from.node = internal_id_iter->second; - - // internal_id_iter = ext_to_int_id_map.find(current_restriction.via.node); - // if (internal_id_iter == ext_to_int_id_map.end()) - // { - // SimpleLogger().Write(logDEBUG) << "Unmapped via node " << - // current_restriction.via.node - // << " of restriction"; - // continue; - // } - - // current_restriction.via.node = internal_id_iter->second; - - // internal_id_iter = ext_to_int_id_map.find(current_restriction.to.node); - // if (internal_id_iter == ext_to_int_id_map.end()) - // { - // SimpleLogger().Write(logDEBUG) << "Unmapped to node " << current_restriction.to.node - // << " of restriction"; - // continue; - // } - // current_restriction.to.node = internal_id_iter->second; - // } - - EdgeWeight weight; - NodeID source, target; - unsigned nameID; - int length; - short dir; // direction (0 = open, 1 = forward, 2+ = open) - bool is_roundabout, ignore_in_grid, is_access_restricted, is_split; - TravelMode travel_mode; - - EdgeID m; - input_stream.read(reinterpret_cast(&m), sizeof(unsigned)); - edge_list.reserve(m); - SimpleLogger().Write() << " and " << m << " edges "; - - for (EdgeID i = 0; i < m; ++i) - { - input_stream.read(reinterpret_cast(&source), sizeof(unsigned)); - input_stream.read(reinterpret_cast(&target), sizeof(unsigned)); - input_stream.read(reinterpret_cast(&length), sizeof(int)); - input_stream.read(reinterpret_cast(&dir), sizeof(short)); - input_stream.read(reinterpret_cast(&weight), sizeof(int)); - input_stream.read(reinterpret_cast(&nameID), sizeof(unsigned)); - input_stream.read(reinterpret_cast(&is_roundabout), sizeof(bool)); - input_stream.read(reinterpret_cast(&ignore_in_grid), sizeof(bool)); - input_stream.read(reinterpret_cast(&is_access_restricted), sizeof(bool)); - input_stream.read(reinterpret_cast(&travel_mode), sizeof(TravelMode)); - input_stream.read(reinterpret_cast(&is_split), sizeof(bool)); - - BOOST_ASSERT_MSG(length > 0, "loaded null length edge"); - BOOST_ASSERT_MSG(weight > 0, "loaded null weight"); - BOOST_ASSERT_MSG(0 <= dir && dir <= 2, "loaded bogus direction"); - - // bool forward = true; - // bool backward = true; - // if (1 == dir) - // { - // backward = false; - // } - // if (2 == dir) - // { - // forward = false; - // } - - // translate the external NodeIDs to internal IDs - auto internal_id_iter = ext_to_int_id_map.find(source); - if (ext_to_int_id_map.find(source) == ext_to_int_id_map.end()) - { -#ifndef NDEBUG - SimpleLogger().Write(logWARNING) << " unresolved source NodeID: " << source; -#endif - continue; - } - source = internal_id_iter->second; - internal_id_iter = ext_to_int_id_map.find(target); - if (ext_to_int_id_map.find(target) == ext_to_int_id_map.end()) - { -#ifndef NDEBUG - SimpleLogger().Write(logWARNING) << "unresolved target NodeID : " << target; -#endif - continue; - } - target = internal_id_iter->second; - BOOST_ASSERT_MSG(source != SPECIAL_NODEID && target != SPECIAL_NODEID, - "nonexisting source or target"); - - if (source > target) - { - std::swap(source, target); - // std::swap(forward, backward); - } - - edge_list.emplace_back(source, target); - } - ext_to_int_id_map.clear(); - - tbb::parallel_sort(edge_list.begin(), edge_list.end()); - - // for (unsigned i = 1; i < edge_list.size(); ++i) - // { - // if ((edge_list[i - 1].target == edge_list[i].target) && - // (edge_list[i - 1].source == edge_list[i].source)) - // { - // const bool edge_flags_equivalent = (edge_list[i - 1].forward == edge_list[i].forward) - // && - // (edge_list[i - 1].backward == - // edge_list[i].backward); - // const bool edge_flags_are_superset1 = - // (edge_list[i - 1].forward && edge_list[i - 1].backward) && - // (edge_list[i].forward != edge_list[i].backward); - // const bool edge_flags_are_superset_2 = - // (edge_list[i].forward && edge_list[i].backward) && - // (edge_list[i - 1].forward != edge_list[i - 1].backward); - - // if (edge_flags_equivalent) - // { - // edge_list[i].weight = std::min(edge_list[i - 1].weight, edge_list[i].weight); - // edge_list[i - 1].source = SPECIAL_NODEID; - // } - // else if (edge_flags_are_superset1) - // { - // if (edge_list[i - 1].weight <= edge_list[i].weight) - // { - // // edge i-1 is smaller and goes in both directions. Throw away the other edge - // edge_list[i].source = SPECIAL_NODEID; - // } - // else - // { - // // edge i-1 is open in both directions, but edge i is smaller in one - // direction. - // // Close edge i-1 in this direction - // edge_list[i - 1].forward = !edge_list[i].forward; - // edge_list[i - 1].backward = !edge_list[i].backward; - // } - // } - // else if (edge_flags_are_superset_2) - // { - // if (edge_list[i - 1].weight <= edge_list[i].weight) - // { - // // edge i-1 is smaller for one direction. edge i is open in both. close edge - // i - // // in the other direction - // edge_list[i].forward = !edge_list[i - 1].forward; - // edge_list[i].backward = !edge_list[i - 1].backward; - // } - // else - // { - // // edge i is smaller and goes in both direction. Throw away edge i-1 - // edge_list[i - 1].source = SPECIAL_NODEID; - // } - // } - // } - // } - // const auto new_end_iter = - // std::remove_if(edge_list.begin(), edge_list.end(), [](const EdgeT &edge) - // { - // return edge.source == SPECIAL_NODEID || edge.target == SPECIAL_NODEID; - // }); - // edge_list.erase(new_end_iter, edge_list.end()); // remove excess candidates. - // edge_list.shrink_to_fit(); - SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges"; - return n; -} - -template -NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream, - std::vector &edge_list, - std::vector &barrier_node_list, - std::vector &traffic_light_node_list, - std::vector *int_to_ext_node_id_map, - std::vector &restriction_list) -{ - const FingerPrint fingerprint_orig; - FingerPrint fingerprint_loaded; - input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(FingerPrint)); - - if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) - { - SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n" - "Reprocess to get rid of this warning."; - } - - std::unordered_map ext_to_int_id_map; - - NodeID n; - input_stream.read(reinterpret_cast(&n), sizeof(NodeID)); - SimpleLogger().Write() << "Importing n = " << n << " nodes "; - - ExternalMemoryNode current_node; - for (NodeID i = 0; i < n; ++i) - { - input_stream.read(reinterpret_cast(¤t_node), sizeof(ExternalMemoryNode)); - int_to_ext_node_id_map->emplace_back(current_node.lat, current_node.lon, - current_node.node_id); - ext_to_int_id_map.emplace(current_node.node_id, i); - if (current_node.barrier) - { - barrier_node_list.emplace_back(i); - } - if (current_node.traffic_lights) - { - traffic_light_node_list.emplace_back(i); - } - } - - // tighten vector sizes - barrier_node_list.shrink_to_fit(); - traffic_light_node_list.shrink_to_fit(); - - // renumber nodes in turn restrictions + // renumber ids referenced in restrictions for (TurnRestriction ¤t_restriction : restriction_list) { auto internal_id_iter = ext_to_int_id_map.find(current_restriction.from.node); @@ -346,6 +111,70 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream, current_restriction.to.node = internal_id_iter->second; } + return number_of_usable_restrictions; +} + + +/** + * Reads the beginning of an .osrm file and produces: + * - list of barrier nodes + * - list of traffic lights + * - index to original node id map + * - original node id to index map + */ +NodeID loadNodesFromFile(std::istream &input_stream, + std::vector &barrier_node_list, + std::vector &traffic_light_node_list, + std::vector &int_to_ext_node_id_map, + std::unordered_map &ext_to_int_id_map) +{ + const FingerPrint fingerprint_orig; + FingerPrint fingerprint_loaded; + input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(FingerPrint)); + + if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) + { + SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n" + "Reprocess to get rid of this warning."; + } + + NodeID n; + input_stream.read(reinterpret_cast(&n), sizeof(NodeID)); + SimpleLogger().Write() << "Importing n = " << n << " nodes "; + + ExternalMemoryNode current_node; + for (NodeID i = 0; i < n; ++i) + { + input_stream.read(reinterpret_cast(¤t_node), sizeof(ExternalMemoryNode)); + int_to_ext_node_id_map.emplace_back(current_node.lat, current_node.lon, + current_node.node_id); + ext_to_int_id_map.emplace(current_node.node_id, i); + if (current_node.barrier) + { + barrier_node_list.emplace_back(i); + } + if (current_node.traffic_lights) + { + traffic_light_node_list.emplace_back(i); + } + } + + // tighten vector sizes + barrier_node_list.shrink_to_fit(); + traffic_light_node_list.shrink_to_fit(); + + return n; +} + +/** + * Reads a .osrm file and produces the edges. Edges reference nodes in the old + * OSM based format, we need to renumber it here. + */ +template +NodeID loadEdgesFromFile(std::istream &input_stream, + const std::unordered_map &ext_to_int_id_map, + std::vector &edge_list) +{ EdgeWeight weight; NodeID source, target; unsigned nameID; @@ -419,10 +248,10 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream, edge_list.emplace_back(source, target, nameID, weight, forward, backward, is_roundabout, ignore_in_grid, is_access_restricted, travel_mode, is_split); } - ext_to_int_id_map.clear(); tbb::parallel_sort(edge_list.begin(), edge_list.end()); + // Removes multi-edges between nodes for (unsigned i = 1; i < edge_list.size(); ++i) { if ((edge_list[i - 1].target == edge_list[i].target) && @@ -482,7 +311,8 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream, edge_list.erase(new_end_iter, edge_list.end()); // remove excess candidates. edge_list.shrink_to_fit(); SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges"; - return n; + + return m; } template From d64e6e6c1f8b1d3de30fe96bbc6db1aac786b4bf Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 23 Apr 2015 17:48:41 +0200 Subject: [PATCH 34/87] Move more function from Run in subfunctions --- contractor/contractor.hpp | 2 +- contractor/processing_chain.cpp | 73 ++++++++++++++++++--------------- contractor/processing_chain.hpp | 13 +++--- 3 files changed, 48 insertions(+), 40 deletions(-) diff --git a/contractor/contractor.hpp b/contractor/contractor.hpp index ccfabbc8f..e4a278c71 100644 --- a/contractor/contractor.hpp +++ b/contractor/contractor.hpp @@ -186,6 +186,7 @@ class Contractor } // clear input vector input_edge_list.clear(); + // FIXME not sure if we need this edges.shrink_to_fit(); tbb::parallel_sort(edges.begin(), edges.end()); @@ -953,7 +954,6 @@ class Contractor } std::shared_ptr contractor_graph; - std::vector contracted_edge_list; stxxl::vector external_edge_list; std::vector orig_node_id_to_new_id_map; XORFastHash fast_hash; diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 7ad682944..1d20ee6bd 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -123,20 +123,18 @@ int Prepare::Process(int argc, char *argv[]) rtree_nodes_output_path = osrm_input_path.string() + ".ramIndex"; rtree_leafs_output_path = osrm_input_path.string() + ".fileIndex"; - /*** Setup Scripting Environment ***/ // Create a new lua state - SimpleLogger().Write() << "Generating edge-expanded graph representation"; TIMER_START(expansion); - std::vector node_based_edge_list; + auto node_based_edge_list = osrm::make_unique>();; DeallocatingVector edge_based_edge_list; auto internal_to_external_node_map = osrm::make_unique>(); auto graph_size = BuildEdgeExpandedGraph(*internal_to_external_node_map, - node_based_edge_list, edge_based_edge_list); + *node_based_edge_list, edge_based_edge_list); auto number_of_node_based_nodes = graph_size.first; auto number_of_edge_based_nodes = graph_size.second; @@ -146,49 +144,26 @@ int Prepare::Process(int argc, char *argv[]) SimpleLogger().Write() << "building r-tree ..."; TIMER_START(rtree); - BuildRTree(node_based_edge_list, *internal_to_external_node_map); + BuildRTree(*node_based_edge_list, *internal_to_external_node_map); TIMER_STOP(rtree); - RangebasedCRC32 crc32; - if (crc32.using_hardware()) - { - SimpleLogger().Write() << "using hardware based CRC32 computation"; - } - else - { - SimpleLogger().Write() << "using software based CRC32 computation"; - } - const unsigned crc32_value = crc32(node_based_edge_list); - node_based_edge_list.clear(); - node_based_edge_list.shrink_to_fit(); - SimpleLogger().Write() << "CRC32: " << crc32_value; + const unsigned crc32_value = CalculateEdgeChecksum(std::move(node_based_edge_list)); SimpleLogger().Write() << "writing node map ..."; WriteNodeMapping(std::move(internal_to_external_node_map)); - /*** - * Contracting the edge-expanded graph - */ - - SimpleLogger().Write() << "initializing contractor"; - auto contractor = - osrm::make_unique(number_of_edge_based_nodes, edge_based_edge_list); + // Contracting the edge-expanded graph TIMER_START(contraction); - contractor->Run(); + DeallocatingVector contracted_edge_list; + ContractGraph(number_of_edge_based_nodes, edge_based_edge_list, contracted_edge_list); TIMER_STOP(contraction); SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec"; - DeallocatingVector contracted_edge_list; - contractor->GetEdges(contracted_edge_list); - contractor.reset(); - - /*** - * Sorting contracted edges in a way that the static query graph can read some in in-place. - */ + // Sorting contracted edges in a way that the static query graph can read some in in-place. tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end()); const unsigned contracted_edge_count = contracted_edge_list.size(); @@ -254,8 +229,8 @@ int Prepare::Process(int argc, char *argv[]) hsgr_output_stream.write((char *)&node_array[0], sizeof(StaticGraph::NodeArrayEntry) * node_array_size); } - // serialize all edges + // serialize all edges SimpleLogger().Write() << "Building edge array"; edge = 0; int number_of_used_edges = 0; @@ -309,6 +284,24 @@ int Prepare::Process(int argc, char *argv[]) return 0; } +unsigned Prepare::CalculateEdgeChecksum(std::unique_ptr> node_based_edge_list) +{ + RangebasedCRC32 crc32; + if (crc32.using_hardware()) + { + SimpleLogger().Write() << "using hardware based CRC32 computation"; + } + else + { + SimpleLogger().Write() << "using software based CRC32 computation"; + } + + const unsigned crc32_value = crc32(*node_based_edge_list); + SimpleLogger().Write() << "CRC32: " << crc32_value; + + return crc32_value; +} + /** \brief Parses command line arguments \param argc count of arguments @@ -542,6 +535,18 @@ Prepare::BuildEdgeExpandedGraph(std::vector &internal_to_external_nod return std::make_pair(number_of_node_based_nodes, number_of_edge_based_nodes); } +/** + \brief Build contracted graph. + */ +void Prepare::ContractGraph(const std::size_t number_of_edge_based_nodes, + DeallocatingVector& edge_based_edge_list, + DeallocatingVector& contracted_edge_list) +{ + Contractor contractor(number_of_edge_based_nodes, edge_based_edge_list); + contractor.Run(); + contractor.GetEdges(contracted_edge_list); +} + /** \brief Writing info on original (node-based) nodes */ diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index 9d8f08f9a..fa35b34a8 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -62,11 +62,14 @@ class Prepare EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile); void LoadRestrictionMap(const std::unordered_map &external_to_internal_node_map, RestrictionMap &restriction_map); - std::shared_ptr - LoadNodeBasedGraph(std::vector &barrier_node_list, - std::vector &traffic_light_list, - RestrictionMap &restriction_map, - std::vector& internal_to_external_node_map); + unsigned CalculateEdgeChecksum(std::unique_ptr> node_based_edge_list); + void ContractGraph(const std::size_t number_of_edge_based_nodes, + DeallocatingVector& edge_based_edge_list, + DeallocatingVector& contracted_edge_list); + std::shared_ptr LoadNodeBasedGraph(std::vector &barrier_node_list, + std::vector &traffic_light_list, + RestrictionMap &restriction_map, + std::vector& internal_to_external_node_map); std::pair BuildEdgeExpandedGraph(std::vector &internal_to_external_node_map, std::vector &node_based_edge_list, From 1f985d04a2226dc37b474dd2f9f1bd0dd24aaacd Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 23 Apr 2015 18:53:36 +0200 Subject: [PATCH 35/87] Move writing graph to an own function --- contractor/processing_chain.cpp | 78 ++++++++++++++++++--------------- contractor/processing_chain.hpp | 3 ++ 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 1d20ee6bd..2b202f0c8 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -114,8 +114,6 @@ int Prepare::Process(int argc, char *argv[]) LogPolicy::GetInstance().Unmute(); - FingerPrint fingerprint_orig; - node_output_path = osrm_input_path.string() + ".nodes"; edge_output_path = osrm_input_path.string() + ".edges"; geometry_output_path = osrm_input_path.string() + ".geometry"; @@ -148,34 +146,58 @@ int Prepare::Process(int argc, char *argv[]) TIMER_STOP(rtree); - - const unsigned crc32_value = CalculateEdgeChecksum(std::move(node_based_edge_list)); - SimpleLogger().Write() << "writing node map ..."; WriteNodeMapping(std::move(internal_to_external_node_map)); // Contracting the edge-expanded graph TIMER_START(contraction); - DeallocatingVector contracted_edge_list; - ContractGraph(number_of_edge_based_nodes, edge_based_edge_list, contracted_edge_list); + auto contracted_edge_list = osrm::make_unique>(); + ContractGraph(number_of_edge_based_nodes, edge_based_edge_list, *contracted_edge_list); TIMER_STOP(contraction); SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec"; - // Sorting contracted edges in a way that the static query graph can read some in in-place. + std::size_t number_of_used_edges = WriteContractedGraph(number_of_edge_based_nodes, + std::move(node_based_edge_list), + std::move(contracted_edge_list)); - tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end()); - const unsigned contracted_edge_count = contracted_edge_list.size(); + TIMER_STOP(preparing); + + SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds"; + SimpleLogger().Write() << "Expansion : " << (number_of_node_based_nodes / TIMER_SEC(expansion)) + << " nodes/sec and " + << (number_of_edge_based_nodes / TIMER_SEC(expansion)) << " edges/sec"; + + SimpleLogger().Write() << "Contraction: " + << (number_of_edge_based_nodes / TIMER_SEC(contraction)) + << " nodes/sec and " << number_of_used_edges / TIMER_SEC(contraction) + << " edges/sec"; + + SimpleLogger().Write() << "finished preprocessing"; + + return 0; +} + +std::size_t Prepare::WriteContractedGraph(unsigned number_of_edge_based_nodes, + std::unique_ptr> node_based_edge_list, + std::unique_ptr> contracted_edge_list) +{ + const unsigned crc32_value = CalculateEdgeChecksum(std::move(node_based_edge_list)); + + // Sorting contracted edges in a way that the static query graph can read some in in-place. + tbb::parallel_sort(contracted_edge_list->begin(), contracted_edge_list->end()); + const unsigned contracted_edge_count = contracted_edge_list->size(); SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count << " edges"; + FingerPrint fingerprint_orig; boost::filesystem::ofstream hsgr_output_stream(graph_output_path, std::ios::binary); hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint)); const unsigned max_used_node_id = 1 + [&contracted_edge_list] { unsigned tmp_max = 0; - for (const QueryEdge &edge : contracted_edge_list) + for (const QueryEdge &edge : *contracted_edge_list) { BOOST_ASSERT(SPECIAL_NODEID != edge.source); BOOST_ASSERT(SPECIAL_NODEID != edge.target); @@ -200,7 +222,7 @@ int Prepare::Process(int argc, char *argv[]) for (const auto node : osrm::irange(0u, max_used_node_id)) { last_edge = edge; - while ((edge < contracted_edge_count) && (contracted_edge_list[edge].source == node)) + while ((edge < contracted_edge_count) && ((*contracted_edge_list)[edge].source == node)) { ++edge; } @@ -236,12 +258,12 @@ int Prepare::Process(int argc, char *argv[]) int number_of_used_edges = 0; StaticGraph::EdgeArrayEntry current_edge; - for (const auto edge : osrm::irange(0, contracted_edge_list.size())) + for (const auto edge : osrm::irange(0, contracted_edge_list->size())) { // no eigen loops - BOOST_ASSERT(contracted_edge_list[edge].source != contracted_edge_list[edge].target); - current_edge.target = contracted_edge_list[edge].target; - current_edge.data = contracted_edge_list[edge].data; + BOOST_ASSERT(contracted_edge_list->(edge).source != contracted_edge_list->(edge).target); + current_edge.target = (*contracted_edge_list)[edge].target; + current_edge.data = (*contracted_edge_list)[edge].data; // every target needs to be valid BOOST_ASSERT(current_edge.target < max_used_node_id); @@ -249,12 +271,12 @@ int Prepare::Process(int argc, char *argv[]) if (current_edge.data.distance <= 0) { SimpleLogger().Write(logWARNING) << "Edge: " << edge - << ",source: " << contracted_edge_list[edge].source - << ", target: " << contracted_edge_list[edge].target + << ",source: " << contracted_edge_list->at(edge).source + << ", target: " << contracted_edge_list->at(edge).target << ", dist: " << current_edge.data.distance; SimpleLogger().Write(logWARNING) << "Failed at adjacency list of node " - << contracted_edge_list[edge].source << "/" + << contracted_edge_list->at(edge).source << "/" << node_array.size() - 1; return 1; } @@ -264,24 +286,8 @@ int Prepare::Process(int argc, char *argv[]) ++number_of_used_edges; } - hsgr_output_stream.close(); - TIMER_STOP(preparing); - - SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds"; - SimpleLogger().Write() << "Expansion : " << (number_of_node_based_nodes / TIMER_SEC(expansion)) - << " nodes/sec and " - << (number_of_edge_based_nodes / TIMER_SEC(expansion)) << " edges/sec"; - - SimpleLogger().Write() << "Contraction: " - << (number_of_edge_based_nodes / TIMER_SEC(contraction)) - << " nodes/sec and " << number_of_used_edges / TIMER_SEC(contraction) - << " edges/sec"; - - node_array.clear(); - SimpleLogger().Write() << "finished preprocessing"; - - return 0; + return number_of_used_edges; } unsigned Prepare::CalculateEdgeChecksum(std::unique_ptr> node_based_edge_list) diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index fa35b34a8..a07623a59 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -66,6 +66,9 @@ class Prepare void ContractGraph(const std::size_t number_of_edge_based_nodes, DeallocatingVector& edge_based_edge_list, DeallocatingVector& contracted_edge_list); + std::size_t WriteContractedGraph(unsigned number_of_edge_based_nodes, + std::unique_ptr> node_based_edge_list, + std::unique_ptr> contracted_edge_list); std::shared_ptr LoadNodeBasedGraph(std::vector &barrier_node_list, std::vector &traffic_light_list, RestrictionMap &restriction_map, From a46bcf45d5ec2234fea912b2a8b6515892fe51f6 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 24 Apr 2015 14:51:25 +0200 Subject: [PATCH 36/87] Move option parsing to own class --- contractor/contractor_options.cpp | 136 ++++++++++++++++++++++++ contractor/contractor_options.hpp | 68 ++++++++++++ contractor/processing_chain.cpp | 165 ++---------------------------- contractor/processing_chain.hpp | 23 +---- extract.cpp | 2 +- extractor/extractor.cpp | 20 ++-- extractor/extractor.hpp | 9 +- prepare.cpp | 70 ++++++++++--- 8 files changed, 295 insertions(+), 198 deletions(-) create mode 100644 contractor/contractor_options.cpp create mode 100644 contractor/contractor_options.hpp diff --git a/contractor/contractor_options.cpp b/contractor/contractor_options.cpp new file mode 100644 index 000000000..b6a1e2caa --- /dev/null +++ b/contractor/contractor_options.cpp @@ -0,0 +1,136 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "contractor_options.hpp" + +#include "../util/git_sha.hpp" +#include "../util/ini_file.hpp" +#include "../util/simple_logger.hpp" + +#include +#include + +#include + +return_code +ContractorOptions::ParseArguments(int argc, char *argv[], ContractorConfig &contractor_config) +{ + // declare a group of options that will be allowed only on command line + boost::program_options::options_description generic_options("Options"); + generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message")( + "config,c", boost::program_options::value(&contractor_config.config_file_path) + ->default_value("contractor.ini"), + "Path to a configuration file."); + + // declare a group of options that will be allowed both on command line and in config file + boost::program_options::options_description config_options("Configuration"); + config_options.add_options()( + "restrictions,r", + boost::program_options::value(&contractor_config.restrictions_path), + "Restrictions file in .osrm.restrictions format")( + "profile,p", boost::program_options::value(&contractor_config.profile_path) + ->default_value("profile.lua"), + "Path to LUA routing profile")( + "threads,t", boost::program_options::value(&contractor_config.requested_num_threads) + ->default_value(tbb::task_scheduler_init::default_num_threads()), + "Number of threads to use"); + + // hidden options, will be allowed both on command line and in config file, but will not be + // shown to the user + boost::program_options::options_description hidden_options("Hidden options"); + hidden_options.add_options()( + "input,i", boost::program_options::value(&contractor_config.osrm_input_path), + "Input file in .osm, .osm.bz2 or .osm.pbf format"); + + // positional option + boost::program_options::positional_options_description positional_options; + positional_options.add("input", 1); + + // combine above options for parsing + boost::program_options::options_description cmdline_options; + cmdline_options.add(generic_options).add(config_options).add(hidden_options); + + boost::program_options::options_description config_file_options; + config_file_options.add(config_options).add(hidden_options); + + boost::program_options::options_description visible_options( + "Usage: " + boost::filesystem::basename(argv[0]) + " [options]"); + visible_options.add(generic_options).add(config_options); + + // parse command line options + boost::program_options::variables_map option_variables; + boost::program_options::store(boost::program_options::command_line_parser(argc, argv) + .options(cmdline_options) + .positional(positional_options) + .run(), + option_variables); + + const auto &temp_config_path = option_variables["config"].as(); + if (boost::filesystem::is_regular_file(temp_config_path)) + { + boost::program_options::store(boost::program_options::parse_config_file( + temp_config_path.string().c_str(), cmdline_options, true), + option_variables); + } + + if (option_variables.count("version")) + { + SimpleLogger().Write() << g_GIT_DESCRIPTION; + return return_code::fail; + } + + if (option_variables.count("help")) + { + SimpleLogger().Write() << "\n" << visible_options; + return return_code::fail; + } + + boost::program_options::notify(option_variables); + + if (!option_variables.count("restrictions")) + { + contractor_config.restrictions_path = contractor_config.osrm_input_path.string() + ".restrictions"; + } + + if (!option_variables.count("input")) + { + SimpleLogger().Write() << "\n" << visible_options; + return return_code::fail; + } + + return return_code::ok; +} + +void ContractorOptions::GenerateOutputFilesNames(ContractorConfig &contractor_config) +{ + contractor_config.node_output_path = contractor_config.osrm_input_path.string() + ".nodes"; + contractor_config.edge_output_path = contractor_config.osrm_input_path.string() + ".edges"; + contractor_config.geometry_output_path = contractor_config.osrm_input_path.string() + ".geometry"; + contractor_config.graph_output_path = contractor_config.osrm_input_path.string() + ".hsgr"; + contractor_config.rtree_nodes_output_path = contractor_config.osrm_input_path.string() + ".ramIndex"; + contractor_config.rtree_leafs_output_path = contractor_config.osrm_input_path.string() + ".fileIndex"; +} diff --git a/contractor/contractor_options.hpp b/contractor/contractor_options.hpp new file mode 100644 index 000000000..87541af9d --- /dev/null +++ b/contractor/contractor_options.hpp @@ -0,0 +1,68 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef CONTRACTOR_OPTIONS_HPP +#define CONTRACTOR_OPTIONS_HPP + +#include + +#include + +enum class return_code : unsigned +{ + ok, + fail, + exit +}; + +struct ContractorConfig +{ + ContractorConfig() noexcept : requested_num_threads(0) {} + + boost::filesystem::path config_file_path; + boost::filesystem::path osrm_input_path; + boost::filesystem::path restrictions_path; + boost::filesystem::path profile_path; + + std::string node_output_path; + std::string edge_output_path; + std::string geometry_output_path; + std::string graph_output_path; + std::string rtree_nodes_output_path; + std::string rtree_leafs_output_path; + + unsigned requested_num_threads; +}; + +struct ContractorOptions +{ + static return_code ParseArguments(int argc, char *argv[], ContractorConfig &extractor_config); + + static void GenerateOutputFilesNames(ContractorConfig &extractor_config); +}; + +#endif // EXTRACTOR_OPTIONS_HPP diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 2b202f0c8..ffbad0b1f 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -48,7 +48,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#include #include #include @@ -57,11 +56,9 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -Prepare::Prepare() : requested_num_threads(1) {} - Prepare::~Prepare() {} -int Prepare::Process(int argc, char *argv[]) +int Prepare::Run() { #ifdef WIN32 #pragma message("Memory consumption on Windows can be higher due to different bit packing") @@ -72,55 +69,8 @@ int Prepare::Process(int argc, char *argv[]) "changing EdgeBasedEdge type has influence on memory consumption!"); #endif - LogPolicy::GetInstance().Unmute(); TIMER_START(preparing); - if (!ParseArguments(argc, argv)) - { - return 0; - } - if (!boost::filesystem::is_regular_file(osrm_input_path)) - { - SimpleLogger().Write(logWARNING) << "Input file " << osrm_input_path.string() << " not found!"; - return 1; - } - - if (!boost::filesystem::is_regular_file(profile_path)) - { - SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string() << " not found!"; - return 1; - } - - if (1 > requested_num_threads) - { - SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger"; - return 1; - } - - const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); - - SimpleLogger().Write() << "Input file: " << osrm_input_path.filename().string(); - SimpleLogger().Write() << "Restrictions file: " << restrictions_path.filename().string(); - SimpleLogger().Write() << "Profile: " << profile_path.filename().string(); - SimpleLogger().Write() << "Threads: " << requested_num_threads; - if (recommended_num_threads != requested_num_threads) - { - SimpleLogger().Write(logWARNING) << "The recommended number of threads is " - << recommended_num_threads - << "! This setting may have performance side-effects."; - } - - tbb::task_scheduler_init init(requested_num_threads); - - LogPolicy::GetInstance().Unmute(); - - node_output_path = osrm_input_path.string() + ".nodes"; - edge_output_path = osrm_input_path.string() + ".edges"; - geometry_output_path = osrm_input_path.string() + ".geometry"; - graph_output_path = osrm_input_path.string() + ".hsgr"; - rtree_nodes_output_path = osrm_input_path.string() + ".ramIndex"; - rtree_leafs_output_path = osrm_input_path.string() + ".fileIndex"; - // Create a new lua state SimpleLogger().Write() << "Generating edge-expanded graph representation"; @@ -192,7 +142,7 @@ std::size_t Prepare::WriteContractedGraph(unsigned number_of_edge_based_nodes, << " edges"; FingerPrint fingerprint_orig; - boost::filesystem::ofstream hsgr_output_stream(graph_output_path, std::ios::binary); + boost::filesystem::ofstream hsgr_output_stream(config.graph_output_path, std::ios::binary); hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint)); const unsigned max_used_node_id = 1 + [&contracted_edge_list] { @@ -308,101 +258,6 @@ unsigned Prepare::CalculateEdgeChecksum(std::unique_ptr(&config_file_path) - ->default_value("contractor.ini"), - "Path to a configuration file."); - - // declare a group of options that will be allowed both on command line and in config file - boost::program_options::options_description config_options("Configuration"); - config_options.add_options()( - "restrictions,r", - boost::program_options::value(&restrictions_path), - "Restrictions file in .osrm.restrictions format")( - "profile,p", boost::program_options::value(&profile_path) - ->default_value("profile.lua"), - "Path to LUA routing profile")( - "threads,t", boost::program_options::value(&requested_num_threads) - ->default_value(tbb::task_scheduler_init::default_num_threads()), - "Number of threads to use"); - - // hidden options, will be allowed both on command line and in config file, but will not be - // shown to the user - boost::program_options::options_description hidden_options("Hidden options"); - hidden_options.add_options()( - "input,i", boost::program_options::value(&osrm_input_path), - "Input file in .osm, .osm.bz2 or .osm.pbf format"); - - // positional option - boost::program_options::positional_options_description positional_options; - positional_options.add("input", 1); - - // combine above options for parsing - boost::program_options::options_description cmdline_options; - cmdline_options.add(generic_options).add(config_options).add(hidden_options); - - boost::program_options::options_description config_file_options; - config_file_options.add(config_options).add(hidden_options); - - boost::program_options::options_description visible_options( - "Usage: " + boost::filesystem::basename(argv[0]) + " [options]"); - visible_options.add(generic_options).add(config_options); - - // parse command line options - boost::program_options::variables_map option_variables; - boost::program_options::store(boost::program_options::command_line_parser(argc, argv) - .options(cmdline_options) - .positional(positional_options) - .run(), - option_variables); - - const auto &temp_config_path = option_variables["config"].as(); - if (boost::filesystem::is_regular_file(temp_config_path)) - { - boost::program_options::store(boost::program_options::parse_config_file( - temp_config_path.string().c_str(), cmdline_options, true), - option_variables); - } - - if (option_variables.count("version")) - { - SimpleLogger().Write() << g_GIT_DESCRIPTION; - return false; - } - - if (option_variables.count("help")) - { - SimpleLogger().Write() << "\n" << visible_options; - return false; - } - - boost::program_options::notify(option_variables); - - if (!option_variables.count("restrictions")) - { - restrictions_path = std::string(osrm_input_path.string() + ".restrictions"); - } - - if (!option_variables.count("input")) - { - SimpleLogger().Write() << "\n" << visible_options; - return false; - } - - return true; -} - /** \brief Setups scripting environment (lua-scripting) Also initializes speed profile. @@ -414,10 +269,10 @@ void Prepare::SetupScriptingEnvironment( luaL_openlibs(lua_state); // adjust lua load path - luaAddScriptFolderToLoadPath(lua_state, profile_path.string().c_str()); + luaAddScriptFolderToLoadPath(lua_state, config.profile_path.string().c_str()); // Now call our function in a lua script - if (0 != luaL_dofile(lua_state, profile_path.string().c_str())) + if (0 != luaL_dofile(lua_state, config.profile_path.string().c_str())) { std::stringstream msg; msg << lua_tostring(lua_state, -1) << " occured in scripting block"; @@ -451,7 +306,7 @@ void Prepare::SetupScriptingEnvironment( void Prepare::LoadRestrictionMap(const std::unordered_map &external_to_internal_node_map, RestrictionMap &restriction_map) { - boost::filesystem::ifstream input_stream(restrictions_path, std::ios::in | std::ios::binary); + boost::filesystem::ifstream input_stream(config.restrictions_path, std::ios::in | std::ios::binary); std::vector restriction_list; loadRestrictionsFromFile(input_stream, external_to_internal_node_map, restriction_list); @@ -473,7 +328,7 @@ Prepare::LoadNodeBasedGraph(std::vector &barrier_node_list, std::vector edge_list; std::unordered_map external_to_internal_node_map; - boost::filesystem::ifstream input_stream(osrm_input_path, std::ios::in | std::ios::binary); + boost::filesystem::ifstream input_stream(config.osrm_input_path, std::ios::in | std::ios::binary); NodeID number_of_node_based_nodes = loadNodesFromFile(input_stream, barrier_node_list, traffic_light_list, @@ -527,7 +382,7 @@ Prepare::BuildEdgeExpandedGraph(std::vector &internal_to_external_nod internal_to_external_node_map, speed_profile); - edge_based_graph_factory.Run(edge_output_path, geometry_output_path, lua_state); + edge_based_graph_factory.Run(config.edge_output_path, config.geometry_output_path, lua_state); lua_close(lua_state); const std::size_t number_of_edge_based_nodes = @@ -558,7 +413,7 @@ void Prepare::ContractGraph(const std::size_t number_of_edge_based_nodes, */ void Prepare::WriteNodeMapping(std::unique_ptr> internal_to_external_node_map) { - boost::filesystem::ofstream node_stream(node_output_path, std::ios::binary); + boost::filesystem::ofstream node_stream(config.node_output_path, std::ios::binary); const unsigned size_of_mapping = internal_to_external_node_map->size(); node_stream.write((char *)&size_of_mapping, sizeof(unsigned)); if (size_of_mapping > 0) @@ -576,6 +431,6 @@ void Prepare::WriteNodeMapping(std::unique_ptr> internal_ */ void Prepare::BuildRTree(const std::vector &node_based_edge_list, const std::vector& internal_to_external_node_map) { - StaticRTree(node_based_edge_list, rtree_nodes_output_path.c_str(), - rtree_leafs_output_path.c_str(), internal_to_external_node_map); + StaticRTree(node_based_edge_list, config.rtree_nodes_output_path.c_str(), + config.rtree_leafs_output_path.c_str(), internal_to_external_node_map); } diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index a07623a59..2e4b2cd1f 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -28,6 +28,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef PROCESSING_CHAIN_HPP #define PROCESSING_CHAIN_HPP +#include "contractor_options.hpp" #include "edge_based_graph_factory.hpp" #include "../data_structures/query_edge.hpp" #include "../data_structures/static_graph.hpp" @@ -50,14 +51,14 @@ class Prepare using InputEdge = DynamicGraph::InputEdge; using StaticEdge = StaticGraph::InputEdge; - explicit Prepare(); + explicit Prepare(const ContractorConfig& contractor_config) + : config(contractor_config) {} Prepare(const Prepare &) = delete; ~Prepare(); - int Process(int argc, char *argv[]); + int Run(); protected: - bool ParseArguments(int argc, char *argv[]); void SetupScriptingEnvironment(lua_State *myLuaState, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile); void LoadRestrictionMap(const std::unordered_map &external_to_internal_node_map, @@ -80,22 +81,8 @@ class Prepare void WriteNodeMapping(std::unique_ptr> internal_to_external_node_map); void BuildRTree(const std::vector &node_based_edge_list, const std::vector &internal_to_external_node_map); - private: - - unsigned requested_num_threads; - boost::filesystem::path config_file_path; - boost::filesystem::path osrm_input_path; - boost::filesystem::path restrictions_path; - boost::filesystem::path preinfo_path; - boost::filesystem::path profile_path; - - std::string node_output_path; - std::string edge_output_path; - std::string geometry_output_path; - std::string graph_output_path; - std::string rtree_nodes_output_path; - std::string rtree_leafs_output_path; + ContractorConfig config; }; #endif // PROCESSING_CHAIN_HPP diff --git a/extract.cpp b/extract.cpp index a14762983..5f88cc301 100644 --- a/extract.cpp +++ b/extract.cpp @@ -73,7 +73,7 @@ int main(int argc, char *argv[]) << " not found!"; return 1; } - return extractor().run(extractor_config); + return extractor(extractor_config).run(); } catch (const std::exception &e) { diff --git a/extractor/extractor.cpp b/extractor/extractor.cpp index fce0f8898..90fad4eec 100644 --- a/extractor/extractor.cpp +++ b/extractor/extractor.cpp @@ -82,7 +82,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * .restrictions : Turn restrictions that are used my osrm-prepare to construct the edge-expanded graph * */ -int extractor::run(const ExtractorConfig &extractor_config) +int extractor::run() { try { @@ -91,20 +91,20 @@ int extractor::run(const ExtractorConfig &extractor_config) const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); const auto number_of_threads = - std::min(recommended_num_threads, extractor_config.requested_num_threads); + std::min(recommended_num_threads, config.requested_num_threads); tbb::task_scheduler_init init(number_of_threads); - SimpleLogger().Write() << "Input file: " << extractor_config.input_path.filename().string(); - SimpleLogger().Write() << "Profile: " << extractor_config.profile_path.filename().string(); + SimpleLogger().Write() << "Input file: " << config.input_path.filename().string(); + SimpleLogger().Write() << "Profile: " << config.profile_path.filename().string(); SimpleLogger().Write() << "Threads: " << number_of_threads; // setup scripting environment - ScriptingEnvironment scripting_environment(extractor_config.profile_path.string().c_str()); + ScriptingEnvironment scripting_environment(config.profile_path.string().c_str()); ExtractionContainers extraction_containers; auto extractor_callbacks = osrm::make_unique(extraction_containers); - const osmium::io::File input_file(extractor_config.input_path.string()); + const osmium::io::File input_file(config.input_path.string()); osmium::io::Reader reader(input_file); const osmium::io::Header header = reader.header(); @@ -131,7 +131,7 @@ int extractor::run(const ExtractorConfig &extractor_config) } SimpleLogger().Write() << "timestamp: " << timestamp; - boost::filesystem::ofstream timestamp_out(extractor_config.timestamp_file_name); + boost::filesystem::ofstream timestamp_out(config.timestamp_file_name); timestamp_out.write(timestamp.c_str(), timestamp.length()); timestamp_out.close(); @@ -236,12 +236,12 @@ int extractor::run(const ExtractorConfig &extractor_config) return 1; } - extraction_containers.PrepareData(extractor_config.output_file_name, - extractor_config.restriction_file_name); + extraction_containers.PrepareData(config.output_file_name, + config.restriction_file_name); TIMER_STOP(extracting); SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s"; SimpleLogger().Write() << "To prepare the data for routing, run: " - << "./osrm-prepare " << extractor_config.output_file_name + << "./osrm-prepare " << config.output_file_name << std::endl; } catch (std::exception &e) diff --git a/extractor/extractor.hpp b/extractor/extractor.hpp index 8ea56c3cb..72defca05 100644 --- a/extractor/extractor.hpp +++ b/extractor/extractor.hpp @@ -30,8 +30,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "extractor_options.hpp" -struct extractor +class extractor { - int run(const ExtractorConfig &extractor_config); +public: + extractor(const ExtractorConfig &extractor_config) + : config(extractor_config) {} + int run(); +private: + ExtractorConfig config; }; #endif /* EXTRACTOR_HPP */ diff --git a/prepare.cpp b/prepare.cpp index af10df035..5de320463 100644 --- a/prepare.cpp +++ b/prepare.cpp @@ -26,10 +26,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "contractor/processing_chain.hpp" +#include "contractor/contractor_options.hpp" #include "util/simple_logger.hpp" #include +#include + #include #include @@ -37,21 +40,64 @@ int main(int argc, char *argv[]) { try { - return Prepare().Process(argc, argv); - } - catch (boost::program_options::too_many_positional_options_error &) - { - SimpleLogger().Write(logWARNING) << "Only one file can be specified"; - return 1; - } - catch (boost::program_options::error &e) - { - SimpleLogger().Write(logWARNING) << e.what(); - return 1; + LogPolicy::GetInstance().Unmute(); + ContractorConfig contractor_config; + + const return_code result = ContractorOptions::ParseArguments(argc, argv, contractor_config); + + if (return_code::fail == result) + { + return 1; + } + + if (return_code::exit == result) + { + return 0; + } + + ContractorOptions::GenerateOutputFilesNames(contractor_config); + + if (1 > contractor_config.requested_num_threads) + { + SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger"; + return 1; + } + + const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); + + if (recommended_num_threads != contractor_config.requested_num_threads) + { + SimpleLogger().Write(logWARNING) << "The recommended number of threads is " + << recommended_num_threads + << "! This setting may have performance side-effects."; + } + + if (!boost::filesystem::is_regular_file(contractor_config.osrm_input_path)) + { + SimpleLogger().Write(logWARNING) + << "Input file " << contractor_config.osrm_input_path.string() << " not found!"; + return 1; + } + + if (!boost::filesystem::is_regular_file(contractor_config.profile_path)) + { + SimpleLogger().Write(logWARNING) << "Profile " << contractor_config.profile_path.string() + << " not found!"; + return 1; + } + + SimpleLogger().Write() << "Input file: " << contractor_config.osrm_input_path.filename().string(); + SimpleLogger().Write() << "Restrictions file: " << contractor_config.restrictions_path.filename().string(); + SimpleLogger().Write() << "Profile: " << contractor_config.profile_path.filename().string(); + SimpleLogger().Write() << "Threads: " << contractor_config.requested_num_threads; + + tbb::task_scheduler_init init(contractor_config.requested_num_threads); + + return Prepare(contractor_config).Run(); } catch (const std::exception &e) { - SimpleLogger().Write(logWARNING) << "Exception occured: " << e.what() << std::endl; + SimpleLogger().Write(logWARNING) << "[exception] " << e.what(); return 1; } } From e76d8df246eac6bf94729139afc07bd21506379d Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 24 Apr 2015 15:31:26 +0200 Subject: [PATCH 37/87] Fix tools to build with new graph reader interface --- CMakeLists.txt | 4 +- tools/components.cpp | 147 ++++++++++++++--------------- tools/graph_compare.cpp | 199 ---------------------------------------- 3 files changed, 68 insertions(+), 282 deletions(-) delete mode 100644 tools/graph_compare.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index dd862d8dc..6fbf409e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -328,12 +328,10 @@ if(WITH_TOOLS OR BUILD_TOOLS) if(UNIX AND NOT APPLE) target_link_libraries(osrm-unlock-all rt) endif() - add_executable(osrm-check-hsgr tools/check-hsgr.cpp $ $ $) + add_executable(osrm-check-hsgr tools/check-hsgr.cpp $ $ $ $) target_link_libraries(osrm-check-hsgr ${Boost_LIBRARIES}) add_executable(osrm-springclean tools/springclean.cpp $ $ $ $) target_link_libraries(osrm-springclean ${Boost_LIBRARIES}) - add_executable(osrm-graph-compare tools/graph_compare.cpp $ $ $ $ $ $ $) - target_link_libraries(osrm-graph-compare ${Boost_LIBRARIES} ${TBB_LIBRARIES}) install(TARGETS osrm-cli DESTINATION bin) install(TARGETS osrm-io-benchmark DESTINATION bin) diff --git a/tools/components.cpp b/tools/components.cpp index 7ac64f8d9..f35d7196d 100644 --- a/tools/components.cpp +++ b/tools/components.cpp @@ -76,12 +76,73 @@ void DeleteFileIfExists(const std::string &file_name) } } +void LoadRestrictions(const char* path, + std::unique_ptr> ext_to_int_id_map, + std::vector& restriction_list) +{ + std::ifstream input_stream(path, std::ios::binary); + if (!input_stream.is_open()) + { + throw osrm::exception("Cannot open restriction file"); + } + loadRestrictionsFromFile(input_stream, *ext_to_int_id_map, restriction_list); +} + +std::size_t LoadGraph(const char* path, + std::vector& coordinate_list, + std::vector& barrier_node_list, + std::unordered_map& ext_to_int_id_map, + std::vector& graph_edge_list) +{ + std::ifstream input_stream(path, std::ifstream::in | std::ifstream::binary); + if (!input_stream.is_open()) + { + throw osrm::exception("Cannot open osrm file"); + } + + // load graph data + std::vector edge_list; + std::vector traffic_light_node_list; + + auto number_of_nodes = loadNodesFromFile(input_stream, barrier_node_list, + traffic_light_node_list, + coordinate_list, + ext_to_int_id_map); + + auto number_of_edges = loadEdgesFromFile(input_stream, ext_to_int_id_map, edge_list); + + traffic_light_node_list.clear(); + traffic_light_node_list.shrink_to_fit(); + + // Building an node-based graph + for (const auto &input_edge : edge_list) + { + if (input_edge.source == input_edge.target) + { + continue; + } + + if (input_edge.forward) + { + graph_edge_list.emplace_back(input_edge.source, input_edge.target, + (std::max)(input_edge.weight, 1), input_edge.name_id); + } + if (input_edge.backward) + { + graph_edge_list.emplace_back(input_edge.target, input_edge.source, + (std::max)(input_edge.weight, 1), input_edge.name_id); + } + } + + return number_of_nodes; +} + int main(int argc, char *argv[]) { std::vector coordinate_list; std::vector restriction_list; - std::vector bollard_node_list; - std::vector traffic_lights_list; + std::vector barrier_node_list; + auto ext_to_int_id_map = osrm::make_unique>(); LogPolicy::GetInstance().Unmute(); try @@ -95,83 +156,10 @@ int main(int argc, char *argv[]) } SimpleLogger().Write() << "Using restrictions from file: " << argv[2]; - std::ifstream restriction_ifstream(argv[2], std::ios::binary); - const FingerPrint fingerprint_orig; - FingerPrint fingerprint_loaded; - restriction_ifstream.read(reinterpret_cast(&fingerprint_loaded), - sizeof(FingerPrint)); - // check fingerprint and warn if necessary - if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) - { - SimpleLogger().Write(logWARNING) << argv[2] << " was prepared with a different build. " - "Reprocess to get rid of this warning."; - } - - if (!restriction_ifstream.good()) - { - throw osrm::exception("Could not access files"); - } - uint32_t usable_restrictions = 0; - restriction_ifstream.read(reinterpret_cast(&usable_restrictions), sizeof(uint32_t)); - restriction_list.resize(usable_restrictions); - - // load restrictions - if (usable_restrictions > 0) - { - restriction_ifstream.read(reinterpret_cast(&restriction_list[0]), - usable_restrictions * sizeof(TurnRestriction)); - } - restriction_ifstream.close(); - - std::ifstream input_stream(argv[1], std::ifstream::in | std::ifstream::binary); - if (!input_stream.is_open()) - { - throw osrm::exception("Cannot open osrm file"); - } - - // load graph data - std::vector edge_list; - const NodeID number_of_nodes = - readBinaryOSRMGraphFromStream(input_stream, edge_list, bollard_node_list, - traffic_lights_list, &coordinate_list, restriction_list); - input_stream.close(); - - BOOST_ASSERT_MSG(restriction_list.size() == usable_restrictions, - "size of restriction_list changed"); - - SimpleLogger().Write() << restriction_list.size() << " restrictions, " - << bollard_node_list.size() << " bollard nodes, " - << traffic_lights_list.size() << " traffic lights"; - - traffic_lights_list.clear(); - traffic_lights_list.shrink_to_fit(); - - // Building an node-based graph std::vector graph_edge_list; -// DeallocatingVector graph_edge_list; - for (const auto &input_edge : edge_list) - { - if (input_edge.source == input_edge.target) - { - continue; - } - - if (input_edge.forward) - { - graph_edge_list.emplace_back(input_edge.source, input_edge.target, - (std::max)(input_edge.weight, 1), input_edge.name_id); - } - if (input_edge.backward) - { - graph_edge_list.emplace_back(input_edge.target, input_edge.source, - (std::max)(input_edge.weight, 1), input_edge.name_id); - } - } - edge_list.clear(); - edge_list.shrink_to_fit(); - BOOST_ASSERT_MSG(0 == edge_list.size() && 0 == edge_list.capacity(), - "input edge vector not properly deallocated"); + auto number_of_nodes = LoadGraph(argv[1], coordinate_list, barrier_node_list, *ext_to_int_id_map, graph_edge_list); + LoadRestrictions(argv[2], std::move(ext_to_int_id_map), restriction_list); tbb::parallel_sort(graph_edge_list.begin(), graph_edge_list.end()); const auto graph = std::make_shared(number_of_nodes, graph_edge_list); @@ -181,9 +169,8 @@ int main(int argc, char *argv[]) SimpleLogger().Write() << "Starting SCC graph traversal"; RestrictionMap restriction_map(restriction_list); - auto tarjan = osrm::make_unique>(graph, - restriction_map, - bollard_node_list); + auto tarjan = osrm::make_unique>(graph, restriction_map, + barrier_node_list); tarjan->run(); SimpleLogger().Write() << "identified: " << tarjan->get_number_of_components() << " many components"; diff --git a/tools/graph_compare.cpp b/tools/graph_compare.cpp deleted file mode 100644 index 8adc06e78..000000000 --- a/tools/graph_compare.cpp +++ /dev/null @@ -1,199 +0,0 @@ -#include "../data_structures/dynamic_graph.hpp" -#include "../data_structures/import_edge.hpp" -#include "../data_structures/query_node.hpp" -#include "../data_structures/restriction.hpp" -#include "../data_structures/static_graph.hpp" -#include "../util/fingerprint.hpp" -#include "../util/graph_loader.hpp" -#include "../util/integer_range.hpp" -#include "../util/make_unique.hpp" -#include "../util/osrm_exception.hpp" -#include "../util/simple_logger.hpp" - -#include "../typedefs.h" - -#include -#include - -struct TarjanEdgeData -{ - TarjanEdgeData() : distance(INVALID_EDGE_WEIGHT), name_id(INVALID_NAMEID) {} - TarjanEdgeData(unsigned distance, unsigned name_id) : distance(distance), name_id(name_id) {} - unsigned distance; - unsigned name_id; -}; - -using StaticTestGraph = StaticGraph; -using DynamicTestGraph = StaticGraph; -using StaticEdge = StaticTestGraph::InputEdge; -using DynamicEdge = DynamicTestGraph::InputEdge; - -int main(int argc, char *argv[]) -{ - std::vector coordinate_list; - std::vector restriction_list; - std::vector bollard_node_list; - std::vector traffic_lights_list; - - LogPolicy::GetInstance().Unmute(); - try - { - // enable logging - if (argc < 3) - { - SimpleLogger().Write(logWARNING) << "usage:\n" << argv[0] - << " "; - return -1; - } - - SimpleLogger().Write() << "Using restrictions from file: " << argv[2]; - std::ifstream restriction_ifstream(argv[2], std::ios::binary); - const FingerPrint fingerprint_orig; - FingerPrint fingerprint_loaded; - restriction_ifstream.read(reinterpret_cast(&fingerprint_loaded), - sizeof(FingerPrint)); - - // check fingerprint and warn if necessary - if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) - { - SimpleLogger().Write(logWARNING) << argv[2] << " was prepared with a different build. " - "Reprocess to get rid of this warning."; - } - - if (!restriction_ifstream.good()) - { - throw osrm::exception("Could not access files"); - } - uint32_t usable_restrictions = 0; - restriction_ifstream.read(reinterpret_cast(&usable_restrictions), sizeof(uint32_t)); - restriction_list.resize(usable_restrictions); - - // load restrictions - if (usable_restrictions > 0) - { - restriction_ifstream.read(reinterpret_cast(&restriction_list[0]), - usable_restrictions * sizeof(TurnRestriction)); - } - restriction_ifstream.close(); - - std::ifstream input_stream(argv[1], std::ifstream::in | std::ifstream::binary); - if (!input_stream.is_open()) - { - throw osrm::exception("Cannot open osrm file"); - } - - // load graph data - std::vector edge_list; - const NodeID number_of_nodes = - readBinaryOSRMGraphFromStream(input_stream, edge_list, bollard_node_list, - traffic_lights_list, &coordinate_list, restriction_list); - input_stream.close(); - - BOOST_ASSERT_MSG(restriction_list.size() == usable_restrictions, - "size of restriction_list changed"); - - SimpleLogger().Write() << restriction_list.size() << " restrictions, " - << bollard_node_list.size() << " bollard nodes, " - << traffic_lights_list.size() << " traffic lights"; - - traffic_lights_list.clear(); - traffic_lights_list.shrink_to_fit(); - - // Building an node-based graph - std::vector static_graph_edge_list; - std::vector dynamic_graph_edge_list; - for (const auto &input_edge : edge_list) - { - if (input_edge.source == input_edge.target) - { - continue; - } - - if (input_edge.forward) - { - static_graph_edge_list.emplace_back(input_edge.source, input_edge.target, - (std::max)(input_edge.weight, 1), - input_edge.name_id); - dynamic_graph_edge_list.emplace_back(input_edge.source, input_edge.target, - (std::max)(input_edge.weight, 1), - input_edge.name_id); - } - if (input_edge.backward) - { - dynamic_graph_edge_list.emplace_back(input_edge.target, input_edge.source, - (std::max)(input_edge.weight, 1), - input_edge.name_id); - static_graph_edge_list.emplace_back(input_edge.target, input_edge.source, - (std::max)(input_edge.weight, 1), - input_edge.name_id); - } - } - edge_list.clear(); - edge_list.shrink_to_fit(); - BOOST_ASSERT_MSG(0 == edge_list.size() && 0 == edge_list.capacity(), - "input edge vector not properly deallocated"); - - tbb::parallel_sort(static_graph_edge_list.begin(), static_graph_edge_list.end()); - tbb::parallel_sort(dynamic_graph_edge_list.begin(), dynamic_graph_edge_list.end()); - - auto static_graph = - osrm::make_unique(number_of_nodes, static_graph_edge_list); - auto dynamic_graph = - osrm::make_unique(number_of_nodes, dynamic_graph_edge_list); - - SimpleLogger().Write() << "Starting static/dynamic graph comparison"; - - BOOST_ASSERT(static_graph->GetNumberOfNodes() == dynamic_graph->GetNumberOfNodes()); - BOOST_ASSERT(static_graph->GetNumberOfEdges() == dynamic_graph->GetNumberOfEdges()); - for (const auto node : osrm::irange(0u, static_graph->GetNumberOfNodes())) - { - const auto static_range = static_graph->GetAdjacentEdgeRange(node); - const auto dynamic_range = dynamic_graph->GetAdjacentEdgeRange(node); - SimpleLogger().Write() << "checking node " << node << "/" - << static_graph->GetNumberOfNodes(); - - BOOST_ASSERT(static_range.size() == dynamic_range.size()); - const auto static_begin = static_graph->BeginEdges(node); - const auto dynamic_begin = dynamic_graph->BeginEdges(node); - - // check raw interface - for (const auto i : osrm::irange(0u, static_range.size())) - { - const auto static_target = static_graph->GetTarget(static_begin + i); - const auto dynamic_target = dynamic_graph->GetTarget(dynamic_begin + i); - BOOST_ASSERT(static_target == dynamic_target); - - const auto static_data = static_graph->GetEdgeData(static_begin + i); - const auto dynamic_data = dynamic_graph->GetEdgeData(dynamic_begin + i); - - BOOST_ASSERT(static_data.distance == dynamic_data.distance); - BOOST_ASSERT(static_data.name_id == dynamic_data.name_id); - } - - // check range interface - std::vector static_target_ids, dynamic_target_ids; - std::vector static_edge_data, dynamic_edge_data; - for (const auto static_id : static_range) - { - static_target_ids.push_back(static_graph->GetTarget(static_id)); - static_edge_data.push_back(static_graph->GetEdgeData(static_id)); - } - for (const auto dynamic_id : dynamic_range) - { - dynamic_target_ids.push_back(dynamic_graph->GetTarget(dynamic_id)); - dynamic_edge_data.push_back(dynamic_graph->GetEdgeData(dynamic_id)); - } - BOOST_ASSERT(static_target_ids.size() == dynamic_target_ids.size()); - BOOST_ASSERT(std::equal(std::begin(static_target_ids), std::end(static_target_ids), - std::begin(dynamic_target_ids))); - } - - SimpleLogger().Write() << "Graph comparison finished successfully"; - } - catch (const std::exception &e) - { - SimpleLogger().Write(logWARNING) << "[exception] " << e.what(); - } - - return 0; -} From 17a4463f594125b6cefd1e9ec1f7149dbbf456c4 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 4 May 2015 18:43:16 +0200 Subject: [PATCH 38/87] More assertions --- contractor/edge_based_graph_factory.cpp | 30 ++++++++++++++++++++++--- data_structures/dynamic_graph.hpp | 3 +++ data_structures/node_based_graph.hpp | 17 ++++++++++++-- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index 241d43ec9..c760f0f43 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -290,12 +290,33 @@ void EdgeBasedGraphFactory::CompressGeometry() continue; } + /* + * reverse_e2 forward_e2 + * u <---------- v -----------> w + * ----------> <----------- + * forward_e1 reverse_e1 + * + * Will be compressed to: + * + * reverse_e1 + * u <---------- w + * ----------> + * forward_e1 + * + * If the edges are compatible. + * + */ + const bool reverse_edge_order = !(m_node_based_graph->GetEdgeData(m_node_based_graph->BeginEdges(node_v)).forward); const EdgeID forward_e2 = m_node_based_graph->BeginEdges(node_v) + reverse_edge_order; BOOST_ASSERT(SPECIAL_EDGEID != forward_e2); + BOOST_ASSERT(forward_e2 >= m_node_based_graph->BeginEdges(node_v) && + forward_e2 < m_node_based_graph->EndEdges(node_v)); const EdgeID reverse_e2 = m_node_based_graph->BeginEdges(node_v) + 1 - reverse_edge_order; BOOST_ASSERT(SPECIAL_EDGEID != reverse_e2); + BOOST_ASSERT(reverse_e2 >= m_node_based_graph->BeginEdges(node_v) && + reverse_e2 < m_node_based_graph->EndEdges(node_v)); const EdgeData &fwd_edge_data2 = m_node_based_graph->GetEdgeData(forward_e2); const EdgeData &rev_edge_data2 = m_node_based_graph->GetEdgeData(reverse_e2); @@ -325,6 +346,11 @@ void EdgeBasedGraphFactory::CompressGeometry() if ( // TODO: rename to IsCompatibleTo fwd_edge_data1.IsEqualTo(fwd_edge_data2) && rev_edge_data1.IsEqualTo(rev_edge_data2)) { + BOOST_ASSERT(m_node_based_graph->GetEdgeData(forward_e1).nameID == + m_node_based_graph->GetEdgeData(reverse_e1).nameID); + BOOST_ASSERT(m_node_based_graph->GetEdgeData(forward_e2).nameID == + m_node_based_graph->GetEdgeData(reverse_e2).nameID); + // Get distances before graph is modified const int forward_weight1 = m_node_based_graph->GetEdgeData(forward_e1).distance; const int forward_weight2 = m_node_based_graph->GetEdgeData(forward_e2).distance; @@ -336,7 +362,7 @@ void EdgeBasedGraphFactory::CompressGeometry() const int reverse_weight2 = m_node_based_graph->GetEdgeData(reverse_e2).distance; BOOST_ASSERT(0 != reverse_weight1); - BOOST_ASSERT(0 != forward_weight2); + BOOST_ASSERT(0 != reverse_weight2); const bool has_node_penalty = m_traffic_lights.find(node_v) != m_traffic_lights.end(); @@ -378,8 +404,6 @@ void EdgeBasedGraphFactory::CompressGeometry() reverse_weight2 + (has_node_penalty ? speed_profile.traffic_signal_penalty : 0)); ++removed_node_count; - BOOST_ASSERT(m_node_based_graph->GetEdgeData(forward_e1).nameID == - m_node_based_graph->GetEdgeData(reverse_e1).nameID); } } SimpleLogger().Write() << "removed " << removed_node_count << " nodes"; diff --git a/data_structures/dynamic_graph.hpp b/data_structures/dynamic_graph.hpp index 43e6c3aca..b18204c61 100644 --- a/data_structures/dynamic_graph.hpp +++ b/data_structures/dynamic_graph.hpp @@ -85,6 +85,9 @@ template class DynamicGraph edge_list.resize(number_of_nodes); } + /** + * Constructs a DynamicGraph from a list of edges sorted by source node id. + */ template DynamicGraph(const NodeIterator nodes, const ContainerT &graph) { number_of_nodes = nodes; diff --git a/data_structures/node_based_graph.hpp b/data_structures/node_based_graph.hpp index e3a0b594f..eda1752f1 100644 --- a/data_structures/node_based_graph.hpp +++ b/data_structures/node_based_graph.hpp @@ -125,8 +125,15 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector::max(); - // remove parallel edges + // remove parallel edges and set current distance values while (i < edges_list.size() && edges_list[i].source == source && edges_list[i].target == target) { @@ -167,17 +174,23 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector(forward_edge.data.distance) != std::numeric_limits::max()) { forward_edge.data.backward = true; + BOOST_ASSERT(edge_count < i); edges_list[edge_count++] = forward_edge; } } else { // insert seperate edges + // this case can only happen if we merged a bi-directional edge with a directional + // edge above, this incrementing i and making it safe to overwrite the next element + // as well if (static_cast(forward_edge.data.distance) != std::numeric_limits::max()) { + BOOST_ASSERT(edge_count < i); edges_list[edge_count++] = forward_edge; } if (static_cast(reverse_edge.data.distance) != std::numeric_limits::max()) { + BOOST_ASSERT(edge_count < i); edges_list[edge_count++] = reverse_edge; } } From 6ce2726a875c1b98626fdd29ee76bcfb3155c899 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 12 May 2015 01:13:13 +0200 Subject: [PATCH 39/87] Fix return codes for osrm-prepare --- contractor/contractor_options.cpp | 4 ++-- features/options/prepare/help.feature | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contractor/contractor_options.cpp b/contractor/contractor_options.cpp index b6a1e2caa..175613f03 100644 --- a/contractor/contractor_options.cpp +++ b/contractor/contractor_options.cpp @@ -100,13 +100,13 @@ ContractorOptions::ParseArguments(int argc, char *argv[], ContractorConfig &cont if (option_variables.count("version")) { SimpleLogger().Write() << g_GIT_DESCRIPTION; - return return_code::fail; + return return_code::exit; } if (option_variables.count("help")) { SimpleLogger().Write() << "\n" << visible_options; - return return_code::fail; + return return_code::exit; } boost::program_options::notify(option_variables); diff --git a/features/options/prepare/help.feature b/features/options/prepare/help.feature index 49497ae85..dae522d56 100644 --- a/features/options/prepare/help.feature +++ b/features/options/prepare/help.feature @@ -17,7 +17,7 @@ Feature: osrm-prepare command line options: help And stdout should contain "--profile" And stdout should contain "--threads" And stdout should contain 15 lines - And it should exit with code 0 + And it should exit with code 1 Scenario: osrm-prepare - Help, short When I run "osrm-prepare -h" From abc0952247cbf972167ad9d03833bd7aa7687d53 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 28 May 2015 22:22:02 +0200 Subject: [PATCH 40/87] Fix accessing DeallocatingVector --- contractor/processing_chain.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index ffbad0b1f..567ef272a 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -211,7 +211,7 @@ std::size_t Prepare::WriteContractedGraph(unsigned number_of_edge_based_nodes, for (const auto edge : osrm::irange(0, contracted_edge_list->size())) { // no eigen loops - BOOST_ASSERT(contracted_edge_list->(edge).source != contracted_edge_list->(edge).target); + BOOST_ASSERT((*contracted_edge_list)[edge].source != (*contracted_edge_list)[edge].target); current_edge.target = (*contracted_edge_list)[edge].target; current_edge.data = (*contracted_edge_list)[edge].data; @@ -221,12 +221,12 @@ std::size_t Prepare::WriteContractedGraph(unsigned number_of_edge_based_nodes, if (current_edge.data.distance <= 0) { SimpleLogger().Write(logWARNING) << "Edge: " << edge - << ",source: " << contracted_edge_list->at(edge).source - << ", target: " << contracted_edge_list->at(edge).target + << ",source: " << (*contracted_edge_list)[edge].source + << ", target: " << (*contracted_edge_list)[edge].target << ", dist: " << current_edge.data.distance; SimpleLogger().Write(logWARNING) << "Failed at adjacency list of node " - << contracted_edge_list->at(edge).source << "/" + << (*contracted_edge_list)[edge].source << "/" << node_array.size() - 1; return 1; } From 4cab617c25773d6d823d957a5030b0963db6b156 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 29 May 2015 09:13:28 +0200 Subject: [PATCH 41/87] Fix Coverity issue by initializing member --- data_structures/restriction_map.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_structures/restriction_map.hpp b/data_structures/restriction_map.hpp index e2a89a008..98b360c25 100644 --- a/data_structures/restriction_map.hpp +++ b/data_structures/restriction_map.hpp @@ -91,7 +91,7 @@ template <> struct hash class RestrictionMap { public: - RestrictionMap() = default; + RestrictionMap() : m_count(0) {}; RestrictionMap(const std::vector &restriction_list); // Replace end v with w in each turn restriction containing u as via node From e763953562d761cd9ce87f14d425bb2a3c69b46b Mon Sep 17 00:00:00 2001 From: Daniel Patterson Date: Tue, 26 May 2015 02:01:58 -0700 Subject: [PATCH 42/87] Address #1424 by using the original fix --- features/testbot/via.feature | 2 +- routing_algorithms/shortest_path.hpp | 58 +++++++--------------------- 2 files changed, 16 insertions(+), 44 deletions(-) diff --git a/features/testbot/via.feature b/features/testbot/via.feature index 9eb8f556e..7e4efe65e 100644 --- a/features/testbot/via.feature +++ b/features/testbot/via.feature @@ -97,6 +97,6 @@ Feature: Via points | waypoints | route | distance | turns | | 1,3 | ab | 200m +-1 | head,destination | | 3,1 | ab,bc,cd,da,ab | 800m +-1 | head,right,right,right,right,destination | - | 1,2,3 | ab | 200m +-1 | head,destination | + | 1,2,3 | ab,ab | 200m +-1 | head,via,destination | | 1,3,2 | ab,bc,cd,da,ab | 1100m +-1 | head,right,right,right,right,destination | | 3,2,1 | ab,bc,cd,da,ab,bc,cd,da,ab | 1600m +-1 | head,right,right,right,right,right,right,right,right,destination | diff --git a/routing_algorithms/shortest_path.hpp b/routing_algorithms/shortest_path.hpp index 4b3cc49a4..3dedc2b76 100644 --- a/routing_algorithms/shortest_path.hpp +++ b/routing_algorithms/shortest_path.hpp @@ -92,7 +92,9 @@ class ShortestPathRouting final const bool allow_u_turn = current_leg > 0 && uturn_indicators.size() > current_leg && uturn_indicators[current_leg - 1]; - EdgeWeight min_edge_offset = 0; + const EdgeWeight min_edge_offset = + std::min(phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), + phantom_node_pair.source_phantom.GetReverseWeightPlusOffset()); // insert new starting nodes into forward heap, adjusted by previous distances. if ((allow_u_turn || search_from_1st_node) && @@ -100,58 +102,32 @@ class ShortestPathRouting final { forward_heap1.Insert( phantom_node_pair.source_phantom.forward_node_id, - (allow_u_turn ? 0 : distance1) - - phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), + -phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), phantom_node_pair.source_phantom.forward_node_id); - min_edge_offset = - std::min(min_edge_offset, - (allow_u_turn ? 0 : distance1) - - phantom_node_pair.source_phantom.GetForwardWeightPlusOffset()); // SimpleLogger().Write(logDEBUG) << "fwd-a2 insert: " << - // phantom_node_pair.source_phantom.forward_node_id << ", w: " << (allow_u_turn ? 0 - // : distance1) - phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(); + // phantom_node_pair.source_phantom.forward_node_id << ", w: " << -phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(); forward_heap2.Insert( phantom_node_pair.source_phantom.forward_node_id, - (allow_u_turn ? 0 : distance1) - - phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), + -phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), phantom_node_pair.source_phantom.forward_node_id); - min_edge_offset = - std::min(min_edge_offset, - (allow_u_turn ? 0 : distance1) - - phantom_node_pair.source_phantom.GetForwardWeightPlusOffset()); // SimpleLogger().Write(logDEBUG) << "fwd-b2 insert: " << - // phantom_node_pair.source_phantom.forward_node_id << ", w: " << (allow_u_turn ? 0 - // : distance1) - phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(); + // phantom_node_pair.source_phantom.forward_node_id << ", w: " << -phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(); } if ((allow_u_turn || search_from_2nd_node) && phantom_node_pair.source_phantom.reverse_node_id != SPECIAL_NODEID) { forward_heap1.Insert( phantom_node_pair.source_phantom.reverse_node_id, - (allow_u_turn ? 0 : distance2) - - phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(), + -phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(), phantom_node_pair.source_phantom.reverse_node_id); - min_edge_offset = - std::min(min_edge_offset, - (allow_u_turn ? 0 : distance2) - - phantom_node_pair.source_phantom.GetReverseWeightPlusOffset()); // SimpleLogger().Write(logDEBUG) << "fwd-a2 insert: " << - // phantom_node_pair.source_phantom.reverse_node_id << - // ", w: " << (allow_u_turn ? 0 : distance2) - - // phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(); + // phantom_node_pair.source_phantom.reverse_node_id << ", w: " << -phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(); forward_heap2.Insert( phantom_node_pair.source_phantom.reverse_node_id, - (allow_u_turn ? 0 : distance2) - - phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(), + -phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(), phantom_node_pair.source_phantom.reverse_node_id); - min_edge_offset = - std::min(min_edge_offset, - (allow_u_turn ? 0 : distance2) - - phantom_node_pair.source_phantom.GetReverseWeightPlusOffset()); // SimpleLogger().Write(logDEBUG) << "fwd-b2 insert: " << - // phantom_node_pair.source_phantom.reverse_node_id << - // ", w: " << (allow_u_turn ? 0 : distance2) - - // phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(); + // phantom_node_pair.source_phantom.reverse_node_id << ", w: " << -phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(); } // insert new backward nodes into backward heap, unadjusted. @@ -161,9 +137,7 @@ class ShortestPathRouting final phantom_node_pair.target_phantom.GetForwardWeightPlusOffset(), phantom_node_pair.target_phantom.forward_node_id); // SimpleLogger().Write(logDEBUG) << "rev-a insert: " << - // phantom_node_pair.target_phantom.forward_node_id << - // ", w: " << - // phantom_node_pair.target_phantom.GetForwardWeightPlusOffset(); + // phantom_node_pair.target_phantom.forward_node_id << ", w: " << phantom_node_pair.target_phantom.GetForwardWeightPlusOffset(); } if (phantom_node_pair.target_phantom.reverse_node_id != SPECIAL_NODEID) @@ -172,9 +146,7 @@ class ShortestPathRouting final phantom_node_pair.target_phantom.GetReverseWeightPlusOffset(), phantom_node_pair.target_phantom.reverse_node_id); // SimpleLogger().Write(logDEBUG) << "rev-a insert: " << - // phantom_node_pair.target_phantom.reverse_node_id << - // ", w: " << - // phantom_node_pair.target_phantom.GetReverseWeightPlusOffset(); + // phantom_node_pair.target_phantom.reverse_node_id << ", w: " << phantom_node_pair.target_phantom.GetReverseWeightPlusOffset(); } // run two-Target Dijkstra routing step. @@ -332,8 +304,8 @@ class ShortestPathRouting final BOOST_ASSERT(search_from_1st_node != search_from_2nd_node); } - distance1 = local_upper_bound1; - distance2 = local_upper_bound2; + distance1 += local_upper_bound1; + distance2 += local_upper_bound2; ++current_leg; } From f52abc1a62f60afc6eb461a277f38f1cced99f7a Mon Sep 17 00:00:00 2001 From: Daniel Patterson Date: Thu, 28 May 2015 10:51:24 -0700 Subject: [PATCH 43/87] When replacing packed_path1/2, also need to replace distance1/2 to properly track. --- routing_algorithms/shortest_path.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/routing_algorithms/shortest_path.hpp b/routing_algorithms/shortest_path.hpp index 3dedc2b76..9e6b89045 100644 --- a/routing_algorithms/shortest_path.hpp +++ b/routing_algorithms/shortest_path.hpp @@ -272,11 +272,13 @@ class ShortestPathRouting final if (start_id_of_leg1 != last_id_of_packed_legs1) { packed_legs1 = packed_legs2; + distance1 = distance2; BOOST_ASSERT(start_id_of_leg1 == temporary_packed_leg1.front()); } else if (start_id_of_leg2 != last_id_of_packed_legs2) { packed_legs2 = packed_legs1; + distance2 = distance1; BOOST_ASSERT(start_id_of_leg2 == temporary_packed_leg2.front()); } } From 405fcdc48309f6e578fdc3a5a10a722170d07964 Mon Sep 17 00:00:00 2001 From: Daniel Patterson Date: Thu, 28 May 2015 12:57:33 -0700 Subject: [PATCH 44/87] We really need to run these tests. --- features/testbot/via.feature | 1 - 1 file changed, 1 deletion(-) diff --git a/features/testbot/via.feature b/features/testbot/via.feature index 7e4efe65e..5318d15fa 100644 --- a/features/testbot/via.feature +++ b/features/testbot/via.feature @@ -77,7 +77,6 @@ Feature: Via points | 1,3,2 | ab,bc,cd,cd,de,ef,fa,ab,bc | 1600m +-1 | head,straight,straight,via,right,right,right,right,straight,destination | | 3,2,1 | cd,de,ef,fa,ab,bc,bc,cd,de,ef,fa,ab | 2400m +-1 | head,right,right,right,right,straight,via,straight,right,right,right,right,destination | - @bug Scenario: Via points on ring on the same oneway # xa it to avoid only having a single ring, which cna trigger edge cases Given the node map From 039d6acd3e86f557d36aad60b242ef820d9d9775 Mon Sep 17 00:00:00 2001 From: Daniel Patterson Date: Thu, 28 May 2015 14:02:35 -0700 Subject: [PATCH 45/87] Fix test case, it was missing 'via' instructions --- features/testbot/via.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/features/testbot/via.feature b/features/testbot/via.feature index 5318d15fa..52f56a427 100644 --- a/features/testbot/via.feature +++ b/features/testbot/via.feature @@ -97,5 +97,5 @@ Feature: Via points | 1,3 | ab | 200m +-1 | head,destination | | 3,1 | ab,bc,cd,da,ab | 800m +-1 | head,right,right,right,right,destination | | 1,2,3 | ab,ab | 200m +-1 | head,via,destination | - | 1,3,2 | ab,bc,cd,da,ab | 1100m +-1 | head,right,right,right,right,destination | - | 3,2,1 | ab,bc,cd,da,ab,bc,cd,da,ab | 1600m +-1 | head,right,right,right,right,right,right,right,right,destination | + | 1,3,2 | ab,ab,bc,cd,da,ab | 1100m +-1 | head,via,right,right,right,right,destination | + | 3,2,1 | ab,bc,cd,da,ab,ab,bc,cd,da,ab | 1800m | head,right,right,right,right,via,right,right,right,right,destination | From 7d73501b872509cd4208210d23872badb9bd0688 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 30 May 2015 19:26:56 +0200 Subject: [PATCH 46/87] Update bicycle profile to ignore crossing nodes --- profiles/bicycle.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/profiles/bicycle.lua b/profiles/bicycle.lua index 9630a6e4c..4053d3417 100644 --- a/profiles/bicycle.lua +++ b/profiles/bicycle.lua @@ -135,9 +135,14 @@ end function node_function (node, result) -- parse access and barrier tags + local highway = node:get_value_by_key("highway") + local is_crossing = highway and highway == "crossing" + local access = find_access_tag(node, access_tags_hierachy) if access and access ~= "" then - if access_tag_blacklist[access] then + -- access restrictions on crossing nodes are not relevant for + -- the traffic on the road + if access_tag_blacklist[access] and not is_crossing then result.barrier = true end else From 2a9b1311d34cd94dab44ff7e77f850abdbdc2aec Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 30 May 2015 19:30:33 +0200 Subject: [PATCH 47/87] Add test for ignoring crossing nodes --- features/bicycle/access_node.feature | 29 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/features/bicycle/access_node.feature b/features/bicycle/access_node.feature index 5a6f2b5f9..1e08a94ba 100644 --- a/features/bicycle/access_node.feature +++ b/features/bicycle/access_node.feature @@ -7,20 +7,21 @@ Feature: Bike - Access tags on nodes Scenario: Bike - Access tag hierachy on nodes Then routability should be - | node/access | node/vehicle | node/bicycle | bothw | - | | | | x | - | yes | | | x | - | no | | | | - | | yes | | x | - | | no | | | - | no | yes | | x | - | yes | no | | | - | | | yes | x | - | | | no | | - | no | | yes | x | - | yes | | no | | - | | no | yes | x | - | | yes | no | | + | node/access | node/vehicle | node/bicycle | node/highway | bothw | + | | | | | x | + | yes | | | | x | + | no | | | | | + | | yes | | | x | + | | no | | | | + | no | yes | | | x | + | yes | no | | | | + | | | yes | | x | + | | | no | | | + | | | no | crossing | x | + | no | | yes | | x | + | yes | | no | | | + | | no | yes | | x | + | | yes | no | | | Scenario: Bike - Overwriting implied acccess on nodes doesn't overwrite way Then routability should be From a87d89302fb412f4b64a1e77a91fa8cecc70fbfd Mon Sep 17 00:00:00 2001 From: Daniel Patterson Date: Sun, 31 May 2015 09:50:46 -0700 Subject: [PATCH 48/87] Handle POST request when spanning multiple packets --- server/request_parser.cpp | 18 ++++++++++++++++-- server/request_parser.hpp | 1 + 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/server/request_parser.cpp b/server/request_parser.cpp index d57dd305b..07cd52d83 100644 --- a/server/request_parser.cpp +++ b/server/request_parser.cpp @@ -42,7 +42,8 @@ namespace http RequestParser::RequestParser() : state(internal_state::method_start), current_header({"", ""}), - selected_compression(no_compression), is_post_header(false) + selected_compression(no_compression), is_post_header(false), + content_length(0) { } @@ -59,7 +60,7 @@ RequestParser::parse(request ¤t_request, char *begin, char *end) } osrm::tribool result = osrm::tribool::indeterminate; - if(is_post_header && (begin == end)) + if(is_post_header && content_length == 0) { result = osrm::tribool::yes; } @@ -106,6 +107,7 @@ osrm::tribool RequestParser::consume(request ¤t_request, const char input) return osrm::tribool::no; case internal_state::post_request: current_request.uri.push_back(input); + --content_length; return osrm::tribool::indeterminate; case internal_state::method: if (input == ' ') @@ -239,6 +241,17 @@ osrm::tribool RequestParser::consume(request ¤t_request, const char input) { current_request.agent = current_header.value; } + if (boost::iequals(current_header.name, "Content-Length")) + { + try + { + content_length = std::stoi(current_header.value); + } + catch (const std::exception &e) + { + // Ignore the header if the parameter isn't an int + } + } if (input == '\r') { @@ -312,6 +325,7 @@ osrm::tribool RequestParser::consume(request ¤t_request, const char input) { if(is_post_header) { + current_request.uri.push_back('?'); state = internal_state::post_request; return osrm::tribool::indeterminate; } diff --git a/server/request_parser.hpp b/server/request_parser.hpp index 86b08c4cc..3724613a7 100644 --- a/server/request_parser.hpp +++ b/server/request_parser.hpp @@ -90,6 +90,7 @@ class RequestParser header current_header; compression_type selected_compression; bool is_post_header; + int content_length; }; } // namespace http From daa6d02887f3cb79ad53d093b2034d146f986f09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Mon, 1 Jun 2015 09:42:22 +0200 Subject: [PATCH 49/87] Content Type validation added --- server/request_parser.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/request_parser.cpp b/server/request_parser.cpp index 07cd52d83..502e6e0b8 100644 --- a/server/request_parser.cpp +++ b/server/request_parser.cpp @@ -60,7 +60,7 @@ RequestParser::parse(request ¤t_request, char *begin, char *end) } osrm::tribool result = osrm::tribool::indeterminate; - if(is_post_header && content_length == 0) + if(state == internal_state::post_request && content_length <= 0) { result = osrm::tribool::yes; } @@ -252,6 +252,13 @@ osrm::tribool RequestParser::consume(request ¤t_request, const char input) // Ignore the header if the parameter isn't an int } } + if (boost::iequals(current_header.name, "Content-Type")) + { + if (!boost::icontains(current_header.value, "application/x-www-form-urlencoded")) + { + return osrm::tribool::no; + } + } if (input == '\r') { From 4e57e10ba8bcef38affac5300feae9f06cfcbd72 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 30 May 2015 19:00:30 +0200 Subject: [PATCH 50/87] Use 'name (ref)' if both are present --- profiles/car.lua | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/profiles/car.lua b/profiles/car.lua index 3e22afe04..912d98cac 100644 --- a/profiles/car.lua +++ b/profiles/car.lua @@ -345,9 +345,14 @@ function way_function (way, result) local service = way:get_value_by_key("service") -- Set the name that will be used for instructions - if ref and "" ~= ref then + local has_ref = ref and "" ~= ref + local has_name = name and "" ~= name + + if has_name and has_ref then + result.name = name .. " (" .. ref .. ")" + elseif has_ref then result.name = ref - elseif name and "" ~= name then + elseif has_name then result.name = name -- else -- result.name = highway -- if no name exists, use way type From 4146695f3412c895955aab5828c954466662b4e7 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 30 May 2015 19:03:44 +0200 Subject: [PATCH 51/87] Add test to check new way naming convention --- features/car/names.feature | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/features/car/names.feature b/features/car/names.feature index ac4bfc8e2..b6a95ae7f 100644 --- a/features/car/names.feature +++ b/features/car/names.feature @@ -10,13 +10,13 @@ Feature: Car - Street names in instructions | | c | And the ways - | nodes | name | - | ab | My Way | - | bc | Your Way | + | nodes | name | ref | + | ab | My Way | | + | bc | Your Way | A1 | When I route I should get - | from | to | route | - | a | c | My Way,Your Way | + | from | to | route | + | a | c | My Way,Your Way (A1) | @todo Scenario: Car - Use way type to describe unnamed ways From 0aba499c8e8e51a639a551013396de46913c0539 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 29 May 2015 20:48:04 +0200 Subject: [PATCH 52/87] Use spaces instead of tabs --- profiles/foot.lua | 262 +++++++++++++++++++++++----------------------- 1 file changed, 131 insertions(+), 131 deletions(-) diff --git a/profiles/foot.lua b/profiles/foot.lua index 3133cb843..d1f45a6a5 100644 --- a/profiles/foot.lua +++ b/profiles/foot.lua @@ -36,32 +36,32 @@ speeds = { } route_speeds = { - ["ferry"] = 5 + ["ferry"] = 5 } platform_speeds = { - ["platform"] = walking_speed + ["platform"] = walking_speed } amenity_speeds = { - ["parking"] = walking_speed, - ["parking_entrance"] = walking_speed + ["parking"] = walking_speed, + ["parking_entrance"] = walking_speed } man_made_speeds = { - ["pier"] = walking_speed + ["pier"] = walking_speed } surface_speeds = { - ["fine_gravel"] = walking_speed*0.75, - ["gravel"] = walking_speed*0.75, - ["pebbelstone"] = walking_speed*0.75, - ["mud"] = walking_speed*0.5, - ["sand"] = walking_speed*0.5 + ["fine_gravel"] = walking_speed*0.75, + ["gravel"] = walking_speed*0.75, + ["pebbelstone"] = walking_speed*0.75, + ["mud"] = walking_speed*0.5, + ["sand"] = walking_speed*0.5 } -traffic_signal_penalty = 2 -u_turn_penalty = 2 +traffic_signal_penalty = 2 +u_turn_penalty = 2 use_turn_restrictions = false --modes @@ -69,144 +69,144 @@ local mode_normal = 1 local mode_ferry = 2 function get_exceptions(vector) - for i,v in ipairs(restriction_exception_tags) do - vector:Add(v) - end + for i,v in ipairs(restriction_exception_tags) do + vector:Add(v) + end end function node_function (node, result) - local barrier = node:get_value_by_key("barrier") - local access = find_access_tag(node, access_tags_hierachy) - local traffic_signal = node:get_value_by_key("highway") + local barrier = node:get_value_by_key("barrier") + local access = find_access_tag(node, access_tags_hierachy) + local traffic_signal = node:get_value_by_key("highway") - -- flag node if it carries a traffic light - if traffic_signal and traffic_signal == "traffic_signals" then - result.traffic_light = true - end + -- flag node if it carries a traffic light + if traffic_signal and traffic_signal == "traffic_signals" then + result.traffic_light = true + end - -- parse access and barrier tags - if access and access ~= "" then - if access_tag_blacklist[access] then - result.barrier = true - else - result.barrier = false - end - elseif barrier and barrier ~= "" then - if barrier_whitelist[barrier] then - result.barrier = false - else - result.barrier = true - end - end + -- parse access and barrier tags + if access and access ~= "" then + if access_tag_blacklist[access] then + result.barrier = true + else + result.barrier = false + end + elseif barrier and barrier ~= "" then + if barrier_whitelist[barrier] then + result.barrier = false + else + result.barrier = true + end + end - return 1 + return 1 end function way_function (way, result) - -- initial routability check, filters out buildings, boundaries, etc - local highway = way:get_value_by_key("highway") - local route = way:get_value_by_key("route") - local man_made = way:get_value_by_key("man_made") - local railway = way:get_value_by_key("railway") - local amenity = way:get_value_by_key("amenity") - local public_transport = way:get_value_by_key("public_transport") - if (not highway or highway == '') and - (not route or route == '') and - (not railway or railway=='') and - (not amenity or amenity=='') and - (not man_made or man_made=='') and - (not public_transport or public_transport=='') - then - return - end + -- initial routability check, filters out buildings, boundaries, etc + local highway = way:get_value_by_key("highway") + local route = way:get_value_by_key("route") + local man_made = way:get_value_by_key("man_made") + local railway = way:get_value_by_key("railway") + local amenity = way:get_value_by_key("amenity") + local public_transport = way:get_value_by_key("public_transport") + if (not highway or highway == '') and + (not route or route == '') and + (not railway or railway=='') and + (not amenity or amenity=='') and + (not man_made or man_made=='') and + (not public_transport or public_transport=='') + then + return + end - -- don't route on ways that are still under construction - if highway=='construction' then - return - end + -- don't route on ways that are still under construction + if highway=='construction' then + return + end - -- access - local access = find_access_tag(way, access_tags_hierachy) - if access_tag_blacklist[access] then - return - end + -- access + local access = find_access_tag(way, access_tags_hierachy) + if access_tag_blacklist[access] then + return + end - local name = way:get_value_by_key("name") - local ref = way:get_value_by_key("ref") - local junction = way:get_value_by_key("junction") - local onewayClass = way:get_value_by_key("oneway:foot") - local duration = way:get_value_by_key("duration") - local service = way:get_value_by_key("service") - local area = way:get_value_by_key("area") - local foot = way:get_value_by_key("foot") - local surface = way:get_value_by_key("surface") + local name = way:get_value_by_key("name") + local ref = way:get_value_by_key("ref") + local junction = way:get_value_by_key("junction") + local onewayClass = way:get_value_by_key("oneway:foot") + local duration = way:get_value_by_key("duration") + local service = way:get_value_by_key("service") + local area = way:get_value_by_key("area") + local foot = way:get_value_by_key("foot") + local surface = way:get_value_by_key("surface") - -- name - if ref and "" ~= ref and name and "" ~= name then - result.name = name .. ' / ' .. ref + -- name + if ref and "" ~= ref and name and "" ~= name then + result.name = name .. ' / ' .. ref elseif ref and "" ~= ref then - result.name = ref - elseif name and "" ~= name then - result.name = name - elseif highway then - result.name = "{highway:"..highway.."}" -- if no name exists, use way type - -- this encoding scheme is excepted to be a temporary solution - end + result.name = ref + elseif name and "" ~= name then + result.name = name + elseif highway then + result.name = "{highway:"..highway.."}" -- if no name exists, use way type + -- this encoding scheme is excepted to be a temporary solution + end -- roundabouts - if "roundabout" == junction then - result.roundabout = true; - end + if "roundabout" == junction then + result.roundabout = true; + end -- speed - if route_speeds[route] then - -- ferries (doesn't cover routes tagged using relations) - result.ignore_in_grid = true - if duration and durationIsValid(duration) then - result.duration = math.max( 1, parseDuration(duration) ) - else - result.forward_speed = route_speeds[route] - result.backward_speed = route_speeds[route] - end - result.forward_mode = mode_ferry - result.backward_mode = mode_ferry - elseif railway and platform_speeds[railway] then - -- railway platforms (old tagging scheme) - result.forward_speed = platform_speeds[railway] - result.backward_speed = platform_speeds[railway] - elseif platform_speeds[public_transport] then - -- public_transport platforms (new tagging platform) - result.forward_speed = platform_speeds[public_transport] - result.backward_speed = platform_speeds[public_transport] - elseif amenity and amenity_speeds[amenity] then - -- parking areas - result.forward_speed = amenity_speeds[amenity] - result.backward_speed = amenity_speeds[amenity] - elseif speeds[highway] then - -- regular ways - result.forward_speed = speeds[highway] - result.backward_speed = speeds[highway] - elseif access and access_tag_whitelist[access] then - -- unknown way, but valid access tag - result.forward_speed = walking_speed - result.backward_speed = walking_speed - end + if route_speeds[route] then + -- ferries (doesn't cover routes tagged using relations) + result.ignore_in_grid = true + if duration and durationIsValid(duration) then + result.duration = math.max( 1, parseDuration(duration) ) + else + result.forward_speed = route_speeds[route] + result.backward_speed = route_speeds[route] + end + result.forward_mode = mode_ferry + result.backward_mode = mode_ferry + elseif railway and platform_speeds[railway] then + -- railway platforms (old tagging scheme) + result.forward_speed = platform_speeds[railway] + result.backward_speed = platform_speeds[railway] + elseif platform_speeds[public_transport] then + -- public_transport platforms (new tagging platform) + result.forward_speed = platform_speeds[public_transport] + result.backward_speed = platform_speeds[public_transport] + elseif amenity and amenity_speeds[amenity] then + -- parking areas + result.forward_speed = amenity_speeds[amenity] + result.backward_speed = amenity_speeds[amenity] + elseif speeds[highway] then + -- regular ways + result.forward_speed = speeds[highway] + result.backward_speed = speeds[highway] + elseif access and access_tag_whitelist[access] then + -- unknown way, but valid access tag + result.forward_speed = walking_speed + result.backward_speed = walking_speed + end - -- oneway - if onewayClass == "yes" or onewayClass == "1" or onewayClass == "true" then - result.backward_mode = 0 - elseif onewayClass == "no" or onewayClass == "0" or onewayClass == "false" then - -- nothing to do - elseif onewayClass == "-1" then - result.forward_mode = 0 - end + -- oneway + if onewayClass == "yes" or onewayClass == "1" or onewayClass == "true" then + result.backward_mode = 0 + elseif onewayClass == "no" or onewayClass == "0" or onewayClass == "false" then + -- nothing to do + elseif onewayClass == "-1" then + result.forward_mode = 0 + end - -- surfaces - if surface then - surface_speed = surface_speeds[surface] - if surface_speed then - result.forward_speed = math.min(result.forward_speed, surface_speed) - result.backward_speed = math.min(result.backward_speed, surface_speed) - end + -- surfaces + if surface then + surface_speed = surface_speeds[surface] + if surface_speed then + result.forward_speed = math.min(result.forward_speed, surface_speed) + result.backward_speed = math.min(result.backward_speed, surface_speed) end + end end From d85e5def5dbef73b1db5f1e6beab876121b4039b Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 29 May 2015 20:45:34 +0200 Subject: [PATCH 53/87] Add running tracks to foot profile --- profiles/foot.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/profiles/foot.lua b/profiles/foot.lua index d1f45a6a5..231dc3eb3 100644 --- a/profiles/foot.lua +++ b/profiles/foot.lua @@ -60,6 +60,10 @@ surface_speeds = { ["sand"] = walking_speed*0.5 } +leisure_speeds = { + ["track"] = walking_speed +} + traffic_signal_penalty = 2 u_turn_penalty = 2 use_turn_restrictions = false @@ -105,12 +109,14 @@ end function way_function (way, result) -- initial routability check, filters out buildings, boundaries, etc local highway = way:get_value_by_key("highway") + local leisure = way:get_value_by_key("leisure") local route = way:get_value_by_key("route") local man_made = way:get_value_by_key("man_made") local railway = way:get_value_by_key("railway") local amenity = way:get_value_by_key("amenity") local public_transport = way:get_value_by_key("public_transport") if (not highway or highway == '') and + (not leisure or leisure == '') and (not route or route == '') and (not railway or railway=='') and (not amenity or amenity=='') and @@ -182,6 +188,10 @@ function way_function (way, result) -- parking areas result.forward_speed = amenity_speeds[amenity] result.backward_speed = amenity_speeds[amenity] + elseif leisure and leisure_speeds[leisure] then + -- running tracks + result.forward_speed = leisure_speeds[leisure] + result.backward_speed = leisure_speeds[leisure] elseif speeds[highway] then -- regular ways result.forward_speed = speeds[highway] From c493a227650150fdf87b1696281b6a8145ae1b3e Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 30 May 2015 18:25:41 +0200 Subject: [PATCH 54/87] Add test case for leisure=track --- features/foot/way.feature | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/features/foot/way.feature b/features/foot/way.feature index 653c7736c..ff3314f1c 100644 --- a/features/foot/way.feature +++ b/features/foot/way.feature @@ -30,3 +30,9 @@ Feature: Foot - Accessability of different way types | pier | x | | cycleway | | | bridleway | | + + Scenario: Foot - Basic access + Then routability should be + | highway | leisure | forw | + | (nil) | track | x | + From a57fb4f1ab4fce8d38b08e87f269b49aa97806ae Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 9 May 2015 17:21:36 +0200 Subject: [PATCH 55/87] First step into overhauling the edge storage --- CMakeLists.txt | 2 +- contractor/processing_chain.cpp | 6 +- data_structures/import_edge.hpp | 2 - data_structures/node_based_graph.hpp | 6 +- extractor/extraction_containers.cpp | 732 ++++++++++----------- extractor/extraction_containers.hpp | 13 +- extractor/extraction_helper_functions.hpp | 2 +- extractor/extractor.cpp | 3 +- extractor/extractor_callbacks.cpp | 74 ++- extractor/extractor_options.cpp | 4 + extractor/extractor_options.hpp | 1 + extractor/internal_extractor_edge.hpp | 102 +-- tools/components.cpp | 4 +- unit_tests/algorithms/duration_parsing.cpp | 24 +- util/graph_loader.hpp | 75 +-- util/iso_8601_duration_parser.hpp | 2 +- 16 files changed, 530 insertions(+), 522 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6fbf409e7..80eb3d3fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -329,7 +329,7 @@ if(WITH_TOOLS OR BUILD_TOOLS) target_link_libraries(osrm-unlock-all rt) endif() add_executable(osrm-check-hsgr tools/check-hsgr.cpp $ $ $ $) - target_link_libraries(osrm-check-hsgr ${Boost_LIBRARIES}) + target_link_libraries(osrm-check-hsgr ${Boost_LIBRARIES} ${TBB_LIBRARIES}) add_executable(osrm-springclean tools/springclean.cpp $ $ $ $) target_link_libraries(osrm-springclean ${Boost_LIBRARIES}) diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 567ef272a..b51fab76c 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -63,8 +63,8 @@ int Prepare::Run() #ifdef WIN32 #pragma message("Memory consumption on Windows can be higher due to different bit packing") #else - static_assert(sizeof(ImportEdge) == 20, - "changing ImportEdge type has influence on memory consumption!"); + static_assert(sizeof(NodeBasedEdge) == 20, + "changing NodeBasedEdge type has influence on memory consumption!"); static_assert(sizeof(EdgeBasedEdge) == 16, "changing EdgeBasedEdge type has influence on memory consumption!"); #endif @@ -325,7 +325,7 @@ Prepare::LoadNodeBasedGraph(std::vector &barrier_node_list, RestrictionMap &restriction_map, std::vector& internal_to_external_node_map) { - std::vector edge_list; + std::vector edge_list; std::unordered_map external_to_internal_node_map; boost::filesystem::ifstream input_stream(config.osrm_input_path, std::ios::in | std::ios::binary); diff --git a/data_structures/import_edge.hpp b/data_structures/import_edge.hpp index f422de116..27af59846 100644 --- a/data_structures/import_edge.hpp +++ b/data_structures/import_edge.hpp @@ -86,6 +86,4 @@ struct EdgeBasedEdge bool backward : 1; }; -using ImportEdge = NodeBasedEdge; - #endif /* IMPORT_EDGE_HPP */ diff --git a/data_structures/node_based_graph.hpp b/data_structures/node_based_graph.hpp index eda1752f1..f35367924 100644 --- a/data_structures/node_based_graph.hpp +++ b/data_structures/node_based_graph.hpp @@ -74,16 +74,16 @@ struct NodeBasedEdgeData using NodeBasedDynamicGraph = DynamicGraph; -// Factory method to create NodeBasedDynamicGraph from ImportEdges +// Factory method to create NodeBasedDynamicGraph from NodeBasedEdges inline std::shared_ptr -NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector &input_edge_list) +NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector &input_edge_list) { static_assert(sizeof(NodeBasedEdgeData) == 16, "changing node based edge data size changes memory consumption"); DeallocatingVector edges_list; NodeBasedDynamicGraph::InputEdge edge; - for (const ImportEdge &import_edge : input_edge_list) + for (const NodeBasedEdge &import_edge : input_edge_list) { if (import_edge.forward) { diff --git a/extractor/extraction_containers.cpp b/extractor/extraction_containers.cpp index 7c912f33f..281b4cdc1 100644 --- a/extractor/extraction_containers.cpp +++ b/extractor/extraction_containers.cpp @@ -54,6 +54,7 @@ ExtractionContainers::ExtractionContainers() ExtractionContainers::~ExtractionContainers() { + // FIXME isn't this done implicitly of the stxxl::vectors go out of scope? used_node_id_list.clear(); all_nodes_list.clear(); all_edges_list.clear(); @@ -71,394 +72,383 @@ ExtractionContainers::~ExtractionContainers() * - filter nodes list to nodes that are referenced by ways * - merge edges with nodes to include location of start/end points and serialize * - * FIXME: Each of this step should be an own function for readability. */ void ExtractionContainers::PrepareData(const std::string &output_file_name, - const std::string &restrictions_file_name) + const std::string &restrictions_file_name, + const std::string &name_file_name) { try { - unsigned number_of_used_nodes = 0; - unsigned number_of_used_edges = 0; - std::cout << "[extractor] Sorting used nodes ... " << std::flush; - TIMER_START(sorting_used_nodes); - stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory); - TIMER_STOP(sorting_used_nodes); - std::cout << "ok, after " << TIMER_SEC(sorting_used_nodes) << "s" << std::endl; - - std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush; - TIMER_START(erasing_dups); - auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end()); - used_node_id_list.resize(new_end - used_node_id_list.begin()); - TIMER_STOP(erasing_dups); - std::cout << "ok, after " << TIMER_SEC(erasing_dups) << "s" << std::endl; - - std::cout << "[extractor] Sorting all nodes ... " << std::flush; - TIMER_START(sorting_nodes); - stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), ExternalMemoryNodeSTXXLCompare(), - stxxl_memory); - TIMER_STOP(sorting_nodes); - std::cout << "ok, after " << TIMER_SEC(sorting_nodes) << "s" << std::endl; - - std::cout << "[extractor] Sorting used ways ... " << std::flush; - TIMER_START(sort_ways); - stxxl::sort(way_start_end_id_list.begin(), way_start_end_id_list.end(), - FirstAndLastSegmentOfWayStxxlCompare(), stxxl_memory); - TIMER_STOP(sort_ways); - std::cout << "ok, after " << TIMER_SEC(sort_ways) << "s" << std::endl; - - std::cout << "[extractor] Sorting " << restrictions_list.size() - << " restrictions. by from... " << std::flush; - TIMER_START(sort_restrictions); - stxxl::sort(restrictions_list.begin(), restrictions_list.end(), - CmpRestrictionContainerByFrom(), stxxl_memory); - TIMER_STOP(sort_restrictions); - std::cout << "ok, after " << TIMER_SEC(sort_restrictions) << "s" << std::endl; - - std::cout << "[extractor] Fixing restriction starts ... " << std::flush; - TIMER_START(fix_restriction_starts); - auto restrictions_iterator = restrictions_list.begin(); - auto way_start_and_end_iterator = way_start_end_id_list.cbegin(); - - while (way_start_and_end_iterator != way_start_end_id_list.cend() && - restrictions_iterator != restrictions_list.end()) - { - if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.from.way) - { - ++way_start_and_end_iterator; - continue; - } - - if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.from.way) - { - ++restrictions_iterator; - continue; - } - - BOOST_ASSERT(way_start_and_end_iterator->way_id == - restrictions_iterator->restriction.from.way); - const NodeID via_node_id = restrictions_iterator->restriction.via.node; - - if (way_start_and_end_iterator->first_segment_source_id == via_node_id) - { - restrictions_iterator->restriction.from.node = - way_start_and_end_iterator->first_segment_target_id; - } - else if (way_start_and_end_iterator->last_segment_target_id == via_node_id) - { - restrictions_iterator->restriction.from.node = - way_start_and_end_iterator->last_segment_source_id; - } - ++restrictions_iterator; - } - - TIMER_STOP(fix_restriction_starts); - std::cout << "ok, after " << TIMER_SEC(fix_restriction_starts) << "s" << std::endl; - - std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush; - TIMER_START(sort_restrictions_to); - stxxl::sort(restrictions_list.begin(), restrictions_list.end(), - CmpRestrictionContainerByTo(), stxxl_memory); - TIMER_STOP(sort_restrictions_to); - std::cout << "ok, after " << TIMER_SEC(sort_restrictions_to) << "s" << std::endl; - - std::cout << "[extractor] Fixing restriction ends ... " << std::flush; - TIMER_START(fix_restriction_ends); - restrictions_iterator = restrictions_list.begin(); - way_start_and_end_iterator = way_start_end_id_list.cbegin(); - while (way_start_and_end_iterator != way_start_end_id_list.cend() && - restrictions_iterator != restrictions_list.end()) - { - if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.to.way) - { - ++way_start_and_end_iterator; - continue; - } - if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.to.way) - { - ++restrictions_iterator; - continue; - } - BOOST_ASSERT(way_start_and_end_iterator->way_id == - restrictions_iterator->restriction.to.way); - const NodeID via_node_id = restrictions_iterator->restriction.via.node; - - if (way_start_and_end_iterator->first_segment_source_id == via_node_id) - { - restrictions_iterator->restriction.to.node = - way_start_and_end_iterator->first_segment_target_id; - } - else if (way_start_and_end_iterator->last_segment_target_id == via_node_id) - { - restrictions_iterator->restriction.to.node = - way_start_and_end_iterator->last_segment_source_id; - } - ++restrictions_iterator; - } - TIMER_STOP(fix_restriction_ends); - std::cout << "ok, after " << TIMER_SEC(fix_restriction_ends) << "s" << std::endl; - - // serialize restrictions - std::ofstream restrictions_out_stream; - unsigned written_restriction_count = 0; - restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary); - restrictions_out_stream.write((char *)&fingerprint, sizeof(FingerPrint)); - const auto count_position = restrictions_out_stream.tellp(); - restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned)); - - for (const auto &restriction_container : restrictions_list) - { - if (SPECIAL_NODEID != restriction_container.restriction.from.node && - SPECIAL_NODEID != restriction_container.restriction.to.node) - { - restrictions_out_stream.write((char *)&(restriction_container.restriction), - sizeof(TurnRestriction)); - ++written_restriction_count; - } - } - restrictions_out_stream.seekp(count_position); - restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned)); - - restrictions_out_stream.close(); - SimpleLogger().Write() << "usable restrictions: " << written_restriction_count; + PrepareRestrictions(); + WriteRestrictions(restrictions_file_name); std::ofstream file_out_stream; file_out_stream.open(output_file_name.c_str(), std::ios::binary); file_out_stream.write((char *)&fingerprint, sizeof(FingerPrint)); - file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned)); - std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush; - TIMER_START(write_nodes); - // identify all used nodes by a merging step of two sorted lists - auto node_iterator = all_nodes_list.begin(); - auto node_id_iterator = used_node_id_list.begin(); - while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end()) - { - if (*node_id_iterator < node_iterator->node_id) - { - ++node_id_iterator; - continue; - } - if (*node_id_iterator > node_iterator->node_id) - { - ++node_iterator; - continue; - } - BOOST_ASSERT(*node_id_iterator == node_iterator->node_id); - file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode)); + PrepareNodes(); + WriteNodes(file_out_stream); + PrepareEdges(); + WriteEdges(file_out_stream); - ++number_of_used_nodes; - ++node_id_iterator; - ++node_iterator; - } - - TIMER_STOP(write_nodes); - std::cout << "ok, after " << TIMER_SEC(write_nodes) << "s" << std::endl; - - std::cout << "[extractor] setting number of nodes ... " << std::flush; - std::ios::pos_type previous_file_position = file_out_stream.tellp(); - file_out_stream.seekp(std::ios::beg + sizeof(FingerPrint)); - file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned)); - file_out_stream.seekp(previous_file_position); - - std::cout << "ok" << std::endl; - - // Sort edges by start. - std::cout << "[extractor] Sorting edges by start ... " << std::flush; - TIMER_START(sort_edges_by_start); - stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), stxxl_memory); - TIMER_STOP(sort_edges_by_start); - std::cout << "ok, after " << TIMER_SEC(sort_edges_by_start) << "s" << std::endl; - - std::cout << "[extractor] Setting start coords ... " << std::flush; - TIMER_START(set_start_coords); - file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned)); - // Traverse list of edges and nodes in parallel and set start coord - node_iterator = all_nodes_list.begin(); - auto edge_iterator = all_edges_list.begin(); - while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end()) - { - if (edge_iterator->start < node_iterator->node_id) - { - ++edge_iterator; - continue; - } - if (edge_iterator->start > node_iterator->node_id) - { - node_iterator++; - continue; - } - - BOOST_ASSERT(edge_iterator->start == node_iterator->node_id); - edge_iterator->source_coordinate.lat = node_iterator->lat; - edge_iterator->source_coordinate.lon = node_iterator->lon; - ++edge_iterator; - } - TIMER_STOP(set_start_coords); - std::cout << "ok, after " << TIMER_SEC(set_start_coords) << "s" << std::endl; - - // Sort Edges by target - std::cout << "[extractor] Sorting edges by target ... " << std::flush; - TIMER_START(sort_edges_by_target); - stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(), - stxxl_memory); - TIMER_STOP(sort_edges_by_target); - std::cout << "ok, after " << TIMER_SEC(sort_edges_by_target) << "s" << std::endl; - - std::cout << "[extractor] Setting target coords ... " << std::flush; - TIMER_START(set_target_coords); - // Traverse list of edges and nodes in parallel and set target coord - node_iterator = all_nodes_list.begin(); - edge_iterator = all_edges_list.begin(); - - // Also serializes the edges - while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end()) - { - if (edge_iterator->target < node_iterator->node_id) - { - ++edge_iterator; - continue; - } - if (edge_iterator->target > node_iterator->node_id) - { - ++node_iterator; - continue; - } - BOOST_ASSERT(edge_iterator->target == node_iterator->node_id); - if (edge_iterator->source_coordinate.lat != std::numeric_limits::min() && - edge_iterator->source_coordinate.lon != std::numeric_limits::min()) - { - BOOST_ASSERT(edge_iterator->speed != -1); - edge_iterator->target_coordinate.lat = node_iterator->lat; - edge_iterator->target_coordinate.lon = node_iterator->lon; - - const double distance = coordinate_calculation::euclidean_distance( - edge_iterator->source_coordinate.lat, edge_iterator->source_coordinate.lon, - node_iterator->lat, node_iterator->lon); - - const double weight = (distance * 10.) / (edge_iterator->speed / 3.6); - int integer_weight = std::max( - 1, (int)std::floor( - (edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5)); - // FIXME: This means we have a _minimum_ edge length of 1m - // maybe use dm as base unit? - const int integer_distance = std::max(1, (int)distance); - const short zero = 0; - const short one = 1; - const bool yes = true; - const bool no = false; - - file_out_stream.write((char *)&edge_iterator->start, sizeof(unsigned)); - file_out_stream.write((char *)&edge_iterator->target, sizeof(unsigned)); - file_out_stream.write((char *)&integer_distance, sizeof(int)); - switch (edge_iterator->direction) - { - case ExtractionWay::notSure: - file_out_stream.write((char *)&zero, sizeof(short)); - break; - case ExtractionWay::oneway: - file_out_stream.write((char *)&one, sizeof(short)); - break; - case ExtractionWay::bidirectional: - file_out_stream.write((char *)&zero, sizeof(short)); - break; - case ExtractionWay::opposite: - file_out_stream.write((char *)&one, sizeof(short)); - break; - default: - throw osrm::exception("edge has broken direction"); - } - - file_out_stream.write((char *)&integer_weight, sizeof(int)); - file_out_stream.write((char *)&edge_iterator->name_id, sizeof(unsigned)); - if (edge_iterator->is_roundabout) - { - file_out_stream.write((char *)&yes, sizeof(bool)); - } - else - { - file_out_stream.write((char *)&no, sizeof(bool)); - } - if (edge_iterator->is_in_tiny_cc) - { - file_out_stream.write((char *)&yes, sizeof(bool)); - } - else - { - file_out_stream.write((char *)&no, sizeof(bool)); - } - if (edge_iterator->is_access_restricted) - { - file_out_stream.write((char *)&yes, sizeof(bool)); - } - else - { - file_out_stream.write((char *)&no, sizeof(bool)); - } - - // cannot take adress of bit field, so use local - const TravelMode travel_mode = edge_iterator->travel_mode; - file_out_stream.write((char *)&travel_mode, sizeof(TravelMode)); - - if (edge_iterator->is_split) - { - file_out_stream.write((char *)&yes, sizeof(bool)); - } - else - { - file_out_stream.write((char *)&no, sizeof(bool)); - } - ++number_of_used_edges; - } - ++edge_iterator; - } - TIMER_STOP(set_target_coords); - std::cout << "ok, after " << TIMER_SEC(set_target_coords) << "s" << std::endl; - - std::cout << "[extractor] setting number of edges ... " << std::flush; - - file_out_stream.seekp(previous_file_position); - file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned)); file_out_stream.close(); - std::cout << "ok" << std::endl; - std::cout << "[extractor] writing street name index ... " << std::flush; - TIMER_START(write_name_index); - std::string name_file_streamName = (output_file_name + ".names"); - boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary); - - unsigned total_length = 0; - std::vector name_lengths; - for (const std::string &temp_string : name_list) - { - const unsigned string_length = - std::min(static_cast(temp_string.length()), 255u); - name_lengths.push_back(string_length); - total_length += string_length; - } - - RangeTable<> table(name_lengths); - name_file_stream << table; - - name_file_stream.write((char *)&total_length, sizeof(unsigned)); - // write all chars consecutively - for (const std::string &temp_string : name_list) - { - const unsigned string_length = - std::min(static_cast(temp_string.length()), 255u); - name_file_stream.write(temp_string.c_str(), string_length); - } - - name_file_stream.close(); - TIMER_STOP(write_name_index); - std::cout << "ok, after " << TIMER_SEC(write_name_index) << "s" << std::endl; - - SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes and " - << number_of_used_edges << " edges"; + WriteNames(name_file_name); } catch (const std::exception &e) { std::cerr << "Caught Execption:" << e.what() << std::endl; } } + +void ExtractionContainers::WriteNames(const std::string& names_file_name) const +{ + std::cout << "[extractor] writing street name index ... " << std::flush; + TIMER_START(write_name_index); + boost::filesystem::ofstream name_file_stream(names_file_name, std::ios::binary); + + unsigned total_length = 0; + std::vector name_lengths; + for (const std::string &temp_string : name_list) + { + const unsigned string_length = + std::min(static_cast(temp_string.length()), 255u); + name_lengths.push_back(string_length); + total_length += string_length; + } + + // builds and writes the index + RangeTable<> name_index_range(name_lengths); + name_file_stream << name_index_range; + + name_file_stream.write((char *)&total_length, sizeof(unsigned)); + // write all chars consecutively + for (const std::string &temp_string : name_list) + { + const unsigned string_length = + std::min(static_cast(temp_string.length()), 255u); + name_file_stream.write(temp_string.c_str(), string_length); + } + + name_file_stream.close(); + TIMER_STOP(write_name_index); + std::cout << "ok, after " << TIMER_SEC(write_name_index) << "s" << std::endl; +} + +void ExtractionContainers::PrepareNodes() +{ + std::cout << "[extractor] Sorting used nodes ... " << std::flush; + TIMER_START(sorting_used_nodes); + stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory); + TIMER_STOP(sorting_used_nodes); + std::cout << "ok, after " << TIMER_SEC(sorting_used_nodes) << "s" << std::endl; + + std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush; + TIMER_START(erasing_dups); + auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end()); + used_node_id_list.resize(new_end - used_node_id_list.begin()); + TIMER_STOP(erasing_dups); + std::cout << "ok, after " << TIMER_SEC(erasing_dups) << "s" << std::endl; + + std::cout << "[extractor] Sorting all nodes ... " << std::flush; + TIMER_START(sorting_nodes); + stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), ExternalMemoryNodeSTXXLCompare(), + stxxl_memory); + TIMER_STOP(sorting_nodes); + std::cout << "ok, after " << TIMER_SEC(sorting_nodes) << "s" << std::endl; +} + +void ExtractionContainers::PrepareEdges() +{ + // Sort edges by start. + std::cout << "[extractor] Sorting edges by start ... " << std::flush; + TIMER_START(sort_edges_by_start); + stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), stxxl_memory); + TIMER_STOP(sort_edges_by_start); + std::cout << "ok, after " << TIMER_SEC(sort_edges_by_start) << "s" << std::endl; + + std::cout << "[extractor] Setting start coords ... " << std::flush; + TIMER_START(set_start_coords); + // Traverse list of edges and nodes in parallel and set start coord + auto node_iterator = all_nodes_list.begin(); + auto edge_iterator = all_edges_list.begin(); + while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end()) + { + if (edge_iterator->result.source < node_iterator->node_id) + { + ++edge_iterator; + continue; + } + if (edge_iterator->result.source > node_iterator->node_id) + { + node_iterator++; + continue; + } + + BOOST_ASSERT(edge_iterator->result.source == node_iterator->node_id); + edge_iterator->source_coordinate.lat = node_iterator->lat; + edge_iterator->source_coordinate.lon = node_iterator->lon; + ++edge_iterator; + } + TIMER_STOP(set_start_coords); + std::cout << "ok, after " << TIMER_SEC(set_start_coords) << "s" << std::endl; + + // Sort Edges by target + std::cout << "[extractor] Sorting edges by target ... " << std::flush; + TIMER_START(sort_edges_by_target); + stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(), + stxxl_memory); + TIMER_STOP(sort_edges_by_target); + std::cout << "ok, after " << TIMER_SEC(sort_edges_by_target) << "s" << std::endl; + + // Compute edge weights + std::cout << "[extractor] Computing edge weights ... " << std::flush; + TIMER_START(compute_weights); + node_iterator = all_nodes_list.begin(); + edge_iterator = all_edges_list.begin(); + while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end()) + { + if (edge_iterator->result.target < node_iterator->node_id) + { + // FIXME we are skipping edges here: That means the data is broken! + ++edge_iterator; + continue; + } + if (edge_iterator->result.target > node_iterator->node_id) + { + ++node_iterator; + continue; + } + BOOST_ASSERT(edge_iterator->result.target == node_iterator->node_id); + if (edge_iterator->source_coordinate.lat != std::numeric_limits::min() && + edge_iterator->source_coordinate.lon != std::numeric_limits::min()) + { + BOOST_ASSERT(edge_iterator->weight_data.speed != -1); + + const double distance = coordinate_calculation::euclidean_distance( + edge_iterator->source_coordinate.lat, edge_iterator->source_coordinate.lon, + node_iterator->lat, node_iterator->lon); + + const double weight = [distance](const InternalExtractorEdge::WeightData& data) { + switch (data.type) + { + case InternalExtractorEdge::WeightType::EDGE_DURATION: + case InternalExtractorEdge::WeightType::WAY_DURATION: + return data.duration * 10.; + break; + case InternalExtractorEdge::WeightType::SPEED: + return (distance * 10.) / (data.speed / 3.6); + break; + default: + osrm::exception("invalid weight type"); + } + return -1.0; + }(edge_iterator->weight_data); + + edge_iterator->result.weight = std::max(1, (int)std::floor(weight + .5)); + } + ++edge_iterator; + } + TIMER_STOP(compute_weights); + std::cout << "ok, after " << TIMER_SEC(compute_weights) << "s" << std::endl; +} + +void ExtractionContainers::WriteEdges(std::ofstream& file_out_stream) const +{ + std::cout << "[extractor] Writing used egdes ... " << std::flush; + TIMER_START(write_edges); + // Traverse list of edges and nodes in parallel and set target coord + unsigned number_of_used_edges = 0; + + auto start_position = file_out_stream.tellp(); + file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned)); + + for (const auto& edge : all_edges_list) + { + if (edge.result.weight > 0) + { + file_out_stream.write((char*) &edge.result, sizeof(NodeBasedEdge)); + number_of_used_edges++; + } + } + TIMER_STOP(write_edges); + std::cout << "ok, after " << TIMER_SEC(write_edges) << "s" << std::endl; + + std::cout << "[extractor] setting number of edges ... " << std::flush; + file_out_stream.seekp(start_position); + file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned)); + std::cout << "ok" << std::endl; + + SimpleLogger().Write() << "Processed " << number_of_used_edges << " nodes"; +} + +void ExtractionContainers::WriteNodes(std::ofstream& file_out_stream) const +{ + unsigned number_of_used_nodes = 0; + // write dummy value, will be overwritten later + file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned)); + std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush; + TIMER_START(write_nodes); + // identify all used nodes by a merging step of two sorted lists + auto node_iterator = all_nodes_list.begin(); + auto node_id_iterator = used_node_id_list.begin(); + while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end()) + { + if (*node_id_iterator < node_iterator->node_id) + { + ++node_id_iterator; + continue; + } + if (*node_id_iterator > node_iterator->node_id) + { + ++node_iterator; + continue; + } + BOOST_ASSERT(*node_id_iterator == node_iterator->node_id); + + file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode)); + + ++number_of_used_nodes; + ++node_id_iterator; + ++node_iterator; + } + TIMER_STOP(write_nodes); + std::cout << "ok, after " << TIMER_SEC(write_nodes) << "s" << std::endl; + + std::cout << "[extractor] setting number of nodes ... " << std::flush; + std::ios::pos_type previous_file_position = file_out_stream.tellp(); + file_out_stream.seekp(std::ios::beg + sizeof(FingerPrint)); + file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned)); + file_out_stream.seekp(previous_file_position); + std::cout << "ok" << std::endl; + + SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes"; +} + +void ExtractionContainers::WriteRestrictions(const std::string& path) const +{ + // serialize restrictions + std::ofstream restrictions_out_stream; + unsigned written_restriction_count = 0; + restrictions_out_stream.open(path.c_str(), std::ios::binary); + restrictions_out_stream.write((char *)&fingerprint, sizeof(FingerPrint)); + const auto count_position = restrictions_out_stream.tellp(); + restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned)); + + for (const auto &restriction_container : restrictions_list) + { + if (SPECIAL_NODEID != restriction_container.restriction.from.node && + SPECIAL_NODEID != restriction_container.restriction.to.node) + { + restrictions_out_stream.write((char *)&(restriction_container.restriction), + sizeof(TurnRestriction)); + ++written_restriction_count; + } + } + restrictions_out_stream.seekp(count_position); + restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned)); + restrictions_out_stream.close(); + SimpleLogger().Write() << "usable restrictions: " << written_restriction_count; +} + +void ExtractionContainers::PrepareRestrictions() +{ + std::cout << "[extractor] Sorting used ways ... " << std::flush; + TIMER_START(sort_ways); + stxxl::sort(way_start_end_id_list.begin(), way_start_end_id_list.end(), + FirstAndLastSegmentOfWayStxxlCompare(), stxxl_memory); + TIMER_STOP(sort_ways); + std::cout << "ok, after " << TIMER_SEC(sort_ways) << "s" << std::endl; + + std::cout << "[extractor] Sorting " << restrictions_list.size() + << " restrictions. by from... " << std::flush; + TIMER_START(sort_restrictions); + stxxl::sort(restrictions_list.begin(), restrictions_list.end(), + CmpRestrictionContainerByFrom(), stxxl_memory); + TIMER_STOP(sort_restrictions); + std::cout << "ok, after " << TIMER_SEC(sort_restrictions) << "s" << std::endl; + + std::cout << "[extractor] Fixing restriction starts ... " << std::flush; + TIMER_START(fix_restriction_starts); + auto restrictions_iterator = restrictions_list.begin(); + auto way_start_and_end_iterator = way_start_end_id_list.cbegin(); + + while (way_start_and_end_iterator != way_start_end_id_list.cend() && + restrictions_iterator != restrictions_list.end()) + { + if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.from.way) + { + ++way_start_and_end_iterator; + continue; + } + + if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.from.way) + { + ++restrictions_iterator; + continue; + } + + BOOST_ASSERT(way_start_and_end_iterator->way_id == + restrictions_iterator->restriction.from.way); + const NodeID via_node_id = restrictions_iterator->restriction.via.node; + + if (way_start_and_end_iterator->first_segment_source_id == via_node_id) + { + restrictions_iterator->restriction.from.node = + way_start_and_end_iterator->first_segment_target_id; + } + else if (way_start_and_end_iterator->last_segment_target_id == via_node_id) + { + restrictions_iterator->restriction.from.node = + way_start_and_end_iterator->last_segment_source_id; + } + ++restrictions_iterator; + } + + TIMER_STOP(fix_restriction_starts); + std::cout << "ok, after " << TIMER_SEC(fix_restriction_starts) << "s" << std::endl; + + std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush; + TIMER_START(sort_restrictions_to); + stxxl::sort(restrictions_list.begin(), restrictions_list.end(), + CmpRestrictionContainerByTo(), stxxl_memory); + TIMER_STOP(sort_restrictions_to); + std::cout << "ok, after " << TIMER_SEC(sort_restrictions_to) << "s" << std::endl; + + std::cout << "[extractor] Fixing restriction ends ... " << std::flush; + TIMER_START(fix_restriction_ends); + restrictions_iterator = restrictions_list.begin(); + way_start_and_end_iterator = way_start_end_id_list.cbegin(); + while (way_start_and_end_iterator != way_start_end_id_list.cend() && + restrictions_iterator != restrictions_list.end()) + { + if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.to.way) + { + ++way_start_and_end_iterator; + continue; + } + if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.to.way) + { + ++restrictions_iterator; + continue; + } + BOOST_ASSERT(way_start_and_end_iterator->way_id == + restrictions_iterator->restriction.to.way); + const NodeID via_node_id = restrictions_iterator->restriction.via.node; + + if (way_start_and_end_iterator->first_segment_source_id == via_node_id) + { + restrictions_iterator->restriction.to.node = + way_start_and_end_iterator->first_segment_target_id; + } + else if (way_start_and_end_iterator->last_segment_target_id == via_node_id) + { + restrictions_iterator->restriction.to.node = + way_start_and_end_iterator->last_segment_source_id; + } + ++restrictions_iterator; + } + TIMER_STOP(fix_restriction_ends); + std::cout << "ok, after " << TIMER_SEC(fix_restriction_ends) << "s" << std::endl; +} diff --git a/extractor/extraction_containers.hpp b/extractor/extraction_containers.hpp index d04f59549..56109def8 100644 --- a/extractor/extraction_containers.hpp +++ b/extractor/extraction_containers.hpp @@ -39,6 +39,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** * Uses external memory containers from stxxl to store all the data that * is collected by the extractor callbacks. + * + * The data is the filtered, aggregated and finally written to disk. */ class ExtractionContainers { @@ -49,6 +51,14 @@ class ExtractionContainers #else const static unsigned stxxl_memory = ((sizeof(std::size_t) == 4) ? INT_MAX : UINT_MAX); #endif + void PrepareNodes(); + void PrepareRestrictions(); + void PrepareEdges(); + + void WriteNodes(std::ofstream& file_out_stream) const; + void WriteRestrictions(const std::string& restrictions_file_name) const; + void WriteEdges(std::ofstream& file_out_stream) const; + void WriteNames(const std::string& names_file_name) const; public: using STXXLNodeIDVector = stxxl::vector; using STXXLNodeVector = stxxl::vector; @@ -70,7 +80,8 @@ class ExtractionContainers ~ExtractionContainers(); void PrepareData(const std::string &output_file_name, - const std::string &restrictions_file_name); + const std::string &restrictions_file_name, + const std::string &names_file_name); }; #endif /* EXTRACTION_CONTAINERS_HPP */ diff --git a/extractor/extraction_helper_functions.hpp b/extractor/extraction_helper_functions.hpp index d10200abb..e61e54344 100644 --- a/extractor/extraction_helper_functions.hpp +++ b/extractor/extraction_helper_functions.hpp @@ -102,7 +102,7 @@ unsigned parseDuration(const std::string &s) minutes = cast::string_to_int(result[1]); hours = cast::string_to_int(result[0]); } - return 10 * (3600 * hours + 60 * minutes + seconds); + return (3600 * hours + 60 * minutes + seconds); } } else if (iso_8601_duration_is_valid(s)) diff --git a/extractor/extractor.cpp b/extractor/extractor.cpp index 90fad4eec..c1a0a2127 100644 --- a/extractor/extractor.cpp +++ b/extractor/extractor.cpp @@ -237,7 +237,8 @@ int extractor::run() } extraction_containers.PrepareData(config.output_file_name, - config.restriction_file_name); + config.restriction_file_name, + config.names_file_name); TIMER_STOP(extracting); SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s"; SimpleLogger().Write() << "To prepare the data for routing, run: " diff --git a/extractor/extractor_callbacks.cpp b/extractor/extractor_callbacks.cpp index deef831c3..bab309d1d 100644 --- a/extractor/extractor_callbacks.cpp +++ b/extractor/extractor_callbacks.cpp @@ -108,17 +108,40 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti << " of size " << input_way.nodes().size(); return; } + + InternalExtractorEdge::WeightData forward_weight_data; + InternalExtractorEdge::WeightData backward_weight_data; + if (0 < parsed_way.duration) { - // TODO: iterate all way segments and set duration corresponding to the length of each - // segment - const_cast(parsed_way).forward_speed = - parsed_way.duration / (input_way.nodes().size() - 1); - const_cast(parsed_way).backward_speed = - parsed_way.duration / (input_way.nodes().size() - 1); + const unsigned num_edges = (input_way.nodes().size() - 1); + // FIXME We devide by the numer of nodes here, but should rather consider + // the length of each segment. We would eigther have to compute the length + // of the whole way here (we can't: no node coordinates) or push that back + // to the container and keep a reference to the way. + forward_weight_data.duration = parsed_way.duration / num_edges; + forward_weight_data.type = InternalExtractorEdge::WeightType::WAY_DURATION; + backward_weight_data.duration = parsed_way.duration / num_edges; + backward_weight_data.type = InternalExtractorEdge::WeightType::WAY_DURATION; + } + else + { + if (parsed_way.forward_speed > 0 && + parsed_way.forward_travel_mode != TRAVEL_MODE_INACCESSIBLE) + { + forward_weight_data.speed = parsed_way.forward_speed; + forward_weight_data.type = InternalExtractorEdge::WeightType::SPEED; + } + if (parsed_way.backward_speed > 0 && + parsed_way.backward_travel_mode != TRAVEL_MODE_INACCESSIBLE) + { + backward_weight_data.speed = parsed_way.backward_speed; + backward_weight_data.type = InternalExtractorEdge::WeightType::SPEED; + } } - if (std::numeric_limits::epsilon() >= std::abs(-1. - parsed_way.forward_speed)) + if (forward_weight_data.type == InternalExtractorEdge::WeightType::INVALID + && backward_weight_data.type == InternalExtractorEdge::WeightType::INVALID) { SimpleLogger().Write(logDEBUG) << "found way with bogus speed, id: " << input_way.id(); return; @@ -144,36 +167,34 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti ((parsed_way.forward_speed != parsed_way.backward_speed) || (parsed_way.forward_travel_mode != parsed_way.backward_travel_mode)); - auto pair_wise_segment_split = + // lambda to add edge to container + auto pair_wise_segment_split_forward = [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) { - // SimpleLogger().Write() << "adding edge (" << first_node.ref() << "," << - // last_node.ref() << "), fwd speed: " << parsed_way.forward_speed; + const bool forward_only = split_edge || TRAVEL_MODE_INACCESSIBLE == parsed_way.backward_travel_mode; external_memory.all_edges_list.push_back(InternalExtractorEdge( - first_node.ref(), last_node.ref(), - ((split_edge || TRAVEL_MODE_INACCESSIBLE == parsed_way.backward_travel_mode) - ? ExtractionWay::oneway - : ExtractionWay::bidirectional), - parsed_way.forward_speed, name_id, parsed_way.roundabout, parsed_way.ignore_in_grid, - (0 < parsed_way.duration), parsed_way.is_access_restricted, - parsed_way.forward_travel_mode, split_edge)); + first_node.ref(), last_node.ref(), name_id, forward_weight_data, + true, !forward_only, parsed_way.roundabout, parsed_way.ignore_in_grid, + parsed_way.is_access_restricted, parsed_way.forward_travel_mode, split_edge)); external_memory.used_node_id_list.push_back(first_node.ref()); }; const bool is_opposite_way = TRAVEL_MODE_INACCESSIBLE == parsed_way.forward_travel_mode; + // traverse way in reverse in this case if (is_opposite_way) { + // why don't we have to swap the parsed_way.forward/backward speed here as well const_cast(parsed_way).forward_travel_mode = parsed_way.backward_travel_mode; const_cast(parsed_way).backward_travel_mode = TRAVEL_MODE_INACCESSIBLE; osrm::for_each_pair(input_way.nodes().crbegin(), input_way.nodes().crend(), - pair_wise_segment_split); + pair_wise_segment_split_forward); external_memory.used_node_id_list.push_back(input_way.nodes().front().ref()); } else { osrm::for_each_pair(input_way.nodes().cbegin(), input_way.nodes().cend(), - pair_wise_segment_split); + pair_wise_segment_split_forward); external_memory.used_node_id_list.push_back(input_way.nodes().back().ref()); } @@ -187,22 +208,19 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti if (split_edge) { // Only true if the way should be split - BOOST_ASSERT(parsed_way.backward_travel_mode > 0); + BOOST_ASSERT(parsed_way.backward_travel_mode != TRAVEL_MODE_INACCESSIBLE); auto pair_wise_segment_split_2 = [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) { - // SimpleLogger().Write() << "adding edge (" << last_node.ref() << "," << - // first_node.ref() << "), bwd speed: " << parsed_way.backward_speed; external_memory.all_edges_list.push_back(InternalExtractorEdge( - last_node.ref(), first_node.ref(), ExtractionWay::oneway, parsed_way.backward_speed, - name_id, parsed_way.roundabout, parsed_way.ignore_in_grid, - (0 < parsed_way.duration), parsed_way.is_access_restricted, - parsed_way.backward_travel_mode, split_edge)); + last_node.ref(), first_node.ref(), name_id, backward_weight_data, + true, false, parsed_way.roundabout, parsed_way.ignore_in_grid, + parsed_way.is_access_restricted, parsed_way.backward_travel_mode, split_edge)); + external_memory.used_node_id_list.push_back(last_node.ref()); }; if (is_opposite_way) { - // SimpleLogger().Write() << "opposite2"; osrm::for_each_pair(input_way.nodes().crbegin(), input_way.nodes().crend(), pair_wise_segment_split_2); external_memory.used_node_id_list.push_back(input_way.nodes().front().ref()); @@ -211,7 +229,7 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti { osrm::for_each_pair(input_way.nodes().cbegin(), input_way.nodes().cend(), pair_wise_segment_split_2); - external_memory.used_node_id_list.push_back(input_way.nodes().back().ref()); + external_memory.used_node_id_list.push_back(input_way.nodes().front().ref()); } external_memory.way_start_end_id_list.push_back( diff --git a/extractor/extractor_options.cpp b/extractor/extractor_options.cpp index 9ae5cd4c1..23cbacc74 100644 --- a/extractor/extractor_options.cpp +++ b/extractor/extractor_options.cpp @@ -135,6 +135,7 @@ void ExtractorOptions::GenerateOutputFilesNames(ExtractorConfig &extractor_confi boost::filesystem::path &input_path = extractor_config.input_path; extractor_config.output_file_name = input_path.string(); extractor_config.restriction_file_name = input_path.string(); + extractor_config.names_file_name = input_path.string(); extractor_config.timestamp_file_name = input_path.string(); std::string::size_type pos = extractor_config.output_file_name.find(".osm.bz2"); if (pos == std::string::npos) @@ -156,12 +157,14 @@ void ExtractorOptions::GenerateOutputFilesNames(ExtractorConfig &extractor_confi { extractor_config.output_file_name.append(".osrm"); extractor_config.restriction_file_name.append(".osrm.restrictions"); + extractor_config.names_file_name.append(".osrm.names"); extractor_config.timestamp_file_name.append(".osrm.timestamp"); } else { extractor_config.output_file_name.replace(pos, 5, ".osrm"); extractor_config.restriction_file_name.replace(pos, 5, ".osrm.restrictions"); + extractor_config.names_file_name.replace(pos, 5, ".osrm.names"); extractor_config.timestamp_file_name.replace(pos, 5, ".osrm.timestamp"); } } @@ -169,6 +172,7 @@ void ExtractorOptions::GenerateOutputFilesNames(ExtractorConfig &extractor_confi { extractor_config.output_file_name.replace(pos, 8, ".osrm"); extractor_config.restriction_file_name.replace(pos, 8, ".osrm.restrictions"); + extractor_config.names_file_name.replace(pos, 8, ".osrm.names"); extractor_config.timestamp_file_name.replace(pos, 8, ".osrm.timestamp"); } } diff --git a/extractor/extractor_options.hpp b/extractor/extractor_options.hpp index 5712ea01a..c164f6a12 100644 --- a/extractor/extractor_options.hpp +++ b/extractor/extractor_options.hpp @@ -48,6 +48,7 @@ struct ExtractorConfig std::string output_file_name; std::string restriction_file_name; + std::string names_file_name; std::string timestamp_file_name; unsigned requested_num_threads; diff --git a/extractor/internal_extractor_edge.hpp b/extractor/internal_extractor_edge.hpp index 27e1af146..ea76c560b 100644 --- a/extractor/internal_extractor_edge.hpp +++ b/extractor/internal_extractor_edge.hpp @@ -30,6 +30,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../typedefs.h" #include "../data_structures/travel_mode.hpp" +#include "../data_structures/import_edge.hpp" #include @@ -37,83 +38,94 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. struct InternalExtractorEdge { - InternalExtractorEdge() - : start(0), target(0), speed(0), name_id(0), direction(0), is_roundabout(false), - is_in_tiny_cc(false), is_duration_set(false), is_access_restricted(false), - is_split(false), travel_mode(TRAVEL_MODE_INACCESSIBLE) + // specify the type of the weight data + enum class WeightType : char { + INVALID, + SPEED, + EDGE_DURATION, + WAY_DURATION, + }; + + struct WeightData + { + + WeightData() : duration(0.0), type(WeightType::INVALID) + { + } + + union + { + double duration; + double speed; + }; + WeightType type; + }; + + explicit InternalExtractorEdge() + : result(0, 0, 0, 0, false, false, false, false, false, + TRAVEL_MODE_INACCESSIBLE, false) { } - explicit InternalExtractorEdge(NodeID start, - NodeID target, - short direction, - double speed, - unsigned name_id, - bool is_roundabout, - bool is_in_tiny_cc, - bool is_duration_set, - bool is_access_restricted, - TravelMode travel_mode, - bool is_split) - : start(start), target(target), speed(speed), name_id(name_id), direction(direction), - is_roundabout(is_roundabout), is_in_tiny_cc(is_in_tiny_cc), - is_duration_set(is_duration_set), is_access_restricted(is_access_restricted), - is_split(is_split), travel_mode(travel_mode) + explicit InternalExtractorEdge(NodeID source, + NodeID target, + NodeID name_id, + const WeightData& weight_data, + bool forward, + bool backward, + bool roundabout, + bool in_tiny_cc, + bool access_restricted, + TravelMode travel_mode, + bool is_split) + : result(source, target, name_id, 0, forward, backward, roundabout, + in_tiny_cc, access_restricted, travel_mode, is_split), + weight_data(weight_data) { } + // data that will be written to disk + NodeBasedEdge result; + // intermediate edge weight + WeightData weight_data; + // coordinate of the source node + FixedPointCoordinate source_coordinate; + + // necessary static util functions for stxxl's sorting static InternalExtractorEdge min_value() { - return InternalExtractorEdge(0, 0, 0, 0, 0, false, false, false, false, - TRAVEL_MODE_INACCESSIBLE, false); + return InternalExtractorEdge(0, 0, 0, WeightData(), false, false, false, + false, false, TRAVEL_MODE_INACCESSIBLE, false); } static InternalExtractorEdge max_value() { - return InternalExtractorEdge(SPECIAL_NODEID, SPECIAL_NODEID, 0, 0, 0, false, false, false, - false, TRAVEL_MODE_INACCESSIBLE, false); + return InternalExtractorEdge(SPECIAL_NODEID, SPECIAL_NODEID, 0, WeightData(), false, false, + false, false, false, TRAVEL_MODE_INACCESSIBLE, false); } - - NodeID start; - NodeID target; - double speed; - unsigned name_id; - short direction; - bool is_roundabout : 1; - bool is_in_tiny_cc : 1; - bool is_duration_set : 1; - bool is_access_restricted : 1; - bool is_split : 1; - TravelMode travel_mode : 4; - - FixedPointCoordinate source_coordinate; - FixedPointCoordinate target_coordinate; }; struct CmpEdgeByStartID { using value_type = InternalExtractorEdge; - bool operator()(const InternalExtractorEdge &a, const InternalExtractorEdge &b) const + bool operator()(const InternalExtractorEdge &lhs, const InternalExtractorEdge &rhs) const { - return a.start < b.start; + return lhs.result.source < rhs.result.source; } value_type max_value() { return InternalExtractorEdge::max_value(); } - value_type min_value() { return InternalExtractorEdge::min_value(); } }; struct CmpEdgeByTargetID { using value_type = InternalExtractorEdge; - - bool operator()(const InternalExtractorEdge &a, const InternalExtractorEdge &b) const + bool operator()(const InternalExtractorEdge &lhs, const InternalExtractorEdge &rhs) const { - return a.target < b.target; + return lhs.result.target < rhs.result.target; } value_type max_value() { return InternalExtractorEdge::max_value(); } - value_type min_value() { return InternalExtractorEdge::min_value(); } }; diff --git a/tools/components.cpp b/tools/components.cpp index f35d7196d..d0fcd79ab 100644 --- a/tools/components.cpp +++ b/tools/components.cpp @@ -101,7 +101,7 @@ std::size_t LoadGraph(const char* path, } // load graph data - std::vector edge_list; + std::vector edge_list; std::vector traffic_light_node_list; auto number_of_nodes = loadNodesFromFile(input_stream, barrier_node_list, @@ -109,7 +109,7 @@ std::size_t LoadGraph(const char* path, coordinate_list, ext_to_int_id_map); - auto number_of_edges = loadEdgesFromFile(input_stream, ext_to_int_id_map, edge_list); + loadEdgesFromFile(input_stream, ext_to_int_id_map, edge_list); traffic_light_node_list.clear(); traffic_light_node_list.shrink_to_fit(); diff --git a/unit_tests/algorithms/duration_parsing.cpp b/unit_tests/algorithms/duration_parsing.cpp index ec3b8191a..e3035dce7 100644 --- a/unit_tests/algorithms/duration_parsing.cpp +++ b/unit_tests/algorithms/duration_parsing.cpp @@ -41,24 +41,24 @@ BOOST_AUTO_TEST_CASE(all_necessary_test) BOOST_AUTO_TEST_CASE(common_durations_get_translated) { - BOOST_CHECK_EQUAL(parseDuration("00:01"), 600); - BOOST_CHECK_EQUAL(parseDuration("00:01:01"), 610); - BOOST_CHECK_EQUAL(parseDuration("01:01"), 36600); + BOOST_CHECK_EQUAL(parseDuration("00:01"), 60); + BOOST_CHECK_EQUAL(parseDuration("00:01:01"), 61); + BOOST_CHECK_EQUAL(parseDuration("01:01"), 3660); // check all combinations of iso duration tokens - BOOST_CHECK_EQUAL(parseDuration("PT1M1S"), 610); - BOOST_CHECK_EQUAL(parseDuration("PT1H1S"), 36010); - BOOST_CHECK_EQUAL(parseDuration("PT15M"), 9000); - BOOST_CHECK_EQUAL(parseDuration("PT15S"), 150); - BOOST_CHECK_EQUAL(parseDuration("PT15H"), 540000); - BOOST_CHECK_EQUAL(parseDuration("PT1H15M"), 45000); - BOOST_CHECK_EQUAL(parseDuration("PT1H15M1S"), 45010); + BOOST_CHECK_EQUAL(parseDuration("PT1M1S"), 61); + BOOST_CHECK_EQUAL(parseDuration("PT1H1S"), 3601); + BOOST_CHECK_EQUAL(parseDuration("PT15M"), 900); + BOOST_CHECK_EQUAL(parseDuration("PT15S"), 15); + BOOST_CHECK_EQUAL(parseDuration("PT15H"), 54000); + BOOST_CHECK_EQUAL(parseDuration("PT1H15M"), 4500); + BOOST_CHECK_EQUAL(parseDuration("PT1H15M1S"), 4501); } BOOST_AUTO_TEST_CASE(iso_8601_durations_case_insensitive) { - BOOST_CHECK_EQUAL(parseDuration("PT15m"), 9000); - BOOST_CHECK_EQUAL(parseDuration("PT1h15m"), 45000); + BOOST_CHECK_EQUAL(parseDuration("PT15m"), 900); + BOOST_CHECK_EQUAL(parseDuration("PT1h15m"), 4500); } BOOST_AUTO_TEST_SUITE_END() diff --git a/util/graph_loader.hpp b/util/graph_loader.hpp index 3779f0b9e..0736775b9 100644 --- a/util/graph_loader.hpp +++ b/util/graph_loader.hpp @@ -170,83 +170,56 @@ NodeID loadNodesFromFile(std::istream &input_stream, * Reads a .osrm file and produces the edges. Edges reference nodes in the old * OSM based format, we need to renumber it here. */ -template NodeID loadEdgesFromFile(std::istream &input_stream, const std::unordered_map &ext_to_int_id_map, - std::vector &edge_list) + std::vector &edge_list) { - EdgeWeight weight; - NodeID source, target; - unsigned nameID; - int length; - short dir; // direction (0 = open, 1 = forward, 2+ = open) - bool is_roundabout, ignore_in_grid, is_access_restricted, is_split; - TravelMode travel_mode; - EdgeID m; input_stream.read(reinterpret_cast(&m), sizeof(unsigned)); edge_list.reserve(m); SimpleLogger().Write() << " and " << m << " edges "; + NodeBasedEdge edge(0, 0, 0, 0, false, false, false, false, false, TRAVEL_MODE_INACCESSIBLE, false); for (EdgeID i = 0; i < m; ++i) { - input_stream.read(reinterpret_cast(&source), sizeof(unsigned)); - input_stream.read(reinterpret_cast(&target), sizeof(unsigned)); - input_stream.read(reinterpret_cast(&length), sizeof(int)); - input_stream.read(reinterpret_cast(&dir), sizeof(short)); - input_stream.read(reinterpret_cast(&weight), sizeof(int)); - input_stream.read(reinterpret_cast(&nameID), sizeof(unsigned)); - input_stream.read(reinterpret_cast(&is_roundabout), sizeof(bool)); - input_stream.read(reinterpret_cast(&ignore_in_grid), sizeof(bool)); - input_stream.read(reinterpret_cast(&is_access_restricted), sizeof(bool)); - input_stream.read(reinterpret_cast(&travel_mode), sizeof(TravelMode)); - input_stream.read(reinterpret_cast(&is_split), sizeof(bool)); - - BOOST_ASSERT_MSG(length > 0, "loaded null length edge"); - BOOST_ASSERT_MSG(weight > 0, "loaded null weight"); - BOOST_ASSERT_MSG(0 <= dir && dir <= 2, "loaded bogus direction"); - - bool forward = true; - bool backward = true; - if (1 == dir) - { - backward = false; - } - if (2 == dir) - { - forward = false; - } + input_stream.read(reinterpret_cast(&edge), sizeof(NodeBasedEdge)); + BOOST_ASSERT_MSG(edge.weight > 0, "loaded null weight"); + BOOST_ASSERT_MSG(edge.forward || edge.backward, "loaded null weight"); + BOOST_ASSERT_MSG(edge.travel_mode != TRAVEL_MODE_INACCESSIBLE, "loaded null weight"); // translate the external NodeIDs to internal IDs - auto internal_id_iter = ext_to_int_id_map.find(source); - if (ext_to_int_id_map.find(source) == ext_to_int_id_map.end()) + auto internal_id_iter = ext_to_int_id_map.find(edge.source); + if (ext_to_int_id_map.find(edge.source) == ext_to_int_id_map.end()) { #ifndef NDEBUG - SimpleLogger().Write(logWARNING) << " unresolved source NodeID: " << source; + SimpleLogger().Write(logWARNING) << " unresolved source NodeID: " << edge.source; #endif continue; } - source = internal_id_iter->second; - internal_id_iter = ext_to_int_id_map.find(target); - if (ext_to_int_id_map.find(target) == ext_to_int_id_map.end()) + edge.source = internal_id_iter->second; + internal_id_iter = ext_to_int_id_map.find(edge.target); + if (ext_to_int_id_map.find(edge.target) == ext_to_int_id_map.end()) { #ifndef NDEBUG - SimpleLogger().Write(logWARNING) << "unresolved target NodeID : " << target; + SimpleLogger().Write(logWARNING) << "unresolved target NodeID : " << edge.target; #endif continue; } - target = internal_id_iter->second; - BOOST_ASSERT_MSG(source != SPECIAL_NODEID && target != SPECIAL_NODEID, + edge.target = internal_id_iter->second; + BOOST_ASSERT_MSG(edge.source != SPECIAL_NODEID && edge.target != SPECIAL_NODEID, "nonexisting source or target"); - if (source > target) + if (edge.source > edge.target) { - std::swap(source, target); - std::swap(forward, backward); + std::swap(edge.source, edge.target); + + // std::swap does not work with bit-fields + bool temp = edge.forward; + edge.forward = edge.backward; + edge.backward = temp; } - edge_list.emplace_back(source, target, nameID, weight, forward, backward, is_roundabout, - ignore_in_grid, is_access_restricted, travel_mode, is_split); + edge_list.push_back(edge); } tbb::parallel_sort(edge_list.begin(), edge_list.end()); @@ -304,7 +277,7 @@ NodeID loadEdgesFromFile(std::istream &input_stream, } } const auto new_end_iter = - std::remove_if(edge_list.begin(), edge_list.end(), [](const EdgeT &edge) + std::remove_if(edge_list.begin(), edge_list.end(), [](const NodeBasedEdge &edge) { return edge.source == SPECIAL_NODEID || edge.target == SPECIAL_NODEID; }); diff --git a/util/iso_8601_duration_parser.hpp b/util/iso_8601_duration_parser.hpp index 1981c8982..3e558cee2 100644 --- a/util/iso_8601_duration_parser.hpp +++ b/util/iso_8601_duration_parser.hpp @@ -90,7 +90,7 @@ template struct iso_8601_grammar : qi::grammar unsigned get_duration() const { - unsigned temp = 10 * (3600 * hours + 60 * minutes + seconds); + unsigned temp = (3600 * hours + 60 * minutes + seconds); if (temp == 0) { temp = std::numeric_limits::max(); From 3065de63dd55218fcf93d74d2fc188fcdfaae14c Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 13 May 2015 01:26:54 +0200 Subject: [PATCH 56/87] Move renumbering and edge deduplication in extractor --- contractor/edge_based_graph_factory.cpp | 10 +- contractor/processing_chain.cpp | 22 +-- contractor/processing_chain.hpp | 4 +- data_structures/import_edge.cpp | 7 + data_structures/import_edge.hpp | 3 +- data_structures/node_based_graph.hpp | 3 +- extractor/extraction_containers.cpp | 191 +++++++++++++++++++++--- extractor/extraction_containers.hpp | 2 + extractor/internal_extractor_edge.hpp | 14 ++ tools/components.cpp | 17 +-- util/graph_loader.hpp | 156 ++----------------- 11 files changed, 235 insertions(+), 194 deletions(-) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index c760f0f43..6f414ecd9 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -343,8 +343,14 @@ void EdgeBasedGraphFactory::CompressGeometry() continue; } - if ( // TODO: rename to IsCompatibleTo - fwd_edge_data1.IsEqualTo(fwd_edge_data2) && rev_edge_data1.IsEqualTo(rev_edge_data2)) + // this case can happen if two ways with different names overlap + if (fwd_edge_data1.nameID != rev_edge_data1.nameID || + fwd_edge_data2.nameID != rev_edge_data2.nameID) + { + continue; + } + + if (fwd_edge_data1.IsCompatibleTo(fwd_edge_data2) && rev_edge_data1.IsCompatibleTo(rev_edge_data2)) { BOOST_ASSERT(m_node_based_graph->GetEdgeData(forward_e1).nameID == m_node_based_graph->GetEdgeData(reverse_e1).nameID); diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index b51fab76c..b354a62b3 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -303,44 +303,38 @@ void Prepare::SetupScriptingEnvironment( /** \brief Build load restrictions from .restriction file */ -void Prepare::LoadRestrictionMap(const std::unordered_map &external_to_internal_node_map, - RestrictionMap &restriction_map) +std::shared_ptr Prepare::LoadRestrictionMap() { boost::filesystem::ifstream input_stream(config.restrictions_path, std::ios::in | std::ios::binary); std::vector restriction_list; - loadRestrictionsFromFile(input_stream, external_to_internal_node_map, restriction_list); + loadRestrictionsFromFile(input_stream, restriction_list); SimpleLogger().Write() << " - " << restriction_list.size() << " restrictions."; - restriction_map = RestrictionMap(restriction_list); + return std::make_shared(restriction_list); } /** - \brief Build load node based graph from .osrm file and restrictions from .restrictions file + \brief Load node based graph from .osrm file */ std::shared_ptr Prepare::LoadNodeBasedGraph(std::vector &barrier_node_list, std::vector &traffic_light_list, - RestrictionMap &restriction_map, std::vector& internal_to_external_node_map) { std::vector edge_list; - std::unordered_map external_to_internal_node_map; boost::filesystem::ifstream input_stream(config.osrm_input_path, std::ios::in | std::ios::binary); NodeID number_of_node_based_nodes = loadNodesFromFile(input_stream, barrier_node_list, traffic_light_list, - internal_to_external_node_map, - external_to_internal_node_map); + internal_to_external_node_map); SimpleLogger().Write() << " - " << barrier_node_list.size() << " bollard nodes, " << traffic_light_list.size() << " traffic lights"; - loadEdgesFromFile(input_stream, external_to_internal_node_map, edge_list); - - LoadRestrictionMap(external_to_internal_node_map, restriction_map); + loadEdgesFromFile(input_stream, edge_list); if (edge_list.empty()) { @@ -369,9 +363,9 @@ Prepare::BuildEdgeExpandedGraph(std::vector &internal_to_external_nod auto barrier_node_list = osrm::make_unique>(); auto traffic_light_list = osrm::make_unique>(); - auto restriction_map = std::make_shared(); - auto node_based_graph = LoadNodeBasedGraph(*barrier_node_list, *traffic_light_list, *restriction_map, internal_to_external_node_map); + auto restriction_map = LoadRestrictionMap(); + auto node_based_graph = LoadNodeBasedGraph(*barrier_node_list, *traffic_light_list, internal_to_external_node_map); const std::size_t number_of_node_based_nodes = node_based_graph->GetNumberOfNodes(); diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index 2e4b2cd1f..12f74f3fb 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -61,8 +61,7 @@ class Prepare protected: void SetupScriptingEnvironment(lua_State *myLuaState, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile); - void LoadRestrictionMap(const std::unordered_map &external_to_internal_node_map, - RestrictionMap &restriction_map); + std::shared_ptr LoadRestrictionMap(); unsigned CalculateEdgeChecksum(std::unique_ptr> node_based_edge_list); void ContractGraph(const std::size_t number_of_edge_based_nodes, DeallocatingVector& edge_based_edge_list, @@ -72,7 +71,6 @@ class Prepare std::unique_ptr> contracted_edge_list); std::shared_ptr LoadNodeBasedGraph(std::vector &barrier_node_list, std::vector &traffic_light_list, - RestrictionMap &restriction_map, std::vector& internal_to_external_node_map); std::pair BuildEdgeExpandedGraph(std::vector &internal_to_external_node_map, diff --git a/data_structures/import_edge.cpp b/data_structures/import_edge.cpp index f41b066b1..3f2146ca3 100644 --- a/data_structures/import_edge.cpp +++ b/data_structures/import_edge.cpp @@ -47,6 +47,13 @@ bool NodeBasedEdge::operator<(const NodeBasedEdge &other) const return source < other.source; } +NodeBasedEdge::NodeBasedEdge() + : source(SPECIAL_NODEID), target(SPECIAL_NODEID), name_id(0), weight(0), forward(false), + backward(false), roundabout(false), in_tiny_cc(false), + access_restricted(false), is_split(false), travel_mode(false) +{ +} + NodeBasedEdge::NodeBasedEdge(NodeID source, NodeID target, NodeID name_id, diff --git a/data_structures/import_edge.hpp b/data_structures/import_edge.hpp index 27af59846..e48985073 100644 --- a/data_structures/import_edge.hpp +++ b/data_structures/import_edge.hpp @@ -35,6 +35,7 @@ struct NodeBasedEdge { bool operator<(const NodeBasedEdge &e) const; + NodeBasedEdge(); explicit NodeBasedEdge(NodeID source, NodeID target, NodeID name_id, @@ -58,8 +59,6 @@ struct NodeBasedEdge bool access_restricted : 1; bool is_split : 1; TravelMode travel_mode : 4; - - NodeBasedEdge() = delete; }; struct EdgeBasedEdge diff --git a/data_structures/node_based_graph.hpp b/data_structures/node_based_graph.hpp index f35367924..d33bef459 100644 --- a/data_structures/node_based_graph.hpp +++ b/data_structures/node_based_graph.hpp @@ -64,7 +64,7 @@ struct NodeBasedEdgeData backward = temp_flag; } - bool IsEqualTo(const NodeBasedEdgeData &other) const + bool IsCompatibleTo(const NodeBasedEdgeData &other) const { return (forward == other.forward) && (backward == other.backward) && (nameID == other.nameID) && (ignore_in_grid == other.ignore_in_grid) && @@ -85,6 +85,7 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vectorresult.source == node_iterator->node_id); + + // assign new node id + auto id_iter = external_to_internal_node_id_map.find(node_iterator->node_id); + BOOST_ASSERT(id_iter != external_to_internal_node_id_map.end()); + edge_iterator->result.source = id_iter->second; + edge_iterator->source_coordinate.lat = node_iterator->lat; edge_iterator->source_coordinate.lon = node_iterator->lon; ++edge_iterator; @@ -211,6 +224,10 @@ void ExtractionContainers::PrepareEdges() { if (edge_iterator->result.target < node_iterator->node_id) { + // mark edge as invalid + edge_iterator->result.source = SPECIAL_NODEID; + edge_iterator->result.target = SPECIAL_NODEID; + // FIXME we are skipping edges here: That means the data is broken! ++edge_iterator; continue; @@ -246,12 +263,113 @@ void ExtractionContainers::PrepareEdges() return -1.0; }(edge_iterator->weight_data); - edge_iterator->result.weight = std::max(1, (int)std::floor(weight + .5)); + auto& edge = edge_iterator->result; + edge.weight = std::max(1, (int)std::floor(weight + .5)); + + // assign new node id + auto id_iter = external_to_internal_node_id_map.find(node_iterator->node_id); + BOOST_ASSERT(id_iter != external_to_internal_node_id_map.end()); + edge.target = id_iter->second; + + // if source id > target id -> swap + if (edge.source > edge.target) + { + std::swap(edge.source, edge.target); + + // std::swap does not work with bit-fields + bool temp = edge.forward; + edge.forward = edge.backward; + edge.backward = temp; + } + } + else + { + // mark edge as invalid + edge_iterator->result.source = SPECIAL_NODEID; + edge_iterator->result.target = SPECIAL_NODEID; + + // FIXME we should print a warning here. } ++edge_iterator; } TIMER_STOP(compute_weights); std::cout << "ok, after " << TIMER_SEC(compute_weights) << "s" << std::endl; + + // Sort edges by start. + std::cout << "[extractor] Sorting edges by renumbered start ... " << std::flush; + TIMER_START(sort_edges_by_renumbered_start); + stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartThenTargetID(), stxxl_memory); + TIMER_STOP(sort_edges_by_renumbered_start); + std::cout << "ok, after " << TIMER_SEC(sort_edges_by_renumbered_start) << "s" << std::endl; + + BOOST_ASSERT(all_edges_list.size() > 0); + for (unsigned i = 1; i < all_edges_list.size();) + { + // only invalid edges left + if (all_edges_list[i].result.source == SPECIAL_NODEID) + { + break; + } + + unsigned start_idx = i; + NodeID source = all_edges_list[i].result.source; + NodeID target = all_edges_list[i].result.target; + + int min_forward_weight = std::numeric_limits::max(); + int min_backward_weight = std::numeric_limits::max(); + unsigned min_forward_idx = std::numeric_limits::max(); + unsigned min_backward_idx = std::numeric_limits::max(); + + while (all_edges_list[i].result.source == source && + all_edges_list[i].result.target == target) + { + if (all_edges_list[i].result.forward && all_edges_list[i].result.weight < min_forward_weight) + { + min_forward_idx = i; + } + if (all_edges_list[i].result.backward && all_edges_list[i].result.weight < min_backward_weight) + { + min_backward_idx = i; + } + + // this also increments the outer loop counter! + i++; + } + + BOOST_ASSERT(min_forward_idx == std::numeric_limits::max() || min_forward_idx < i); + BOOST_ASSERT(min_backward_idx == std::numeric_limits::max() || min_backward_idx < i); + + // reset direction for both edges + if (min_forward_idx != std::numeric_limits::max()) + { + all_edges_list[min_forward_idx].result.forward = false; + all_edges_list[min_forward_idx].result.backward = false; + } + if (min_backward_idx != std::numeric_limits::max()) + { + all_edges_list[min_backward_idx].result.forward = false; + all_edges_list[min_backward_idx].result.backward = false; + } + + // set directions that were chosen as min + // note that this needs to come after the ifs above, since + // the minimal forward and backward edge can be the same + if (min_forward_idx != std::numeric_limits::max()) + all_edges_list[min_forward_idx].result.forward = true; + if (min_backward_idx != std::numeric_limits::max()) + all_edges_list[min_backward_idx].result.backward = true; + + // invalidate all unused edges + for (unsigned j = start_idx; j < i; j++) + { + if (j == min_forward_idx || j == min_backward_idx) + { + continue; + } + all_edges_list[j].result.source = SPECIAL_NODEID; + all_edges_list[j].result.target = SPECIAL_NODEID; + } + } } void ExtractionContainers::WriteEdges(std::ofstream& file_out_stream) const @@ -266,11 +384,13 @@ void ExtractionContainers::WriteEdges(std::ofstream& file_out_stream) const for (const auto& edge : all_edges_list) { - if (edge.result.weight > 0) + if (edge.result.source == SPECIAL_NODEID || edge.result.target == SPECIAL_NODEID) { - file_out_stream.write((char*) &edge.result, sizeof(NodeBasedEdge)); - number_of_used_edges++; + continue; } + + file_out_stream.write((char*) &edge.result, sizeof(NodeBasedEdge)); + number_of_used_edges++; } TIMER_STOP(write_edges); std::cout << "ok, after " << TIMER_SEC(write_edges) << "s" << std::endl; @@ -339,6 +459,7 @@ void ExtractionContainers::WriteRestrictions(const std::string& path) const for (const auto &restriction_container : restrictions_list) { if (SPECIAL_NODEID != restriction_container.restriction.from.node && + SPECIAL_NODEID != restriction_container.restriction.via.node && SPECIAL_NODEID != restriction_container.restriction.to.node) { restrictions_out_stream.write((char *)&(restriction_container.restriction), @@ -362,7 +483,7 @@ void ExtractionContainers::PrepareRestrictions() std::cout << "ok, after " << TIMER_SEC(sort_ways) << "s" << std::endl; std::cout << "[extractor] Sorting " << restrictions_list.size() - << " restrictions. by from... " << std::flush; + << " restriction. by from... " << std::flush; TIMER_START(sort_restrictions); stxxl::sort(restrictions_list.begin(), restrictions_list.end(), CmpRestrictionContainerByFrom(), stxxl_memory); @@ -385,23 +506,42 @@ void ExtractionContainers::PrepareRestrictions() if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.from.way) { + SimpleLogger().Write(LogLevel::logDEBUG) << "Restriction references invalid way: " << restrictions_iterator->restriction.from.way; + restrictions_iterator->restriction.from.node = SPECIAL_NODEID; ++restrictions_iterator; continue; } BOOST_ASSERT(way_start_and_end_iterator->way_id == restrictions_iterator->restriction.from.way); + // we do not remap the via id yet, since we will need it for the to node as well const NodeID via_node_id = restrictions_iterator->restriction.via.node; + // check if via is actually valid, if not invalidate + auto via_id_iter = external_to_internal_node_id_map.find(via_node_id); + if(via_id_iter == external_to_internal_node_id_map.end()) + { + SimpleLogger().Write(LogLevel::logDEBUG) << "Restriction references invalid node: " << restrictions_iterator->restriction.via.node; + restrictions_iterator->restriction.via.node = SPECIAL_NODEID; + ++restrictions_iterator; + continue; + } + if (way_start_and_end_iterator->first_segment_source_id == via_node_id) { - restrictions_iterator->restriction.from.node = - way_start_and_end_iterator->first_segment_target_id; + // assign new from node id + auto id_iter = external_to_internal_node_id_map.find( + way_start_and_end_iterator->first_segment_target_id); + BOOST_ASSERT(id_iter != external_to_internal_node_id_map.end()); + restrictions_iterator->restriction.from.node = id_iter->second; } else if (way_start_and_end_iterator->last_segment_target_id == via_node_id) { - restrictions_iterator->restriction.from.node = - way_start_and_end_iterator->last_segment_source_id; + // assign new from node id + auto id_iter = external_to_internal_node_id_map.find( + way_start_and_end_iterator->last_segment_source_id); + BOOST_ASSERT(id_iter != external_to_internal_node_id_map.end()); + restrictions_iterator->restriction.from.node = id_iter->second; } ++restrictions_iterator; } @@ -428,8 +568,16 @@ void ExtractionContainers::PrepareRestrictions() ++way_start_and_end_iterator; continue; } + if (restrictions_iterator->restriction.from.node == SPECIAL_NODEID || + restrictions_iterator->restriction.via.node == SPECIAL_NODEID) + { + ++restrictions_iterator; + continue; + } if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.to.way) { + SimpleLogger().Write(LogLevel::logDEBUG) << "Restriction references invalid way: " << restrictions_iterator->restriction.to.way; + restrictions_iterator->restriction.to.way = SPECIAL_NODEID; ++restrictions_iterator; continue; } @@ -437,15 +585,24 @@ void ExtractionContainers::PrepareRestrictions() restrictions_iterator->restriction.to.way); const NodeID via_node_id = restrictions_iterator->restriction.via.node; + // assign new via node id + auto via_id_iter = external_to_internal_node_id_map.find(via_node_id); + BOOST_ASSERT(via_id_iter != external_to_internal_node_id_map.end()); + restrictions_iterator->restriction.via.node = via_id_iter->second; + if (way_start_and_end_iterator->first_segment_source_id == via_node_id) { - restrictions_iterator->restriction.to.node = - way_start_and_end_iterator->first_segment_target_id; + auto to_id_iter = external_to_internal_node_id_map.find( + way_start_and_end_iterator->first_segment_target_id); + BOOST_ASSERT(to_id_iter != external_to_internal_node_id_map.end()); + restrictions_iterator->restriction.to.node = to_id_iter->second; } else if (way_start_and_end_iterator->last_segment_target_id == via_node_id) { - restrictions_iterator->restriction.to.node = - way_start_and_end_iterator->last_segment_source_id; + auto to_id_iter = external_to_internal_node_id_map.find( + way_start_and_end_iterator->last_segment_source_id); + BOOST_ASSERT(to_id_iter != external_to_internal_node_id_map.end()); + restrictions_iterator->restriction.to.node = to_id_iter->second; } ++restrictions_iterator; } diff --git a/extractor/extraction_containers.hpp b/extractor/extraction_containers.hpp index 56109def8..444f04f02 100644 --- a/extractor/extraction_containers.hpp +++ b/extractor/extraction_containers.hpp @@ -35,6 +35,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../util/fingerprint.hpp" #include +#include /** * Uses external memory containers from stxxl to store all the data that @@ -73,6 +74,7 @@ class ExtractionContainers STXXLStringVector name_list; STXXLRestrictionsVector restrictions_list; STXXLWayIDStartEndVector way_start_end_id_list; + std::unordered_map external_to_internal_node_id_map; const FingerPrint fingerprint; ExtractionContainers(); diff --git a/extractor/internal_extractor_edge.hpp b/extractor/internal_extractor_edge.hpp index ea76c560b..15f3c0d66 100644 --- a/extractor/internal_extractor_edge.hpp +++ b/extractor/internal_extractor_edge.hpp @@ -105,6 +105,20 @@ struct InternalExtractorEdge } }; +struct CmpEdgeByStartThenTargetID +{ + using value_type = InternalExtractorEdge; + bool operator()(const InternalExtractorEdge &lhs, const InternalExtractorEdge &rhs) const + { + return (lhs.result.source < rhs.result.source) || + ((lhs.result.source == rhs.result.source) && + (lhs.result.target < rhs.result.target)); + } + + value_type max_value() { return InternalExtractorEdge::max_value(); } + value_type min_value() { return InternalExtractorEdge::min_value(); } +}; + struct CmpEdgeByStartID { using value_type = InternalExtractorEdge; diff --git a/tools/components.cpp b/tools/components.cpp index d0fcd79ab..660d9225e 100644 --- a/tools/components.cpp +++ b/tools/components.cpp @@ -76,22 +76,19 @@ void DeleteFileIfExists(const std::string &file_name) } } -void LoadRestrictions(const char* path, - std::unique_ptr> ext_to_int_id_map, - std::vector& restriction_list) +void LoadRestrictions(const char* path, std::vector& restriction_list) { std::ifstream input_stream(path, std::ios::binary); if (!input_stream.is_open()) { throw osrm::exception("Cannot open restriction file"); } - loadRestrictionsFromFile(input_stream, *ext_to_int_id_map, restriction_list); + loadRestrictionsFromFile(input_stream, restriction_list); } std::size_t LoadGraph(const char* path, std::vector& coordinate_list, std::vector& barrier_node_list, - std::unordered_map& ext_to_int_id_map, std::vector& graph_edge_list) { std::ifstream input_stream(path, std::ifstream::in | std::ifstream::binary); @@ -106,10 +103,9 @@ std::size_t LoadGraph(const char* path, auto number_of_nodes = loadNodesFromFile(input_stream, barrier_node_list, traffic_light_node_list, - coordinate_list, - ext_to_int_id_map); + coordinate_list); - loadEdgesFromFile(input_stream, ext_to_int_id_map, edge_list); + loadEdgesFromFile(input_stream, edge_list); traffic_light_node_list.clear(); traffic_light_node_list.shrink_to_fit(); @@ -142,7 +138,6 @@ int main(int argc, char *argv[]) std::vector coordinate_list; std::vector restriction_list; std::vector barrier_node_list; - auto ext_to_int_id_map = osrm::make_unique>(); LogPolicy::GetInstance().Unmute(); try @@ -158,8 +153,8 @@ int main(int argc, char *argv[]) SimpleLogger().Write() << "Using restrictions from file: " << argv[2]; std::vector graph_edge_list; - auto number_of_nodes = LoadGraph(argv[1], coordinate_list, barrier_node_list, *ext_to_int_id_map, graph_edge_list); - LoadRestrictions(argv[2], std::move(ext_to_int_id_map), restriction_list); + auto number_of_nodes = LoadGraph(argv[1], coordinate_list, barrier_node_list, graph_edge_list); + LoadRestrictions(argv[2], restriction_list); tbb::parallel_sort(graph_edge_list.begin(), graph_edge_list.end()); const auto graph = std::make_shared(number_of_nodes, graph_edge_list); diff --git a/util/graph_loader.hpp b/util/graph_loader.hpp index 0736775b9..8d7cdd0fb 100644 --- a/util/graph_loader.hpp +++ b/util/graph_loader.hpp @@ -58,7 +58,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * we need to renumber it to the new internal id. */ unsigned loadRestrictionsFromFile(std::istream& input_stream, - const std::unordered_map& ext_to_int_id_map, std::vector& restriction_list) { const FingerPrint fingerprint_orig; @@ -79,38 +78,6 @@ unsigned loadRestrictionsFromFile(std::istream& input_stream, number_of_usable_restrictions * sizeof(TurnRestriction)); } - // renumber ids referenced in restrictions - for (TurnRestriction ¤t_restriction : restriction_list) - { - auto internal_id_iter = ext_to_int_id_map.find(current_restriction.from.node); - if (internal_id_iter == ext_to_int_id_map.end()) - { - SimpleLogger().Write(logDEBUG) << "Unmapped from node " << current_restriction.from.node - << " of restriction"; - continue; - } - current_restriction.from.node = internal_id_iter->second; - - internal_id_iter = ext_to_int_id_map.find(current_restriction.via.node); - if (internal_id_iter == ext_to_int_id_map.end()) - { - SimpleLogger().Write(logDEBUG) << "Unmapped via node " << current_restriction.via.node - << " of restriction"; - continue; - } - - current_restriction.via.node = internal_id_iter->second; - - internal_id_iter = ext_to_int_id_map.find(current_restriction.to.node); - if (internal_id_iter == ext_to_int_id_map.end()) - { - SimpleLogger().Write(logDEBUG) << "Unmapped to node " << current_restriction.to.node - << " of restriction"; - continue; - } - current_restriction.to.node = internal_id_iter->second; - } - return number_of_usable_restrictions; } @@ -119,14 +86,12 @@ unsigned loadRestrictionsFromFile(std::istream& input_stream, * Reads the beginning of an .osrm file and produces: * - list of barrier nodes * - list of traffic lights - * - index to original node id map - * - original node id to index map + * - nodes indexed by their internal (non-osm) id */ NodeID loadNodesFromFile(std::istream &input_stream, std::vector &barrier_node_list, std::vector &traffic_light_node_list, - std::vector &int_to_ext_node_id_map, - std::unordered_map &ext_to_int_id_map) + std::vector &node_array) { const FingerPrint fingerprint_orig; FingerPrint fingerprint_loaded; @@ -146,9 +111,7 @@ NodeID loadNodesFromFile(std::istream &input_stream, for (NodeID i = 0; i < n; ++i) { input_stream.read(reinterpret_cast(¤t_node), sizeof(ExternalMemoryNode)); - int_to_ext_node_id_map.emplace_back(current_node.lat, current_node.lon, - current_node.node_id); - ext_to_int_id_map.emplace(current_node.node_id, i); + node_array.emplace_back(current_node.lat, current_node.lon, current_node.node_id); if (current_node.barrier) { barrier_node_list.emplace_back(i); @@ -167,122 +130,27 @@ NodeID loadNodesFromFile(std::istream &input_stream, } /** - * Reads a .osrm file and produces the edges. Edges reference nodes in the old - * OSM based format, we need to renumber it here. + * Reads a .osrm file and produces the edges. */ NodeID loadEdgesFromFile(std::istream &input_stream, - const std::unordered_map &ext_to_int_id_map, std::vector &edge_list) { EdgeID m; input_stream.read(reinterpret_cast(&m), sizeof(unsigned)); - edge_list.reserve(m); + edge_list.resize(m); SimpleLogger().Write() << " and " << m << " edges "; - NodeBasedEdge edge(0, 0, 0, 0, false, false, false, false, false, TRAVEL_MODE_INACCESSIBLE, false); - for (EdgeID i = 0; i < m; ++i) + input_stream.read((char *) edge_list.data(), m * sizeof(NodeBasedEdge)); + +#ifndef NDEBUG + for (const auto& edge : edge_list) { - input_stream.read(reinterpret_cast(&edge), sizeof(NodeBasedEdge)); BOOST_ASSERT_MSG(edge.weight > 0, "loaded null weight"); - BOOST_ASSERT_MSG(edge.forward || edge.backward, "loaded null weight"); - BOOST_ASSERT_MSG(edge.travel_mode != TRAVEL_MODE_INACCESSIBLE, "loaded null weight"); - - // translate the external NodeIDs to internal IDs - auto internal_id_iter = ext_to_int_id_map.find(edge.source); - if (ext_to_int_id_map.find(edge.source) == ext_to_int_id_map.end()) - { -#ifndef NDEBUG - SimpleLogger().Write(logWARNING) << " unresolved source NodeID: " << edge.source; -#endif - continue; - } - edge.source = internal_id_iter->second; - internal_id_iter = ext_to_int_id_map.find(edge.target); - if (ext_to_int_id_map.find(edge.target) == ext_to_int_id_map.end()) - { -#ifndef NDEBUG - SimpleLogger().Write(logWARNING) << "unresolved target NodeID : " << edge.target; -#endif - continue; - } - edge.target = internal_id_iter->second; - BOOST_ASSERT_MSG(edge.source != SPECIAL_NODEID && edge.target != SPECIAL_NODEID, - "nonexisting source or target"); - - if (edge.source > edge.target) - { - std::swap(edge.source, edge.target); - - // std::swap does not work with bit-fields - bool temp = edge.forward; - edge.forward = edge.backward; - edge.backward = temp; - } - - edge_list.push_back(edge); + BOOST_ASSERT_MSG(edge.forward || edge.backward, "loaded invalid direction"); + BOOST_ASSERT_MSG(edge.travel_mode != TRAVEL_MODE_INACCESSIBLE, "loaded non-accessible"); } +#endif - tbb::parallel_sort(edge_list.begin(), edge_list.end()); - - // Removes multi-edges between nodes - for (unsigned i = 1; i < edge_list.size(); ++i) - { - if ((edge_list[i - 1].target == edge_list[i].target) && - (edge_list[i - 1].source == edge_list[i].source)) - { - const bool edge_flags_equivalent = (edge_list[i - 1].forward == edge_list[i].forward) && - (edge_list[i - 1].backward == edge_list[i].backward); - const bool edge_flags_are_superset1 = - (edge_list[i - 1].forward && edge_list[i - 1].backward) && - (edge_list[i].forward != edge_list[i].backward); - const bool edge_flags_are_superset_2 = - (edge_list[i].forward && edge_list[i].backward) && - (edge_list[i - 1].forward != edge_list[i - 1].backward); - - if (edge_flags_equivalent) - { - edge_list[i].weight = std::min(edge_list[i - 1].weight, edge_list[i].weight); - edge_list[i - 1].source = SPECIAL_NODEID; - } - else if (edge_flags_are_superset1) - { - if (edge_list[i - 1].weight <= edge_list[i].weight) - { - // edge i-1 is smaller and goes in both directions. Throw away the other edge - edge_list[i].source = SPECIAL_NODEID; - } - else - { - // edge i-1 is open in both directions, but edge i is smaller in one direction. - // Close edge i-1 in this direction - edge_list[i - 1].forward = !edge_list[i].forward; - edge_list[i - 1].backward = !edge_list[i].backward; - } - } - else if (edge_flags_are_superset_2) - { - if (edge_list[i - 1].weight <= edge_list[i].weight) - { - // edge i-1 is smaller for one direction. edge i is open in both. close edge i - // in the other direction - edge_list[i].forward = !edge_list[i - 1].forward; - edge_list[i].backward = !edge_list[i - 1].backward; - } - else - { - // edge i is smaller and goes in both direction. Throw away edge i-1 - edge_list[i - 1].source = SPECIAL_NODEID; - } - } - } - } - const auto new_end_iter = - std::remove_if(edge_list.begin(), edge_list.end(), [](const NodeBasedEdge &edge) - { - return edge.source == SPECIAL_NODEID || edge.target == SPECIAL_NODEID; - }); - edge_list.erase(new_end_iter, edge_list.end()); // remove excess candidates. - edge_list.shrink_to_fit(); SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges"; return m; From b7c8fcd0625595dba7373b1af20d7280a963c37a Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 15 May 2015 15:18:34 +0200 Subject: [PATCH 57/87] Also print edge source/target for suspicious edge weights --- contractor/contractor.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contractor/contractor.hpp b/contractor/contractor.hpp index e4a278c71..4468f5e49 100644 --- a/contractor/contractor.hpp +++ b/contractor/contractor.hpp @@ -171,7 +171,8 @@ class Contractor { SimpleLogger().Write(logWARNING) << "Edge weight large -> " - << static_cast(std::max(diter->weight, 1)); + << static_cast(std::max(diter->weight, 1)) << " : " + << static_cast(diter->source) << " -> " << static_cast(diter->target); } #endif edges.emplace_back(diter->source, diter->target, From aba3ec692f7565f2bc0c51f4fa3e4816742a73c3 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 21 May 2015 15:54:07 +0200 Subject: [PATCH 58/87] Verify graph before compression --- contractor/edge_based_graph_factory.cpp | 77 +++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index 6f414ecd9..a65e8c61f 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -40,6 +40,37 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include + +bool checkInvariant(const NodeBasedDynamicGraph& graph, const NodeID source, const EdgeID edge) +{ + const auto& data = graph.GetEdgeData(edge); + if (!data.forward) + { + auto target = graph.GetTarget(edge); + if (target == SPECIAL_NODEID) + { + SimpleLogger().Write(logWARNING) << "Invalid target"; + return false; + } + + auto rev_edge = graph.FindEdge(target, source); + if (rev_edge == SPECIAL_EDGEID) + { + SimpleLogger().Write(logWARNING) << "Edge not found"; + return false; + } + + const auto& rev_data = graph.GetEdgeData(rev_edge); + if (!rev_data.forward) + { + SimpleLogger().Write(logWARNING) << "Reverse edge is not forward"; + return false; + } + } + + return true; +} + EdgeBasedGraphFactory::EdgeBasedGraphFactory(std::shared_ptr node_based_graph, std::shared_ptr restriction_map, std::unique_ptr> barrier_node_list, @@ -234,6 +265,31 @@ void EdgeBasedGraphFactory::Run(const std::string &original_edge_data_filename, lua_State *lua_state) { +#ifndef NDEBUG + SimpleLogger().Write() << "Verifying the graph structure before compression:"; + for (const auto current_node : osrm::irange(0u, m_node_based_graph->GetNumberOfNodes())) + { + for (const auto current_edge : m_node_based_graph->GetAdjacentEdgeRange(current_node)) + { + EdgeData &edge_data = m_node_based_graph->GetEdgeData(current_edge); + if (!edge_data.forward) + { + + auto target = m_node_based_graph->GetTarget(current_edge); + BOOST_ASSERT(target != SPECIAL_NODEID); + + auto rev_edge = m_node_based_graph->FindEdge(target, current_node); + BOOST_ASSERT(rev_edge != SPECIAL_EDGEID); + + const auto& rev_data = m_node_based_graph->GetEdgeData(rev_edge); + BOOST_ASSERT(rev_data.forward); + continue; + } + } + } + SimpleLogger().Write() << " -> graph is ok."; +#endif + TIMER_START(geometry); CompressGeometry(); TIMER_STOP(geometry); @@ -410,6 +466,27 @@ void EdgeBasedGraphFactory::CompressGeometry() reverse_weight2 + (has_node_penalty ? speed_profile.traffic_signal_penalty : 0)); ++removed_node_count; +#ifndef NDEBUG + if (!checkInvariant(*m_node_based_graph, node_u, forward_e1)) + { + SimpleLogger().Write(logWARNING) << "Contracting " << node_u << " " << node_v << " " << node_w; + SimpleLogger().Write(logWARNING) << " coordinates " + << "(" << m_node_info_list[node_u].lat << ", " << m_node_info_list[node_u].lon << ") " + << "(" << m_node_info_list[node_v].lat << ", " << m_node_info_list[node_v].lon << ") " + << "(" << m_node_info_list[node_w].lat << ", " << m_node_info_list[node_w].lon << ") "; + BOOST_ASSERT_MSG(false, "Graph invariant is not fulfilled."); + } + if (!checkInvariant(*m_node_based_graph, node_w, reverse_e1)) + { + SimpleLogger().Write(logWARNING) << "Contracting " << node_u << " " << node_v << " " << node_w; + SimpleLogger().Write(logWARNING) << " coordinates " + << "(" << m_node_info_list[node_u].lat << ", " << m_node_info_list[node_u].lon << ") " + << "(" << m_node_info_list[node_v].lat << ", " << m_node_info_list[node_v].lon << ") " + << "(" << m_node_info_list[node_w].lat << ", " << m_node_info_list[node_w].lon << ") "; + BOOST_ASSERT_MSG(false, "Graph invariant is not fulfilled."); + } +#endif + } } SimpleLogger().Write() << "removed " << removed_node_count << " nodes"; From 2777d53a12ca6753880f1dc025d8a7cc34377bcb Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sun, 24 May 2015 17:25:38 +0200 Subject: [PATCH 59/87] Direct edges in contractor correctly and add better graph validation. --- contractor/edge_based_graph_factory.cpp | 92 ++--------- data_structures/node_based_graph.hpp | 200 ++++++++++++++---------- extractor/extraction_containers.cpp | 155 ++++++++++-------- util/graph_loader.hpp | 18 ++- 4 files changed, 232 insertions(+), 233 deletions(-) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index a65e8c61f..38a90ee99 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -41,35 +41,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include -bool checkInvariant(const NodeBasedDynamicGraph& graph, const NodeID source, const EdgeID edge) -{ - const auto& data = graph.GetEdgeData(edge); - if (!data.forward) - { - auto target = graph.GetTarget(edge); - if (target == SPECIAL_NODEID) - { - SimpleLogger().Write(logWARNING) << "Invalid target"; - return false; - } - - auto rev_edge = graph.FindEdge(target, source); - if (rev_edge == SPECIAL_EDGEID) - { - SimpleLogger().Write(logWARNING) << "Edge not found"; - return false; - } - - const auto& rev_data = graph.GetEdgeData(rev_edge); - if (!rev_data.forward) - { - SimpleLogger().Write(logWARNING) << "Reverse edge is not forward"; - return false; - } - } - - return true; -} EdgeBasedGraphFactory::EdgeBasedGraphFactory(std::shared_ptr node_based_graph, std::shared_ptr restriction_map, @@ -264,32 +235,6 @@ void EdgeBasedGraphFactory::Run(const std::string &original_edge_data_filename, const std::string &geometry_filename, lua_State *lua_state) { - -#ifndef NDEBUG - SimpleLogger().Write() << "Verifying the graph structure before compression:"; - for (const auto current_node : osrm::irange(0u, m_node_based_graph->GetNumberOfNodes())) - { - for (const auto current_edge : m_node_based_graph->GetAdjacentEdgeRange(current_node)) - { - EdgeData &edge_data = m_node_based_graph->GetEdgeData(current_edge); - if (!edge_data.forward) - { - - auto target = m_node_based_graph->GetTarget(current_edge); - BOOST_ASSERT(target != SPECIAL_NODEID); - - auto rev_edge = m_node_based_graph->FindEdge(target, current_node); - BOOST_ASSERT(rev_edge != SPECIAL_EDGEID); - - const auto& rev_data = m_node_based_graph->GetEdgeData(rev_edge); - BOOST_ASSERT(rev_data.forward); - continue; - } - } - } - SimpleLogger().Write() << " -> graph is ok."; -#endif - TIMER_START(geometry); CompressGeometry(); TIMER_STOP(geometry); @@ -466,28 +411,20 @@ void EdgeBasedGraphFactory::CompressGeometry() reverse_weight2 + (has_node_penalty ? speed_profile.traffic_signal_penalty : 0)); ++removed_node_count; -#ifndef NDEBUG - if (!checkInvariant(*m_node_based_graph, node_u, forward_e1)) - { - SimpleLogger().Write(logWARNING) << "Contracting " << node_u << " " << node_v << " " << node_w; - SimpleLogger().Write(logWARNING) << " coordinates " - << "(" << m_node_info_list[node_u].lat << ", " << m_node_info_list[node_u].lon << ") " - << "(" << m_node_info_list[node_v].lat << ", " << m_node_info_list[node_v].lon << ") " - << "(" << m_node_info_list[node_w].lat << ", " << m_node_info_list[node_w].lon << ") "; - BOOST_ASSERT_MSG(false, "Graph invariant is not fulfilled."); - } - if (!checkInvariant(*m_node_based_graph, node_w, reverse_e1)) - { - SimpleLogger().Write(logWARNING) << "Contracting " << node_u << " " << node_v << " " << node_w; - SimpleLogger().Write(logWARNING) << " coordinates " - << "(" << m_node_info_list[node_u].lat << ", " << m_node_info_list[node_u].lon << ") " - << "(" << m_node_info_list[node_v].lat << ", " << m_node_info_list[node_v].lon << ") " - << "(" << m_node_info_list[node_w].lat << ", " << m_node_info_list[node_w].lon << ") "; - BOOST_ASSERT_MSG(false, "Graph invariant is not fulfilled."); - } -#endif } + +#ifndef NDEBUG + if (!validateNeighborHood(*m_node_based_graph, node_v)) + { + SimpleLogger().Write(logWARNING) << "Contracting " << node_u << " " << node_v << " " << node_w; + SimpleLogger().Write(logWARNING) << " coordinates " + << "(" << (m_node_info_list[node_u].lat/COORDINATE_PRECISION) << ", " << (m_node_info_list[node_u].lon/COORDINATE_PRECISION) << ") " + << "(" << (m_node_info_list[node_v].lat/COORDINATE_PRECISION) << ", " << (m_node_info_list[node_v].lon/COORDINATE_PRECISION) << ") " + << "(" << (m_node_info_list[node_w].lat/COORDINATE_PRECISION) << ", " << (m_node_info_list[node_w].lon/COORDINATE_PRECISION) << ") "; + BOOST_ASSERT_MSG(false, "Graph invariant is not fulfilled."); + } +#endif } SimpleLogger().Write() << "removed " << removed_node_count << " nodes"; m_geometry_compressor.PrintStatistics(); @@ -515,14 +452,15 @@ void EdgeBasedGraphFactory::CompressGeometry() */ void EdgeBasedGraphFactory::RenumberEdges() { - // renumber edge based node IDs + // renumber edge based node of outgoing edges unsigned numbered_edges_count = 0; for (const auto current_node : osrm::irange(0u, m_node_based_graph->GetNumberOfNodes())) { for (const auto current_edge : m_node_based_graph->GetAdjacentEdgeRange(current_node)) { EdgeData &edge_data = m_node_based_graph->GetEdgeData(current_edge); - // FIXME when does that happen? why can we skip here? + + // this edge is an incoming edge if (!edge_data.forward) { continue; diff --git a/data_structures/node_based_graph.hpp b/data_structures/node_based_graph.hpp index d33bef459..b69053a97 100644 --- a/data_structures/node_based_graph.hpp +++ b/data_structures/node_based_graph.hpp @@ -74,7 +74,92 @@ struct NodeBasedEdgeData using NodeBasedDynamicGraph = DynamicGraph; +inline bool validateNeighborHood(const NodeBasedDynamicGraph& graph, const NodeID source) +{ + for (auto edge = graph.BeginEdges(source); edge < graph.EndEdges(source); ++edge) + { + const auto& data = graph.GetEdgeData(edge); + if (!data.forward && !data.backward) + { + SimpleLogger().Write(logWARNING) << "Invalid edge directions"; + return false; + } + + auto target = graph.GetTarget(edge); + if (target == SPECIAL_NODEID) + { + SimpleLogger().Write(logWARNING) << "Invalid edge target"; + return false; + } + + bool found_reverse = false; + for (auto rev_edge = graph.BeginEdges(target); rev_edge < graph.EndEdges(target); ++rev_edge) + { + auto rev_target = graph.GetTarget(rev_edge); + if (rev_target == SPECIAL_NODEID) + { + SimpleLogger().Write(logWARNING) << "Invalid reverse edge target"; + return false; + } + + if (rev_target != source) + { + continue; + } + + if (found_reverse) + { + SimpleLogger().Write(logWARNING) << "Found more than one reverse edge"; + return false; + } + + const auto& rev_data = graph.GetEdgeData(rev_edge); + + // edge is incoming, this must be an outgoing edge + if (data.backward && !rev_data.forward) + { + SimpleLogger().Write(logWARNING) << "Found no outgoing edge to an incoming edge!"; + return false; + } + + // edge is bi-directional, reverse must be as well + if (data.forward && data.backward && (!rev_data.forward || !rev_data.backward)) + { + SimpleLogger().Write(logWARNING) << "Found bi-directional edge that is not bi-directional to both ends"; + return false; + } + + found_reverse = true; + + } + + if (!found_reverse) + { + SimpleLogger().Write(logWARNING) << "Could not find reverse edge"; + return false; + } + } + + return true; +} + +// This function checks if the overal graph is undirected (has an edge in each direction). +inline bool validateNodeBasedGraph(const NodeBasedDynamicGraph& graph) +{ + for (auto source = 0u; source < graph.GetNumberOfNodes(); ++source) + { + if (!validateNeighborHood(graph, source)) + { + return false; + } + } + + return true; +} + // Factory method to create NodeBasedDynamicGraph from NodeBasedEdges +// The since DynamicGraph expects directed edges, we need to insert +// two edges for undirected edges. inline std::shared_ptr NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector &input_edge_list) { @@ -83,9 +168,27 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector edges_list; NodeBasedDynamicGraph::InputEdge edge; + + // Since DynamicGraph assumes directed edges we have to make sure we transformed + // the compressed edge format into single directed edges. We do this to make sure + // every node also knows its incoming edges, not only its outgoing edges and use the backward=true + // flag to indicate which is which. + // + // We do the transformation in the following way: + // + // if the edge (a, b) is split: + // 1. this edge must be in only one direction, so its a --> b + // 2. there must be another directed edge b --> a somewhere in the data + // if the edge (a, b) is not split: + // 1. this edge be on of a --> b od a <-> b + // (a <-- b gets reducted to b --> a) + // 2. a --> b will be transformed to a --> b and b <-- a + // 3. a <-> b will be transformed to a <-> b and b <-> a (I think a --> b and b <-- a would work as well though) for (const NodeBasedEdge &import_edge : input_edge_list) { - BOOST_ASSERT(import_edge.forward || import_edge.backward); + // edges that are not forward get converted by flipping the end points + BOOST_ASSERT(import_edge.forward); + if (import_edge.forward) { edge.source = import_edge.source; @@ -93,20 +196,10 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector(import_edge.weight), 1); + edge.data.distance = static_cast(import_edge.weight); BOOST_ASSERT(edge.data.distance > 0); edge.data.shortcut = false; edge.data.roundabout = import_edge.roundabout; @@ -126,82 +219,17 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector::max(); - // remove parallel edges and set current distance values - while (i < edges_list.size() && edges_list[i].source == source && - edges_list[i].target == target) - { - if (edges_list[i].data.forward) - { - forward_edge.data.distance = - std::min(edges_list[i].data.distance, forward_edge.data.distance); - } - if (edges_list[i].data.backward) - { - reverse_edge.data.distance = - std::min(edges_list[i].data.distance, reverse_edge.data.distance); - } - ++i; - } - // merge edges (s,t) and (t,s) into bidirectional edge - if (forward_edge.data.distance == reverse_edge.data.distance) - { - if (static_cast(forward_edge.data.distance) != std::numeric_limits::max()) - { - forward_edge.data.backward = true; - BOOST_ASSERT(edge_count < i); - edges_list[edge_count++] = forward_edge; - } - } - else - { // insert seperate edges - // this case can only happen if we merged a bi-directional edge with a directional - // edge above, this incrementing i and making it safe to overwrite the next element - // as well - if (static_cast(forward_edge.data.distance) != std::numeric_limits::max()) - { - BOOST_ASSERT(edge_count < i); - edges_list[edge_count++] = forward_edge; - } - if (static_cast(reverse_edge.data.distance) != std::numeric_limits::max()) - { - BOOST_ASSERT(edge_count < i); - edges_list[edge_count++] = reverse_edge; - } - } - } - edges_list.resize(edge_count); - SimpleLogger().Write() << "merged " << edges_list.size() - edge_count << " edges out of " - << edges_list.size(); - - return std::make_shared( + auto graph = std::make_shared( static_cast(number_of_nodes), edges_list); + + +#ifndef NDEBUG + BOOST_ASSERT(validateNodeBasedGraph(*graph)); +#endif + + return graph; } #endif // NODE_BASED_GRAPH_HPP diff --git a/extractor/extraction_containers.cpp b/extractor/extraction_containers.cpp index d4b7785c8..38ca029c6 100644 --- a/extractor/extraction_containers.cpp +++ b/extractor/extraction_containers.cpp @@ -184,6 +184,7 @@ void ExtractionContainers::PrepareEdges() { if (edge_iterator->result.source < node_iterator->node_id) { + edge_iterator->result.source = SPECIAL_NODEID; ++edge_iterator; continue; } @@ -193,6 +194,15 @@ void ExtractionContainers::PrepareEdges() continue; } + // remove loops + if (edge_iterator->result.source == edge_iterator->result.target) + { + edge_iterator->result.source = SPECIAL_NODEID; + edge_iterator->result.target = SPECIAL_NODEID; + ++edge_iterator; + continue; + } + BOOST_ASSERT(edge_iterator->result.source == node_iterator->node_id); // assign new node id @@ -222,13 +232,16 @@ void ExtractionContainers::PrepareEdges() edge_iterator = all_edges_list.begin(); while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end()) { + // skip all invalid edges + if (edge_iterator->result.source == SPECIAL_NODEID) + { + ++edge_iterator; + continue; + } + if (edge_iterator->result.target < node_iterator->node_id) { - // mark edge as invalid - edge_iterator->result.source = SPECIAL_NODEID; edge_iterator->result.target = SPECIAL_NODEID; - - // FIXME we are skipping edges here: That means the data is broken! ++edge_iterator; continue; } @@ -237,58 +250,50 @@ void ExtractionContainers::PrepareEdges() ++node_iterator; continue; } + BOOST_ASSERT(edge_iterator->result.target == node_iterator->node_id); - if (edge_iterator->source_coordinate.lat != std::numeric_limits::min() && - edge_iterator->source_coordinate.lon != std::numeric_limits::min()) - { - BOOST_ASSERT(edge_iterator->weight_data.speed != -1); + BOOST_ASSERT(edge_iterator->weight_data.speed >= 0); + BOOST_ASSERT(edge_iterator->source_coordinate.lat != std::numeric_limits::min()); + BOOST_ASSERT(edge_iterator->source_coordinate.lon != std::numeric_limits::min()); - const double distance = coordinate_calculation::euclidean_distance( - edge_iterator->source_coordinate.lat, edge_iterator->source_coordinate.lon, - node_iterator->lat, node_iterator->lon); + const double distance = coordinate_calculation::euclidean_distance( + edge_iterator->source_coordinate.lat, edge_iterator->source_coordinate.lon, + node_iterator->lat, node_iterator->lon); - const double weight = [distance](const InternalExtractorEdge::WeightData& data) { - switch (data.type) - { - case InternalExtractorEdge::WeightType::EDGE_DURATION: - case InternalExtractorEdge::WeightType::WAY_DURATION: - return data.duration * 10.; - break; - case InternalExtractorEdge::WeightType::SPEED: - return (distance * 10.) / (data.speed / 3.6); - break; - default: - osrm::exception("invalid weight type"); - } - return -1.0; - }(edge_iterator->weight_data); - - auto& edge = edge_iterator->result; - edge.weight = std::max(1, (int)std::floor(weight + .5)); - - // assign new node id - auto id_iter = external_to_internal_node_id_map.find(node_iterator->node_id); - BOOST_ASSERT(id_iter != external_to_internal_node_id_map.end()); - edge.target = id_iter->second; - - // if source id > target id -> swap - if (edge.source > edge.target) + const double weight = [distance](const InternalExtractorEdge::WeightData& data) { + switch (data.type) { - std::swap(edge.source, edge.target); - - // std::swap does not work with bit-fields - bool temp = edge.forward; - edge.forward = edge.backward; - edge.backward = temp; + case InternalExtractorEdge::WeightType::EDGE_DURATION: + case InternalExtractorEdge::WeightType::WAY_DURATION: + return data.duration * 10.; + break; + case InternalExtractorEdge::WeightType::SPEED: + return (distance * 10.) / (data.speed / 3.6); + break; + case InternalExtractorEdge::WeightType::INVALID: + osrm::exception("invalid weight type"); } - } - else - { - // mark edge as invalid - edge_iterator->result.source = SPECIAL_NODEID; - edge_iterator->result.target = SPECIAL_NODEID; + return -1.0; + }(edge_iterator->weight_data); - // FIXME we should print a warning here. + auto& edge = edge_iterator->result; + edge.weight = std::max(1, static_cast(std::floor(weight + .5))); + + // assign new node id + auto id_iter = external_to_internal_node_id_map.find(node_iterator->node_id); + BOOST_ASSERT(id_iter != external_to_internal_node_id_map.end()); + edge.target = id_iter->second; + + // orient edges consistently: source id < target id + // important for multi-edge removal + if (edge.source > edge.target) + { + std::swap(edge.source, edge.target); + + // std::swap does not work with bit-fields + bool temp = edge.forward; + edge.forward = edge.backward; + edge.backward = temp; } ++edge_iterator; } @@ -303,13 +308,18 @@ void ExtractionContainers::PrepareEdges() std::cout << "ok, after " << TIMER_SEC(sort_edges_by_renumbered_start) << "s" << std::endl; BOOST_ASSERT(all_edges_list.size() > 0); - for (unsigned i = 1; i < all_edges_list.size();) + for (unsigned i = 0; i < all_edges_list.size();) { // only invalid edges left if (all_edges_list[i].result.source == SPECIAL_NODEID) { break; } + // skip invalid edges + if (all_edges_list[i].result.target == SPECIAL_NODEID) + { + continue; + } unsigned start_idx = i; NodeID source = all_edges_list[i].result.source; @@ -320,6 +330,7 @@ void ExtractionContainers::PrepareEdges() unsigned min_forward_idx = std::numeric_limits::max(); unsigned min_backward_idx = std::numeric_limits::max(); + // find minimal edge in both directions while (all_edges_list[i].result.source == source && all_edges_list[i].result.target == target) { @@ -338,26 +349,34 @@ void ExtractionContainers::PrepareEdges() BOOST_ASSERT(min_forward_idx == std::numeric_limits::max() || min_forward_idx < i); BOOST_ASSERT(min_backward_idx == std::numeric_limits::max() || min_backward_idx < i); + BOOST_ASSERT(min_backward_idx != std::numeric_limits::max() || + min_forward_idx != std::numeric_limits::max()); - // reset direction for both edges - if (min_forward_idx != std::numeric_limits::max()) + if (min_backward_idx == min_forward_idx) { - all_edges_list[min_forward_idx].result.forward = false; - all_edges_list[min_forward_idx].result.backward = false; - } - if (min_backward_idx != std::numeric_limits::max()) - { - all_edges_list[min_backward_idx].result.forward = false; - all_edges_list[min_backward_idx].result.backward = false; - } - - // set directions that were chosen as min - // note that this needs to come after the ifs above, since - // the minimal forward and backward edge can be the same - if (min_forward_idx != std::numeric_limits::max()) + all_edges_list[min_forward_idx].result.is_split = false; all_edges_list[min_forward_idx].result.forward = true; - if (min_backward_idx != std::numeric_limits::max()) - all_edges_list[min_backward_idx].result.backward = true; + all_edges_list[min_forward_idx].result.backward = true; + } + else + { + bool has_forward = min_forward_idx != std::numeric_limits::max(); + bool has_backward = min_backward_idx != std::numeric_limits::max(); + if (has_forward) + { + all_edges_list[min_forward_idx].result.forward = true; + all_edges_list[min_forward_idx].result.backward = false; + all_edges_list[min_forward_idx].result.is_split = has_backward; + } + if (has_backward) + { + std::swap(all_edges_list[min_backward_idx].result.source, + all_edges_list[min_backward_idx].result.target); + all_edges_list[min_backward_idx].result.forward = true; + all_edges_list[min_backward_idx].result.backward = false; + all_edges_list[min_backward_idx].result.is_split = has_forward; + } + } // invalidate all unused edges for (unsigned j = start_idx; j < i; j++) diff --git a/util/graph_loader.hpp b/util/graph_loader.hpp index 8d7cdd0fb..2243538f3 100644 --- a/util/graph_loader.hpp +++ b/util/graph_loader.hpp @@ -142,12 +142,26 @@ NodeID loadEdgesFromFile(std::istream &input_stream, input_stream.read((char *) edge_list.data(), m * sizeof(NodeBasedEdge)); + BOOST_ASSERT(edge_list.size() > 0); + #ifndef NDEBUG - for (const auto& edge : edge_list) + SimpleLogger().Write() << "Validating loaded edges..."; + std::sort(edge_list.begin(), edge_list.end(), + [](const NodeBasedEdge& lhs, const NodeBasedEdge& rhs) + { + return (lhs.source < rhs.source) || (lhs.source == rhs.source && lhs.target < rhs.target); + }); + for (auto i = 1u; i < edge_list.size(); ++i) { + const auto& edge = edge_list[i]; + const auto& prev_edge = edge_list[i-1]; + BOOST_ASSERT_MSG(edge.weight > 0, "loaded null weight"); - BOOST_ASSERT_MSG(edge.forward || edge.backward, "loaded invalid direction"); + BOOST_ASSERT_MSG(edge.forward, "edge must be oriented in forward direction"); BOOST_ASSERT_MSG(edge.travel_mode != TRAVEL_MODE_INACCESSIBLE, "loaded non-accessible"); + + BOOST_ASSERT_MSG(edge.source != edge.target, "loaded edges contain a loop"); + BOOST_ASSERT_MSG(edge.source != prev_edge.source || edge.target != prev_edge.target, "loaded edges contain a multi edge"); } #endif From f12f6a56bab475a840556b08b4365e96a0d675ef Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sun, 24 May 2015 18:44:38 +0200 Subject: [PATCH 60/87] Fix debug message --- extractor/extraction_containers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/extraction_containers.cpp b/extractor/extraction_containers.cpp index 38ca029c6..7097993f8 100644 --- a/extractor/extraction_containers.cpp +++ b/extractor/extraction_containers.cpp @@ -419,7 +419,7 @@ void ExtractionContainers::WriteEdges(std::ofstream& file_out_stream) const file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned)); std::cout << "ok" << std::endl; - SimpleLogger().Write() << "Processed " << number_of_used_edges << " nodes"; + SimpleLogger().Write() << "Processed " << number_of_used_edges << " edges"; } void ExtractionContainers::WriteNodes(std::ofstream& file_out_stream) const From b1ef4cfee983f44bf67246215f6cc3a9131c3a79 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 29 May 2015 01:26:06 +0200 Subject: [PATCH 61/87] Remove debugging code --- contractor/edge_based_graph_factory.cpp | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index 38a90ee99..203c6b632 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -40,8 +40,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include - - EdgeBasedGraphFactory::EdgeBasedGraphFactory(std::shared_ptr node_based_graph, std::shared_ptr restriction_map, std::unique_ptr> barrier_node_list, @@ -413,18 +411,6 @@ void EdgeBasedGraphFactory::CompressGeometry() } - -#ifndef NDEBUG - if (!validateNeighborHood(*m_node_based_graph, node_v)) - { - SimpleLogger().Write(logWARNING) << "Contracting " << node_u << " " << node_v << " " << node_w; - SimpleLogger().Write(logWARNING) << " coordinates " - << "(" << (m_node_info_list[node_u].lat/COORDINATE_PRECISION) << ", " << (m_node_info_list[node_u].lon/COORDINATE_PRECISION) << ") " - << "(" << (m_node_info_list[node_v].lat/COORDINATE_PRECISION) << ", " << (m_node_info_list[node_v].lon/COORDINATE_PRECISION) << ") " - << "(" << (m_node_info_list[node_w].lat/COORDINATE_PRECISION) << ", " << (m_node_info_list[node_w].lon/COORDINATE_PRECISION) << ") "; - BOOST_ASSERT_MSG(false, "Graph invariant is not fulfilled."); - } -#endif } SimpleLogger().Write() << "removed " << removed_node_count << " nodes"; m_geometry_compressor.PrintStatistics(); From 9b0d3dfaeb259ed9bb22fb4fe462d6397ce8ab65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Sun, 31 May 2015 13:57:27 +0200 Subject: [PATCH 62/87] polyline string as parameter added --- algorithms/polyline_compressor.cpp | 38 ++++++++++++++++++++++++++++ algorithms/polyline_compressor.hpp | 4 +++ data_structures/route_parameters.cpp | 8 ++++++ include/osrm/route_parameters.hpp | 2 ++ server/api_grammar.hpp | 7 +++-- 5 files changed, 57 insertions(+), 2 deletions(-) diff --git a/algorithms/polyline_compressor.cpp b/algorithms/polyline_compressor.cpp index d5fd5828b..f4eca7350 100644 --- a/algorithms/polyline_compressor.cpp +++ b/algorithms/polyline_compressor.cpp @@ -88,3 +88,41 @@ PolylineCompressor::get_encoded_string(const std::vector &po } return encode_vector(delta_numbers); } + +std::vector PolylineCompressor::decode_string(const std::string geometry_string) const +{ + std::vector new_coordinates; + int index = 0, len = geometry_string.size(); + int lat = 0, lng = 0; + + while (index < len) + { + int b, shift = 0, result = 0; + do + { + b = geometry_string.at(index++) - 63; + result |= (b & 0x1f) << shift; + shift += 5; + } while (b >= 0x20); + int dlat = ((result & 1) != 0 ? ~(result >> 1) : (result >> 1)); + lat += dlat; + + shift = 0; + result = 0; + do + { + b = geometry_string.at(index++) - 63; + result |= (b & 0x1f) << shift; + shift += 5; + } while (b >= 0x20); + int dlng = ((result & 1) != 0 ? ~(result >> 1) : (result >> 1)); + lng += dlng; + + FixedPointCoordinate p; + p.lat = COORDINATE_PRECISION * (((double) lat / 1E5)); + p.lon = COORDINATE_PRECISION * (((double) lng / 1E5)); + new_coordinates.push_back(p); + } + + return new_coordinates; +} diff --git a/algorithms/polyline_compressor.hpp b/algorithms/polyline_compressor.hpp index 933ac7afa..82440d9e2 100644 --- a/algorithms/polyline_compressor.hpp +++ b/algorithms/polyline_compressor.hpp @@ -30,6 +30,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. struct SegmentInformation; +#include + #include #include @@ -42,6 +44,8 @@ class PolylineCompressor public: std::string get_encoded_string(const std::vector &polyline) const; + + std::vector decode_string(const std::string geometry_string) const; }; #endif /* POLYLINECOMPRESSOR_H_ */ diff --git a/data_structures/route_parameters.cpp b/data_structures/route_parameters.cpp index 3b615e2c4..86b955358 100644 --- a/data_structures/route_parameters.cpp +++ b/data_structures/route_parameters.cpp @@ -31,6 +31,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include +#include "../algorithms/polyline_compressor.hpp" + RouteParameters::RouteParameters() : zoom_level(18), print_instructions(false), alternate_route(true), geometry(true), compression(true), deprecatedAPI(false), uturn_default(false), classify(false), @@ -131,3 +133,9 @@ void RouteParameters::addCoordinate( static_cast(COORDINATE_PRECISION * boost::fusion::at_c<0>(received_coordinates)), static_cast(COORDINATE_PRECISION * boost::fusion::at_c<1>(received_coordinates))); } + +void RouteParameters::getCoordinatesFromGeometry(const std::string geometry_string) +{ + PolylineCompressor pc; + coordinates = pc.decode_string(geometry_string); +} diff --git a/include/osrm/route_parameters.hpp b/include/osrm/route_parameters.hpp index 9babbd763..bc413d73b 100644 --- a/include/osrm/route_parameters.hpp +++ b/include/osrm/route_parameters.hpp @@ -78,6 +78,8 @@ struct RouteParameters void setCompressionFlag(const bool flag); void addCoordinate(const boost::fusion::vector &received_coordinates); + + void getCoordinatesFromGeometry(const std::string geometry_string); short zoom_level; bool print_instructions; diff --git a/server/api_grammar.hpp b/server/api_grammar.hpp index a629cfbf7..158f1f7cc 100644 --- a/server/api_grammar.hpp +++ b/server/api_grammar.hpp @@ -42,7 +42,7 @@ template struct APIGrammar : qi::grammar> -(uturns); query = ('?') >> (+(zoom | output | jsonp | checksum | location | hint | timestamp | u | cmp | language | instruction | geometry | alt_route | old_API | num_results | - matching_beta | gps_precision | classify)); + matching_beta | gps_precision | classify | geometry_string)); zoom = (-qi::lit('&')) >> qi::lit('z') >> '=' >> qi::short_[boost::bind(&HandlerT::setZoomLevel, handler, ::_1)]; @@ -83,17 +83,20 @@ template struct APIGrammar : qi::grammar> qi::lit("classify") >> '=' >> qi::bool_[boost::bind(&HandlerT::setClassify, handler, ::_1)]; + geometry_string = (-qi::lit('&')) >> qi::lit("geometry_string") >> '=' >> + stringforPolyline[boost::bind(&HandlerT::getCoordinatesFromGeometry, handler, ::_1)]; string = +(qi::char_("a-zA-Z")); stringwithDot = +(qi::char_("a-zA-Z0-9_.-")); stringwithPercent = +(qi::char_("a-zA-Z0-9_.-") | qi::char_('[') | qi::char_(']') | (qi::char_('%') >> qi::char_("0-9A-Z") >> qi::char_("0-9A-Z"))); + stringforPolyline = +(qi::char_("a-zA-Z0-9_.-[]{}@?|\\%~`^")); } qi::rule api_call, query; qi::rule service, zoom, output, string, jsonp, checksum, location, hint, timestamp, stringwithDot, stringwithPercent, language, instruction, geometry, cmp, alt_route, u, - uturns, old_API, num_results, matching_beta, gps_precision, classify; + uturns, old_API, num_results, matching_beta, gps_precision, classify, geometry_string, stringforPolyline; HandlerT *handler; }; From 8b62d044532c7ea3af99037fd8562d9eaf74de4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Sun, 31 May 2015 14:05:53 +0200 Subject: [PATCH 63/87] test added --- unit_tests/algorithms/geometry_string.cpp | 69 +++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 unit_tests/algorithms/geometry_string.cpp diff --git a/unit_tests/algorithms/geometry_string.cpp b/unit_tests/algorithms/geometry_string.cpp new file mode 100644 index 000000000..4e0f55f33 --- /dev/null +++ b/unit_tests/algorithms/geometry_string.cpp @@ -0,0 +1,69 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include +#include + +#include "../../algorithms/polyline_compressor.hpp" +#include "../../data_structures/coordinate_calculation.hpp" + +#include + +#include +#include + +BOOST_AUTO_TEST_CASE(geometry_string) +{ + // Polyline string for the 5 coordinates + const std::string polyline = "_c`|@_c`|@o}@_pRo}@}oRm}@apRo}@_pR"; + PolylineCompressor pc; + std::vector coords = pc.decode_string(polyline); + + // Test coordinates; these would be the coordinates we give the loc parameter, + // e.g. loc=10.00,10.0&loc=10.01,10.1... + FixedPointCoordinate coord1(10.00 * COORDINATE_PRECISION, 10.0 * COORDINATE_PRECISION); + FixedPointCoordinate coord2(10.01 * COORDINATE_PRECISION, 10.1 * COORDINATE_PRECISION); + FixedPointCoordinate coord3(10.02 * COORDINATE_PRECISION, 10.2 * COORDINATE_PRECISION); + FixedPointCoordinate coord4(10.03 * COORDINATE_PRECISION, 10.3 * COORDINATE_PRECISION); + FixedPointCoordinate coord5(10.04 * COORDINATE_PRECISION, 10.4 * COORDINATE_PRECISION); + + // Put the test coordinates into the vector for comparison + std::vector cmp_coords; + cmp_coords.emplace_back(coord1); + cmp_coords.emplace_back(coord2); + cmp_coords.emplace_back(coord3); + cmp_coords.emplace_back(coord4); + cmp_coords.emplace_back(coord5); + + BOOST_CHECK_EQUAL(cmp_coords.size(), coords.size()); + + for(unsigned i = 0; i < cmp_coords.size(); ++i) + { + BOOST_CHECK_CLOSE(cmp_coords.at(i).lat, coords.at(i).lat, 0.0001); + BOOST_CHECK_CLOSE(cmp_coords.at(i).lon, coords.at(i).lon, 0.0001); + } +} From ee3b296a9957ddc438557f06501a68c7b7031374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Sun, 31 May 2015 15:18:08 +0200 Subject: [PATCH 64/87] fixed values of test --- unit_tests/algorithms/geometry_string.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/unit_tests/algorithms/geometry_string.cpp b/unit_tests/algorithms/geometry_string.cpp index 4e0f55f33..c545e0d69 100644 --- a/unit_tests/algorithms/geometry_string.cpp +++ b/unit_tests/algorithms/geometry_string.cpp @@ -63,7 +63,12 @@ BOOST_AUTO_TEST_CASE(geometry_string) for(unsigned i = 0; i < cmp_coords.size(); ++i) { - BOOST_CHECK_CLOSE(cmp_coords.at(i).lat, coords.at(i).lat, 0.0001); - BOOST_CHECK_CLOSE(cmp_coords.at(i).lon, coords.at(i).lon, 0.0001); + const double cmp1_lat = coords.at(i).lat; + const double cmp2_lat = cmp_coords.at(i).lat; + BOOST_CHECK_CLOSE(cmp1_lat, cmp2_lat, 0.0001); + + const double cmp1_lon = coords.at(i).lon; + const double cmp2_lon = cmp_coords.at(i).lon; + BOOST_CHECK_CLOSE(cmp1_lon, cmp2_lon, 0.0001); } } From 79d2083a0028f8ee2c09f42aba2555cbaaf0ef50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Sun, 31 May 2015 15:28:31 +0200 Subject: [PATCH 65/87] changed parameter from geometry_string to locs --- server/api_grammar.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/api_grammar.hpp b/server/api_grammar.hpp index 158f1f7cc..470954f74 100644 --- a/server/api_grammar.hpp +++ b/server/api_grammar.hpp @@ -42,7 +42,7 @@ template struct APIGrammar : qi::grammar> -(uturns); query = ('?') >> (+(zoom | output | jsonp | checksum | location | hint | timestamp | u | cmp | language | instruction | geometry | alt_route | old_API | num_results | - matching_beta | gps_precision | classify | geometry_string)); + matching_beta | gps_precision | classify | locs)); zoom = (-qi::lit('&')) >> qi::lit('z') >> '=' >> qi::short_[boost::bind(&HandlerT::setZoomLevel, handler, ::_1)]; @@ -83,7 +83,7 @@ template struct APIGrammar : qi::grammar> qi::lit("classify") >> '=' >> qi::bool_[boost::bind(&HandlerT::setClassify, handler, ::_1)]; - geometry_string = (-qi::lit('&')) >> qi::lit("geometry_string") >> '=' >> + locs = (-qi::lit('&')) >> qi::lit("locs") >> '=' >> stringforPolyline[boost::bind(&HandlerT::getCoordinatesFromGeometry, handler, ::_1)]; string = +(qi::char_("a-zA-Z")); @@ -96,7 +96,7 @@ template struct APIGrammar : qi::grammar api_call, query; qi::rule service, zoom, output, string, jsonp, checksum, location, hint, timestamp, stringwithDot, stringwithPercent, language, instruction, geometry, cmp, alt_route, u, - uturns, old_API, num_results, matching_beta, gps_precision, classify, geometry_string, stringforPolyline; + uturns, old_API, num_results, matching_beta, gps_precision, classify, locs, stringforPolyline; HandlerT *handler; }; From 4d73f98050ed7b8dbdf6572d85982093f9dc69e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Tue, 2 Jun 2015 13:06:06 +0200 Subject: [PATCH 66/87] made geometry_string a const reference --- algorithms/polyline_compressor.cpp | 2 +- algorithms/polyline_compressor.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/algorithms/polyline_compressor.cpp b/algorithms/polyline_compressor.cpp index f4eca7350..6377592b7 100644 --- a/algorithms/polyline_compressor.cpp +++ b/algorithms/polyline_compressor.cpp @@ -89,7 +89,7 @@ PolylineCompressor::get_encoded_string(const std::vector &po return encode_vector(delta_numbers); } -std::vector PolylineCompressor::decode_string(const std::string geometry_string) const +std::vector PolylineCompressor::decode_string(const std::string &geometry_string) const { std::vector new_coordinates; int index = 0, len = geometry_string.size(); diff --git a/algorithms/polyline_compressor.hpp b/algorithms/polyline_compressor.hpp index 82440d9e2..a148200ca 100644 --- a/algorithms/polyline_compressor.hpp +++ b/algorithms/polyline_compressor.hpp @@ -45,7 +45,7 @@ class PolylineCompressor public: std::string get_encoded_string(const std::vector &polyline) const; - std::vector decode_string(const std::string geometry_string) const; + std::vector decode_string(const std::string &geometry_string) const; }; #endif /* POLYLINECOMPRESSOR_H_ */ From 782fba2ce75b347ef109f8561b3004a801f769c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Tue, 2 Jun 2015 13:15:31 +0200 Subject: [PATCH 67/87] updated to 1E6 based polyline format --- algorithms/polyline_compressor.cpp | 4 ++-- unit_tests/algorithms/geometry_string.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/algorithms/polyline_compressor.cpp b/algorithms/polyline_compressor.cpp index 6377592b7..0db75dcc6 100644 --- a/algorithms/polyline_compressor.cpp +++ b/algorithms/polyline_compressor.cpp @@ -119,8 +119,8 @@ std::vector PolylineCompressor::decode_string(const std::s lng += dlng; FixedPointCoordinate p; - p.lat = COORDINATE_PRECISION * (((double) lat / 1E5)); - p.lon = COORDINATE_PRECISION * (((double) lng / 1E5)); + p.lat = COORDINATE_PRECISION * (((double) lat / 1E6)); + p.lon = COORDINATE_PRECISION * (((double) lng / 1E6)); new_coordinates.push_back(p); } diff --git a/unit_tests/algorithms/geometry_string.cpp b/unit_tests/algorithms/geometry_string.cpp index c545e0d69..035b3217c 100644 --- a/unit_tests/algorithms/geometry_string.cpp +++ b/unit_tests/algorithms/geometry_string.cpp @@ -39,7 +39,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. BOOST_AUTO_TEST_CASE(geometry_string) { // Polyline string for the 5 coordinates - const std::string polyline = "_c`|@_c`|@o}@_pRo}@}oRm}@apRo}@_pR"; + const std::string polyline = "_gjaR_gjaR_pR_ibE_pR_ibE_pR_ibE_pR_ibE"; PolylineCompressor pc; std::vector coords = pc.decode_string(polyline); From 71dc10ebeaf58db1000b3eb27297cae64a73f7a9 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 2 Jun 2015 16:59:30 +0200 Subject: [PATCH 68/87] Add failing test case --- features/testbot/via.feature | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/features/testbot/via.feature b/features/testbot/via.feature index 52f56a427..a70ee9f7f 100644 --- a/features/testbot/via.feature +++ b/features/testbot/via.feature @@ -52,6 +52,21 @@ Feature: Via points | a,c,f | ab,bcd,bcd,de,efg | | a,c,f,h | ab,bcd,bcd,de,efg,efg,gh | + Scenario: Duplicate via point + Given the node map + | x | | | | | | + | a | 1 | 2 | 3 | 4 | b | + | | | | | | | + + And the ways + | nodes | + | xa | + | ab | + + When I route I should get + | waypoints | route | turns | + | 1,1,4 | ab,ab | head,via,destination | + Scenario: Via points on ring of oneways # xa it to avoid only having a single ring, which cna trigger edge cases Given the node map From 9967dbbaa93209809c9dab7a99aae2746ee687cf Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 2 Jun 2015 17:51:17 +0200 Subject: [PATCH 69/87] Don't remove small segments at start/begin if they are vias --- descriptors/description_factory.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/descriptors/description_factory.cpp b/descriptors/description_factory.cpp index e4665622f..71dd79513 100644 --- a/descriptors/description_factory.cpp +++ b/descriptors/description_factory.cpp @@ -173,6 +173,7 @@ void DescriptionFactory::Run(const unsigned zoom_level) // } // string0 = string1; // } + // float segment_length = 0.; EdgeWeight segment_duration = 0; @@ -197,7 +198,8 @@ void DescriptionFactory::Run(const unsigned zoom_level) // Post-processing to remove empty or nearly empty path segments if (path_description.size() > 2 && - std::numeric_limits::epsilon() > path_description.back().length) + std::numeric_limits::epsilon() > path_description.back().length && + !(path_description.end() - 2)->is_via_location) { path_description.pop_back(); path_description.back().necessary = true; @@ -206,7 +208,8 @@ void DescriptionFactory::Run(const unsigned zoom_level) } if (path_description.size() > 2 && - std::numeric_limits::epsilon() > path_description.front().length) + std::numeric_limits::epsilon() > path_description.front().length && + !(path_description.begin() + 1)->is_via_location) { path_description.erase(path_description.begin()); path_description.front().turn_instruction = TurnInstruction::HeadOn; From eb711787aef5d2629ea76c2bbaa3d2e939fc37c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Wed, 3 Jun 2015 15:31:20 +0200 Subject: [PATCH 70/87] tests added --- features/step_definitions/post.rb | 44 +++++++++++++++++++++++++++++++ features/support/post.rb | 23 ++++++++++++++++ features/testbot/post.feature | 23 ++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 features/step_definitions/post.rb create mode 100644 features/support/post.rb create mode 100644 features/testbot/post.feature diff --git a/features/step_definitions/post.rb b/features/step_definitions/post.rb new file mode 100644 index 000000000..20fe143c4 --- /dev/null +++ b/features/step_definitions/post.rb @@ -0,0 +1,44 @@ +When /^I request post I should get$/ do |table| + reprocess + actual = [] + OSRMLoader.load(self,"#{prepared_file}.osrm") do + table.hashes.each_with_index do |row,ri| + request_string = row['request'].split("?") + got = {'request' => row['request'] } + response = request_post_url request_string[0], request_string[1] + + row.each_pair do |k,v| + if k =~ /param:(.*)/ + if v=='(nil)' + params[$1]=nil + elsif v!=nil + params[$1]=v + end + got[k]=v + end + end + + if table.headers.include? 'status_code' + # the only thing we want to test is + # an accepted request + got['status_code'] = response.code.to_s + end + + ok = true + row.keys.each do |key| + if FuzzyMatch.match got[key], row[key] + got[key] = row[key] + else + ok = false + end + end + + unless ok + log_fail row,got, { 'route' => {:query => @query, :response => response} } + end + + actual << got + end + end + table.diff! actual +end \ No newline at end of file diff --git a/features/support/post.rb b/features/support/post.rb new file mode 100644 index 000000000..bc532a870 --- /dev/null +++ b/features/support/post.rb @@ -0,0 +1,23 @@ +require 'net/http' + +HOST = "http://127.0.0.1:#{OSRM_PORT}" + +def request_post_url service, param_string + uri = URI.parse"#{HOST}/#{service}" + @query = uri.to_s + Timeout.timeout(OSRM_TIMEOUT) do + params = {} + values = param_string.split("loc=") + locs = [] + values.each do |value| + locs << "#{value}".gsub(/[&]/, '') + end + locs.reject! { |c| c.empty? } + params.merge!(loc: locs) + Net::HTTP.post_form uri, params + end +rescue Errno::ECONNREFUSED => e + raise "*** osrm-routed is not running." +rescue Timeout::Error + raise "*** osrm-routed did not respond." +end diff --git a/features/testbot/post.feature b/features/testbot/post.feature new file mode 100644 index 000000000..ce9ca5036 --- /dev/null +++ b/features/testbot/post.feature @@ -0,0 +1,23 @@ +@post @testbot +Feature: POST request + + Background: + Given the profile "testbot" + + Scenario: Accept POST Request + Given the node locations + | node | lat | lon | + | a | 1.00 | 1.00 | + | b | 1.01 | 1.00 | + + And the ways + | nodes | + | ab | + + When I request post I should get + | request | status_code | + | locate?loc=1.0,1.0 | 200 | + | nearest?loc=1.0,1.0 | 200 | + | viaroute?loc=1,1&loc=1.01,1 | 200 | + | match?loc=1,1&loc=1.01,1 | 200 | + | table?loc=1,1&loc=1.01,1 | 200 | From dce917eb74c6b3fd205193077a57a048c29b057f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Thu, 4 Jun 2015 17:39:54 +0200 Subject: [PATCH 71/87] post tests via query options available --- features/step_definitions/locate.rb | 3 +- features/step_definitions/nearest.rb | 3 +- features/step_definitions/post.rb | 44 ----------- features/support/locate.rb | 25 ++++-- features/support/match.rb | 25 +++++- features/support/nearest.rb | 25 ++++-- features/support/post.rb | 23 ------ features/support/route.rb | 22 +++++- features/testbot/post.feature | 113 ++++++++++++++++++++++++--- 9 files changed, 189 insertions(+), 94 deletions(-) delete mode 100644 features/step_definitions/post.rb delete mode 100644 features/support/post.rb diff --git a/features/step_definitions/locate.rb b/features/step_definitions/locate.rb index 190795c26..4a0b4d806 100644 --- a/features/step_definitions/locate.rb +++ b/features/step_definitions/locate.rb @@ -9,7 +9,8 @@ When /^I request locate I should get$/ do |table| out_node = find_node_by_name row['out'] raise "*** unknown out-node '#{row['out']}" unless out_node - response = request_locate("#{in_node.lat},#{in_node.lon}") + params = @query_params + response = request_locate("#{in_node.lat},#{in_node.lon}", params) if response.code == "200" && response.body.empty? == false json = JSON.parse response.body if json['status'] == 0 diff --git a/features/step_definitions/nearest.rb b/features/step_definitions/nearest.rb index 099f4e06f..7240a2442 100644 --- a/features/step_definitions/nearest.rb +++ b/features/step_definitions/nearest.rb @@ -9,7 +9,8 @@ When /^I request nearest I should get$/ do |table| out_node = find_node_by_name row['out'] raise "*** unknown out-node '#{row['out']}" unless out_node - response = request_nearest("#{in_node.lat},#{in_node.lon}") + params = @query_params + response = request_nearest("#{in_node.lat},#{in_node.lon}", params) if response.code == "200" && response.body.empty? == false json = JSON.parse response.body if json['status'] == 0 diff --git a/features/step_definitions/post.rb b/features/step_definitions/post.rb deleted file mode 100644 index 20fe143c4..000000000 --- a/features/step_definitions/post.rb +++ /dev/null @@ -1,44 +0,0 @@ -When /^I request post I should get$/ do |table| - reprocess - actual = [] - OSRMLoader.load(self,"#{prepared_file}.osrm") do - table.hashes.each_with_index do |row,ri| - request_string = row['request'].split("?") - got = {'request' => row['request'] } - response = request_post_url request_string[0], request_string[1] - - row.each_pair do |k,v| - if k =~ /param:(.*)/ - if v=='(nil)' - params[$1]=nil - elsif v!=nil - params[$1]=v - end - got[k]=v - end - end - - if table.headers.include? 'status_code' - # the only thing we want to test is - # an accepted request - got['status_code'] = response.code.to_s - end - - ok = true - row.keys.each do |key| - if FuzzyMatch.match got[key], row[key] - got[key] = row[key] - else - ok = false - end - end - - unless ok - log_fail row,got, { 'route' => {:query => @query, :response => response} } - end - - actual << got - end - end - table.diff! actual -end \ No newline at end of file diff --git a/features/support/locate.rb b/features/support/locate.rb index 900724703..e8406c9ac 100644 --- a/features/support/locate.rb +++ b/features/support/locate.rb @@ -1,10 +1,25 @@ require 'net/http' -def request_locate_url path +def request_locate_url path, method={} @query = path - uri = URI.parse "#{HOST}/#{path}" + + if method.has_key?("post") + request_method = "POST" + else + request_method = "GET" + end + if request_method.eql? "GET" + uri = URI.parse "#{HOST}/#{path}" + elsif request_method.eql? "POST" + uri = URI.parse "#{HOST}/locate" + end Timeout.timeout(OSRM_TIMEOUT) do - Net::HTTP.get_response uri + if request_method.eql? "GET" + Net::HTTP.get_response uri + elsif request_method.eql? "POST" + path.slice!(0, 11) + Net::HTTP.post_form uri, "loc" => path + end end rescue Errno::ECONNREFUSED => e raise "*** osrm-routed is not running." @@ -12,6 +27,6 @@ rescue Timeout::Error raise "*** osrm-routed did not respond." end -def request_locate a - request_locate_url "locate?loc=#{a}" +def request_locate a, method + request_locate_url "locate?loc=#{a}", method end diff --git a/features/support/match.rb b/features/support/match.rb index bf51189a4..3decd2428 100644 --- a/features/support/match.rb +++ b/features/support/match.rb @@ -13,10 +13,31 @@ def request_matching trace=[], timestamps=[], options={} end params = (trace_params + defaults.merge(options).to_param).join('&') params = nil if params=="" - uri = URI.parse ["#{HOST}/match", params].compact.join('?') + + if options.has_key?("post") + request_method = "POST" + options.delete("post") + else + request_method = "GET" + end + if request_method.eql? "GET" + uri = URI.parse ["#{HOST}/match", params].compact.join('?') + elsif request_method.eql? "POST" + uri = URI.parse "#{HOST}/match" + end @query = uri.to_s Timeout.timeout(OSRM_TIMEOUT) do - Net::HTTP.get_response uri + if request_method.eql? "GET" + Net::HTTP.get_response uri + elsif request_method.eql? "POST" + datas = {} + datas[:loc] = trace.compact.map { |w| "#{w.lat},#{w.lon}" } + if ts.length > 0 + datas[:t] = timestamps.compact.map { |t| "#{t}" } + end + datas.merge! options + Net::HTTP.post_form uri, datas + end end rescue Errno::ECONNREFUSED => e raise "*** osrm-routed is not running." diff --git a/features/support/nearest.rb b/features/support/nearest.rb index 77fc351a9..10e0c76ca 100644 --- a/features/support/nearest.rb +++ b/features/support/nearest.rb @@ -1,10 +1,25 @@ require 'net/http' -def request_nearest_url path +def request_nearest_url path, method={} @query = path - uri = URI.parse "#{HOST}/#{path}" + + if method.has_key?("post") + request_method = "POST" + else + request_method = "GET" + end + if request_method.eql? "GET" + uri = URI.parse "#{HOST}/#{path}" + elsif request_method.eql? "POST" + uri = URI.parse "#{HOST}/nearest" + end Timeout.timeout(OSRM_TIMEOUT) do - Net::HTTP.get_response uri + if request_method.eql? "GET" + Net::HTTP.get_response uri + elsif request_method.eql? "POST" + path.slice!(0, 12) + Net::HTTP.post_form uri, "loc" => path + end end rescue Errno::ECONNREFUSED => e raise "*** osrm-routed is not running." @@ -12,6 +27,6 @@ rescue Timeout::Error raise "*** osrm-routed did not respond." end -def request_nearest a - request_nearest_url "nearest?loc=#{a}" +def request_nearest a, method + request_nearest_url "nearest?loc=#{a}", method end diff --git a/features/support/post.rb b/features/support/post.rb deleted file mode 100644 index bc532a870..000000000 --- a/features/support/post.rb +++ /dev/null @@ -1,23 +0,0 @@ -require 'net/http' - -HOST = "http://127.0.0.1:#{OSRM_PORT}" - -def request_post_url service, param_string - uri = URI.parse"#{HOST}/#{service}" - @query = uri.to_s - Timeout.timeout(OSRM_TIMEOUT) do - params = {} - values = param_string.split("loc=") - locs = [] - values.each do |value| - locs << "#{value}".gsub(/[&]/, '') - end - locs.reject! { |c| c.empty? } - params.merge!(loc: locs) - Net::HTTP.post_form uri, params - end -rescue Errno::ECONNREFUSED => e - raise "*** osrm-routed is not running." -rescue Timeout::Error - raise "*** osrm-routed did not respond." -end diff --git a/features/support/route.rb b/features/support/route.rb index a8c78227e..615a1f880 100644 --- a/features/support/route.rb +++ b/features/support/route.rb @@ -15,10 +15,28 @@ def request_path path, waypoints=[], options={} locs = waypoints.compact.map { |w| "loc=#{w.lat},#{w.lon}" } params = (locs + options.to_param).join('&') params = nil if params=="" - uri = URI.parse ["#{HOST}/#{path}", params].compact.join('?') + + if options.has_key?("post") + request_method = "POST" + options.delete("post") + else + request_method = "GET" + end + if request_method.eql? "GET" + uri = URI.parse ["#{HOST}/#{path}", params].compact.join('?') + elsif request_method.eql? "POST" + uri = URI.parse "#{HOST}/#{path}" + end @query = uri.to_s Timeout.timeout(OSRM_TIMEOUT) do - Net::HTTP.get_response uri + if request_method.eql? "GET" + Net::HTTP.get_response uri + elsif request_method.eql? "POST" + datas = {} + datas[:loc] = waypoints.compact.map { |w| "#{w.lat},#{w.lon}" } + datas.merge! options + Net::HTTP.post_form uri, datas + end end rescue Errno::ECONNREFUSED => e raise "*** osrm-routed is not running." diff --git a/features/testbot/post.feature b/features/testbot/post.feature index ce9ca5036..c2d07c242 100644 --- a/features/testbot/post.feature +++ b/features/testbot/post.feature @@ -4,20 +4,111 @@ Feature: POST request Background: Given the profile "testbot" - Scenario: Accept POST Request + Scenario: Testbot - viaroute POST request Given the node locations - | node | lat | lon | - | a | 1.00 | 1.00 | - | b | 1.01 | 1.00 | + | node | lat | lon | + | a | 55.68740 | 12.52430 | + | b | 55.68745 | 12.52409 | + | c | 55.68711 | 12.52383 | + | x | -55.68740 | 12.52430 | + | y | -55.68745 | 12.52409 | + | z | -55.68711 | 12.52383 | And the ways | nodes | | ab | + | bc | + | xy | + | yz | - When I request post I should get - | request | status_code | - | locate?loc=1.0,1.0 | 200 | - | nearest?loc=1.0,1.0 | 200 | - | viaroute?loc=1,1&loc=1.01,1 | 200 | - | match?loc=1,1&loc=1.01,1 | 200 | - | table?loc=1,1&loc=1.01,1 | 200 | + And the query options + | post | true | + + When I route I should get + | from | to | route | turns | + | a | c | ab,bc | head,left,destination | + | c | a | bc,ab | head,right,destination | + | x | z | xy,yz | head,right,destination | + | z | x | yz,xy | head,left,destination | + + Scenario: Testbot - match POST request + Given the node map + | a | b | c | d | + | e | f | g | h | + + And the ways + | nodes | oneway | + | abcd | yes | + | hgfe | yes | + + And the query options + | post | true | + + When I match I should get + | trace | matchings | + | dcba | hgfe | + + Scenario: Testbot - table POST request + Given the node map + | x | a | b | y | + | | d | e | | + + And the ways + | nodes | oneway | + | abeda | yes | + | xa | | + | by | | + + And the query options + | post | true | + + When I request a travel time matrix I should get + | | x | y | d | e | + | x | 0 | 300 | 400 | 300 | + | y | 500 | 0 | 300 | 200 | + | d | 200 | 300 | 0 | 300 | + | e | 300 | 400 | 100 | 0 | + + Scenario: Testbot - locate POST request + Given the node locations + | node | lat | lon | + | a | -85 | -180 | + | b | 0 | 0 | + | c | 85 | 180 | + | x | -84 | -180 | + | y | 84 | 180 | + + And the ways + | nodes | + | abc | + + And the query options + | post | true | + + When I request locate I should get + | in | out | + | x | a | + | y | c | + + Scenario: Testbot - nearest POST request + Given the node locations + | node | lat | lon | + | a | -85 | -180 | + | b | -85 | -160 | + | c | -85 | -140 | + | x | -84.999 | -180 | + | y | -84.999 | -160 | + | z | -84.999 | -140 | + + And the ways + | nodes | + | abc | + + And the query options + | post | true | + + When I request nearest I should get + | in | out | + | x | a | + | y | b | + | z | c | From 153d38f10c1fbb87221d42766dc49a8f1a2f7986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Fri, 5 Jun 2015 13:26:27 +0200 Subject: [PATCH 72/87] post/get handler added, background section for HTTP request --- features/step_definitions/data.rb | 4 +++ features/step_definitions/locate.rb | 3 +- features/step_definitions/nearest.rb | 3 +- features/support/http.rb | 50 ++++++++++++++++++++++++++++ features/support/locate.rb | 30 +++-------------- features/support/match.rb | 31 ++--------------- features/support/nearest.rb | 30 +++-------------- features/support/route.rb | 28 +++------------- features/testbot/post.feature | 16 +-------- 9 files changed, 73 insertions(+), 122 deletions(-) create mode 100644 features/support/http.rb diff --git a/features/step_definitions/data.rb b/features/step_definitions/data.rb index f1e58bae2..524fd322c 100644 --- a/features/step_definitions/data.rb +++ b/features/step_definitions/data.rb @@ -174,3 +174,7 @@ end Given /^data is loaded with datastore$/ do @load_method = 'datastore' end + +Given /^the HTTP method "([^"]*)"$/ do |method| + @http_method = method +end diff --git a/features/step_definitions/locate.rb b/features/step_definitions/locate.rb index 4a0b4d806..190795c26 100644 --- a/features/step_definitions/locate.rb +++ b/features/step_definitions/locate.rb @@ -9,8 +9,7 @@ When /^I request locate I should get$/ do |table| out_node = find_node_by_name row['out'] raise "*** unknown out-node '#{row['out']}" unless out_node - params = @query_params - response = request_locate("#{in_node.lat},#{in_node.lon}", params) + response = request_locate("#{in_node.lat},#{in_node.lon}") if response.code == "200" && response.body.empty? == false json = JSON.parse response.body if json['status'] == 0 diff --git a/features/step_definitions/nearest.rb b/features/step_definitions/nearest.rb index 7240a2442..099f4e06f 100644 --- a/features/step_definitions/nearest.rb +++ b/features/step_definitions/nearest.rb @@ -9,8 +9,7 @@ When /^I request nearest I should get$/ do |table| out_node = find_node_by_name row['out'] raise "*** unknown out-node '#{row['out']}" unless out_node - params = @query_params - response = request_nearest("#{in_node.lat},#{in_node.lon}", params) + response = request_nearest("#{in_node.lat},#{in_node.lon}") if response.code == "200" && response.body.empty? == false json = JSON.parse response.body if json['status'] == 0 diff --git a/features/support/http.rb b/features/support/http.rb new file mode 100644 index 000000000..2233f8286 --- /dev/null +++ b/features/support/http.rb @@ -0,0 +1,50 @@ +require 'net/http' + +def generate_request_url path + if @http_method.eql? "POST" + pos = path.index('?') - 1 + service = path[0..pos] + uri = URI.parse "#{HOST}/#{service}" + else + uri = URI.parse "#{HOST}/#{path}" + end +end + +def send_simple_request uri, path + Timeout.timeout(OSRM_TIMEOUT) do + if @http_method.eql? "POST" + pos = path.index('=') + 1 + path.slice!(0, pos) + response = Net::HTTP.post_form uri, "loc" => path + else + response = Net::HTTP.get_response uri + end + end +rescue Errno::ECONNREFUSED => e + raise "*** osrm-routed is not running." +rescue Timeout::Error + raise "*** osrm-routed did not respond." +end + +def send_request uri, waypoints, timestamps, options + @query = uri.to_s + Timeout.timeout(OSRM_TIMEOUT) do + if @http_method.eql? "POST" + datas = {} + if waypoints.length > 0 + datas[:loc] = waypoints.compact.map { |w| "#{w.lat},#{w.lon}" } + end + if timestamps.length > 0 + datas[:t] = timestamps.compact.map { |t| "#{t}" } + end + datas.merge! options + response = Net::HTTP.post_form uri, datas + else + response = Net::HTTP.get_response uri + end + end +rescue Errno::ECONNREFUSED => e + raise "*** osrm-routed is not running." +rescue Timeout::Error + raise "*** osrm-routed did not respond." +end diff --git a/features/support/locate.rb b/features/support/locate.rb index e8406c9ac..586d0b393 100644 --- a/features/support/locate.rb +++ b/features/support/locate.rb @@ -1,32 +1,12 @@ require 'net/http' -def request_locate_url path, method={} +def request_locate_url path @query = path - if method.has_key?("post") - request_method = "POST" - else - request_method = "GET" - end - if request_method.eql? "GET" - uri = URI.parse "#{HOST}/#{path}" - elsif request_method.eql? "POST" - uri = URI.parse "#{HOST}/locate" - end - Timeout.timeout(OSRM_TIMEOUT) do - if request_method.eql? "GET" - Net::HTTP.get_response uri - elsif request_method.eql? "POST" - path.slice!(0, 11) - Net::HTTP.post_form uri, "loc" => path - end - end -rescue Errno::ECONNREFUSED => e - raise "*** osrm-routed is not running." -rescue Timeout::Error - raise "*** osrm-routed did not respond." + uri = generate_request_url path + response = send_simple_request uri, path end -def request_locate a, method - request_locate_url "locate?loc=#{a}", method +def request_locate a + request_locate_url "locate?loc=#{a}" end diff --git a/features/support/match.rb b/features/support/match.rb index 3decd2428..2f75f08d8 100644 --- a/features/support/match.rb +++ b/features/support/match.rb @@ -14,34 +14,7 @@ def request_matching trace=[], timestamps=[], options={} params = (trace_params + defaults.merge(options).to_param).join('&') params = nil if params=="" - if options.has_key?("post") - request_method = "POST" - options.delete("post") - else - request_method = "GET" - end - if request_method.eql? "GET" - uri = URI.parse ["#{HOST}/match", params].compact.join('?') - elsif request_method.eql? "POST" - uri = URI.parse "#{HOST}/match" - end - @query = uri.to_s - Timeout.timeout(OSRM_TIMEOUT) do - if request_method.eql? "GET" - Net::HTTP.get_response uri - elsif request_method.eql? "POST" - datas = {} - datas[:loc] = trace.compact.map { |w| "#{w.lat},#{w.lon}" } - if ts.length > 0 - datas[:t] = timestamps.compact.map { |t| "#{t}" } - end - datas.merge! options - Net::HTTP.post_form uri, datas - end - end -rescue Errno::ECONNREFUSED => e - raise "*** osrm-routed is not running." -rescue Timeout::Error - raise "*** osrm-routed did not respond." + uri = generate_request_url ("match" + '?' + params) + response = send_request uri, trace, timestamps, options end diff --git a/features/support/nearest.rb b/features/support/nearest.rb index 10e0c76ca..f76ddc9f6 100644 --- a/features/support/nearest.rb +++ b/features/support/nearest.rb @@ -1,32 +1,12 @@ require 'net/http' -def request_nearest_url path, method={} +def request_nearest_url path @query = path - if method.has_key?("post") - request_method = "POST" - else - request_method = "GET" - end - if request_method.eql? "GET" - uri = URI.parse "#{HOST}/#{path}" - elsif request_method.eql? "POST" - uri = URI.parse "#{HOST}/nearest" - end - Timeout.timeout(OSRM_TIMEOUT) do - if request_method.eql? "GET" - Net::HTTP.get_response uri - elsif request_method.eql? "POST" - path.slice!(0, 12) - Net::HTTP.post_form uri, "loc" => path - end - end -rescue Errno::ECONNREFUSED => e - raise "*** osrm-routed is not running." -rescue Timeout::Error - raise "*** osrm-routed did not respond." + uri = generate_request_url path + response = send_simple_request uri, path end -def request_nearest a, method - request_nearest_url "nearest?loc=#{a}", method +def request_nearest a + request_nearest_url "nearest?loc=#{a}" end diff --git a/features/support/route.rb b/features/support/route.rb index 615a1f880..f7c14e9ff 100644 --- a/features/support/route.rb +++ b/features/support/route.rb @@ -16,32 +16,12 @@ def request_path path, waypoints=[], options={} params = (locs + options.to_param).join('&') params = nil if params=="" - if options.has_key?("post") - request_method = "POST" - options.delete("post") + if params == nil + uri = generate_request_url (path) else - request_method = "GET" + uri = generate_request_url (path + '?' + params) end - if request_method.eql? "GET" - uri = URI.parse ["#{HOST}/#{path}", params].compact.join('?') - elsif request_method.eql? "POST" - uri = URI.parse "#{HOST}/#{path}" - end - @query = uri.to_s - Timeout.timeout(OSRM_TIMEOUT) do - if request_method.eql? "GET" - Net::HTTP.get_response uri - elsif request_method.eql? "POST" - datas = {} - datas[:loc] = waypoints.compact.map { |w| "#{w.lat},#{w.lon}" } - datas.merge! options - Net::HTTP.post_form uri, datas - end - end -rescue Errno::ECONNREFUSED => e - raise "*** osrm-routed is not running." -rescue Timeout::Error - raise "*** osrm-routed did not respond." + response = send_request uri, waypoints, {}, options end def request_url path diff --git a/features/testbot/post.feature b/features/testbot/post.feature index c2d07c242..ff2282406 100644 --- a/features/testbot/post.feature +++ b/features/testbot/post.feature @@ -3,6 +3,7 @@ Feature: POST request Background: Given the profile "testbot" + And the HTTP method "POST" Scenario: Testbot - viaroute POST request Given the node locations @@ -20,9 +21,6 @@ Feature: POST request | bc | | xy | | yz | - - And the query options - | post | true | When I route I should get | from | to | route | turns | @@ -40,9 +38,6 @@ Feature: POST request | nodes | oneway | | abcd | yes | | hgfe | yes | - - And the query options - | post | true | When I match I should get | trace | matchings | @@ -59,9 +54,6 @@ Feature: POST request | xa | | | by | | - And the query options - | post | true | - When I request a travel time matrix I should get | | x | y | d | e | | x | 0 | 300 | 400 | 300 | @@ -81,9 +73,6 @@ Feature: POST request And the ways | nodes | | abc | - - And the query options - | post | true | When I request locate I should get | in | out | @@ -103,9 +92,6 @@ Feature: POST request And the ways | nodes | | abc | - - And the query options - | post | true | When I request nearest I should get | in | out | From b406844c96b8b381ec85ec70062e347af6b53089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Sun, 7 Jun 2015 11:02:24 +0200 Subject: [PATCH 73/87] rearranged send_request parameters --- features/support/http.rb | 2 +- features/support/match.rb | 2 +- features/support/route.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/features/support/http.rb b/features/support/http.rb index 2233f8286..2eb298547 100644 --- a/features/support/http.rb +++ b/features/support/http.rb @@ -26,7 +26,7 @@ rescue Timeout::Error raise "*** osrm-routed did not respond." end -def send_request uri, waypoints, timestamps, options +def send_request uri, waypoints=[], options={}, timestamps=[] @query = uri.to_s Timeout.timeout(OSRM_TIMEOUT) do if @http_method.eql? "POST" diff --git a/features/support/match.rb b/features/support/match.rb index 2f75f08d8..8bcf7a2c6 100644 --- a/features/support/match.rb +++ b/features/support/match.rb @@ -15,6 +15,6 @@ def request_matching trace=[], timestamps=[], options={} params = nil if params=="" uri = generate_request_url ("match" + '?' + params) - response = send_request uri, trace, timestamps, options + response = send_request uri, trace, options, timestamps end diff --git a/features/support/route.rb b/features/support/route.rb index f7c14e9ff..ca353d6e6 100644 --- a/features/support/route.rb +++ b/features/support/route.rb @@ -21,7 +21,7 @@ def request_path path, waypoints=[], options={} else uri = generate_request_url (path + '?' + params) end - response = send_request uri, waypoints, {}, options + response = send_request uri, waypoints, options end def request_url path From d726ce6340e79807d1d3f1861979f42c3bf9bea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Sun, 7 Jun 2015 12:20:03 +0200 Subject: [PATCH 74/87] removed send_simple_request --- features/step_definitions/locate.rb | 2 +- features/step_definitions/nearest.rb | 2 +- features/support/http.rb | 16 ---------------- features/support/locate.rb | 10 +++++----- features/support/nearest.rb | 8 ++++---- 5 files changed, 11 insertions(+), 27 deletions(-) diff --git a/features/step_definitions/locate.rb b/features/step_definitions/locate.rb index 190795c26..2fd8e627b 100644 --- a/features/step_definitions/locate.rb +++ b/features/step_definitions/locate.rb @@ -9,7 +9,7 @@ When /^I request locate I should get$/ do |table| out_node = find_node_by_name row['out'] raise "*** unknown out-node '#{row['out']}" unless out_node - response = request_locate("#{in_node.lat},#{in_node.lon}") + response = request_locate(in_node) if response.code == "200" && response.body.empty? == false json = JSON.parse response.body if json['status'] == 0 diff --git a/features/step_definitions/nearest.rb b/features/step_definitions/nearest.rb index 099f4e06f..481c74167 100644 --- a/features/step_definitions/nearest.rb +++ b/features/step_definitions/nearest.rb @@ -9,7 +9,7 @@ When /^I request nearest I should get$/ do |table| out_node = find_node_by_name row['out'] raise "*** unknown out-node '#{row['out']}" unless out_node - response = request_nearest("#{in_node.lat},#{in_node.lon}") + response = request_nearest(in_node) if response.code == "200" && response.body.empty? == false json = JSON.parse response.body if json['status'] == 0 diff --git a/features/support/http.rb b/features/support/http.rb index 2eb298547..80dad8a85 100644 --- a/features/support/http.rb +++ b/features/support/http.rb @@ -10,22 +10,6 @@ def generate_request_url path end end -def send_simple_request uri, path - Timeout.timeout(OSRM_TIMEOUT) do - if @http_method.eql? "POST" - pos = path.index('=') + 1 - path.slice!(0, pos) - response = Net::HTTP.post_form uri, "loc" => path - else - response = Net::HTTP.get_response uri - end - end -rescue Errno::ECONNREFUSED => e - raise "*** osrm-routed is not running." -rescue Timeout::Error - raise "*** osrm-routed did not respond." -end - def send_request uri, waypoints=[], options={}, timestamps=[] @query = uri.to_s Timeout.timeout(OSRM_TIMEOUT) do diff --git a/features/support/locate.rb b/features/support/locate.rb index 586d0b393..a62e53aa9 100644 --- a/features/support/locate.rb +++ b/features/support/locate.rb @@ -1,12 +1,12 @@ require 'net/http' -def request_locate_url path +def request_locate_url path, node @query = path - + uri = generate_request_url path - response = send_simple_request uri, path + response = send_request uri, [node] end -def request_locate a - request_locate_url "locate?loc=#{a}" +def request_locate node + request_locate_url "locate?loc=#{node.lat},#{node.lon}", node end diff --git a/features/support/nearest.rb b/features/support/nearest.rb index f76ddc9f6..af52b49e0 100644 --- a/features/support/nearest.rb +++ b/features/support/nearest.rb @@ -1,12 +1,12 @@ require 'net/http' -def request_nearest_url path +def request_nearest_url path, node @query = path uri = generate_request_url path - response = send_simple_request uri, path + response = send_request uri, [node] end -def request_nearest a - request_nearest_url "nearest?loc=#{a}" +def request_nearest node + request_nearest_url "nearest?loc=#{node.lat},#{node.lon}", node end From eec4f173a74e360fe2b7ab9f905c0143ecdb89a3 Mon Sep 17 00:00:00 2001 From: Lauren Budorick Date: Tue, 9 Jun 2015 20:16:58 -0700 Subject: [PATCH 75/87] Fix tag misspellings in profiles + tests: forestry, pebblestone --- features/bicycle/access.feature | 6 +++--- features/bicycle/access_node.feature | 6 +++--- features/bicycle/surface.feature | 2 +- features/foot/access.feature | 4 ++-- features/foot/access_node.feature | 4 ++-- features/foot/surface.feature | 2 +- profiles/bicycle.lua | 5 ++--- profiles/foot.lua | 4 ++-- 8 files changed, 16 insertions(+), 17 deletions(-) diff --git a/features/bicycle/access.feature b/features/bicycle/access.feature index 2087327ca..3a2fffbc0 100644 --- a/features/bicycle/access.feature +++ b/features/bicycle/access.feature @@ -120,7 +120,7 @@ Feature: Bike - Access tags on ways | no | | | | | private | | | | | agricultural | | | | - | forestery | | | | + | forestry | | | | | | yes | | x | | | permissive | | x | | | designated | | x | @@ -128,7 +128,7 @@ Feature: Bike - Access tags on ways | | no | | | | | private | | | | | agricultural | | | - | | forestery | | | + | | forestry | | | | | | yes | x | | | | permissive | x | | | | designated | x | @@ -136,7 +136,7 @@ Feature: Bike - Access tags on ways | | | no | | | | | private | | | | | agricultural | | - | | | forestery | | + | | | forestry | | Scenario: Bike - Access tags on both node and way Then routability should be diff --git a/features/bicycle/access_node.feature b/features/bicycle/access_node.feature index 1e08a94ba..05e94feae 100644 --- a/features/bicycle/access_node.feature +++ b/features/bicycle/access_node.feature @@ -46,7 +46,7 @@ Feature: Bike - Access tags on nodes | no | | | | | private | | | | | agricultural | | | | - | forestery | | | | + | forestry | | | | | | yes | | x | | | permissive | | x | | | designated | | x | @@ -54,7 +54,7 @@ Feature: Bike - Access tags on nodes | | no | | | | | private | | | | | agricultural | | | - | | forestery | | | + | | forestry | | | | | | yes | x | | | | permissive | x | | | | designated | x | @@ -62,4 +62,4 @@ Feature: Bike - Access tags on nodes | | | no | | | | | private | | | | | agricultural | | - | | | forestery | | + | | | forestry | | diff --git a/features/bicycle/surface.feature b/features/bicycle/surface.feature index 216307b21..8d303aa12 100644 --- a/features/bicycle/surface.feature +++ b/features/bicycle/surface.feature @@ -16,7 +16,7 @@ Feature: Bike - Surfaces | cycleway | unpaved | 120s | | cycleway | fine_gravel | 120s | | cycleway | gravel | 120s | - | cycleway | pebbelstone | 120s | + | cycleway | pebblestone | 120s | | cycleway | dirt | 120s | | cycleway | earth | 120s | | cycleway | grass | 120s | diff --git a/features/foot/access.feature b/features/foot/access.feature index 4d81d0e45..353076c40 100644 --- a/features/foot/access.feature +++ b/features/foot/access.feature @@ -51,7 +51,7 @@ Feature: Foot - Access tags on ways | no | | | | private | | | | agricultural | | | - | forestery | | | + | forestry | | | | | yes | x | | | permissive | x | | | designated | x | @@ -59,7 +59,7 @@ Feature: Foot - Access tags on ways | | no | | | | private | | | | agricultural | | - | | forestery | | + | | forestry | | Scenario: Foot - Access tags on both node and way Then routability should be diff --git a/features/foot/access_node.feature b/features/foot/access_node.feature index 5a4a1314e..8519d0564 100644 --- a/features/foot/access_node.feature +++ b/features/foot/access_node.feature @@ -39,7 +39,7 @@ Feature: Foot - Access tags on nodes | no | | | | private | | | | agricultural | | | - | forestery | | | + | forestry | | | | no | yes | x | | no | permissive | x | | no | designated | x | @@ -47,4 +47,4 @@ Feature: Foot - Access tags on nodes | yes | no | | | yes | private | | | yes | agricultural | | - | yes | forestery | | + | yes | forestry | | diff --git a/features/foot/surface.feature b/features/foot/surface.feature index 556c3d893..9bcd80c78 100644 --- a/features/foot/surface.feature +++ b/features/foot/surface.feature @@ -10,6 +10,6 @@ Feature: Foot - Surfaces | footway | | 145s ~10% | | footway | fine_gravel | 193s ~10% | | footway | gravel | 193s ~10% | - | footway | pebbelstone | 193s ~10% | + | footway | pebblestone | 193s ~10% | | footway | mud | 289s ~10% | | footway | sand | 289s ~10% | diff --git a/profiles/bicycle.lua b/profiles/bicycle.lua index 4053d3417..f23606390 100644 --- a/profiles/bicycle.lua +++ b/profiles/bicycle.lua @@ -6,7 +6,7 @@ local limit = require("lib/maxspeed").limit -- Begin of globals barrier_whitelist = { [""] = true, ["cycle_barrier"] = true, ["bollard"] = true, ["entrance"] = true, ["cattle_grid"] = true, ["border_control"] = true, ["toll_booth"] = true, ["sally_port"] = true, ["gate"] = true, ["no"] = true } access_tag_whitelist = { ["yes"] = true, ["permissive"] = true, ["designated"] = true } -access_tag_blacklist = { ["no"] = true, ["private"] = true, ["agricultural"] = true, ["forestery"] = true } +access_tag_blacklist = { ["no"] = true, ["private"] = true, ["agricultural"] = true, ["forestry"] = true } access_tag_restricted = { ["destination"] = true, ["delivery"] = true } access_tags_hierachy = { "bicycle", "vehicle", "access" } cycleway_tags = {["track"]=true,["lane"]=true,["opposite"]=true,["opposite_lane"]=true,["opposite_track"]=true,["share_busway"]=true,["sharrow"]=true,["shared"]=true } @@ -81,8 +81,7 @@ surface_speeds = { ["unpaved"] = 6, ["fine_gravel"] = 6, ["gravel"] = 6, - ["fine_gravel"] = 6, - ["pebbelstone"] = 6, + ["pebblestone"] = 6, ["ground"] = 6, ["dirt"] = 6, ["earth"] = 6, diff --git a/profiles/foot.lua b/profiles/foot.lua index 231dc3eb3..4b3bc1cca 100644 --- a/profiles/foot.lua +++ b/profiles/foot.lua @@ -5,7 +5,7 @@ local find_access_tag = require("lib/access").find_access_tag -- Begin of globals barrier_whitelist = { [""] = true, ["cycle_barrier"] = true, ["bollard"] = true, ["entrance"] = true, ["cattle_grid"] = true, ["border_control"] = true, ["toll_booth"] = true, ["sally_port"] = true, ["gate"] = true, ["no"] = true} access_tag_whitelist = { ["yes"] = true, ["foot"] = true, ["permissive"] = true, ["designated"] = true } -access_tag_blacklist = { ["no"] = true, ["private"] = true, ["agricultural"] = true, ["forestery"] = true } +access_tag_blacklist = { ["no"] = true, ["private"] = true, ["agricultural"] = true, ["forestry"] = true } access_tag_restricted = { ["destination"] = true, ["delivery"] = true } access_tags_hierachy = { "foot", "access" } service_tag_restricted = { ["parking_aisle"] = true } @@ -55,7 +55,7 @@ man_made_speeds = { surface_speeds = { ["fine_gravel"] = walking_speed*0.75, ["gravel"] = walking_speed*0.75, - ["pebbelstone"] = walking_speed*0.75, + ["pebblestone"] = walking_speed*0.75, ["mud"] = walking_speed*0.5, ["sand"] = walking_speed*0.5 } From bdbc60b4f7b24e24d0edec7ac1a477c31c7af33e Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 16 Jun 2015 11:21:38 +0200 Subject: [PATCH 76/87] Fix comments in edge based graph factory --- contractor/edge_based_graph_factory.cpp | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index 203c6b632..dbd73a73c 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -193,15 +193,16 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, { BOOST_ASSERT(forward_data.forward); } + else + { + BOOST_ASSERT(!forward_data.forward); + } + if (reverse_data.edgeBasedNodeID != SPECIAL_NODEID) { BOOST_ASSERT(reverse_data.forward); } - if (forward_data.edgeBasedNodeID == SPECIAL_NODEID) - { - BOOST_ASSERT(!forward_data.forward); - } - if (reverse_data.edgeBasedNodeID == SPECIAL_NODEID) + else { BOOST_ASSERT(!reverse_data.forward); } @@ -433,9 +434,8 @@ void EdgeBasedGraphFactory::CompressGeometry() << new_edge_count / (double)original_number_of_edges; } -/** - * Writes the id of the edge in the edge expanded graph (into the edge in the node based graph) - */ +/// Renumbers all _forward_ edges and sets the edgeBasedNodeID. +/// A specific numbering is not important. Any unique ID will do. void EdgeBasedGraphFactory::RenumberEdges() { // renumber edge based node of outgoing edges @@ -498,7 +498,8 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes() const NodeID node_v = m_node_based_graph->GetTarget(e1); BOOST_ASSERT(SPECIAL_NODEID != node_v); - // pick only every other edge + // pick only every other edge, since we have every edge as an outgoing + // and incoming egde if (node_u > node_v) { continue; @@ -524,6 +525,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes() const bool component_is_tiny = size_of_component < 1000; + // we only set edgeBasedNodeID for forward edges if (edge_data.edgeBasedNodeID == SPECIAL_NODEID) { InsertEdgeBasedNode(node_v, node_u, From 8ade26b4a4ff2a91c75542629c065da843e794fd Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 17 Jun 2015 00:51:53 +0200 Subject: [PATCH 77/87] Simplify offeset calculation logic a little bit --- contractor/edge_based_graph_factory.cpp | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index dbd73a73c..e76f7be52 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -128,6 +128,11 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, // TODO: move to lambda function with C++11 int temp_sum = 0; + // u-----------------------------------v + // 0 1 i-1 i n-1 + // x --> x --> ... --> x --> x ... --> x + // |-------------------| + // forward_dist_prefix_sum[i] for (const auto i : osrm::irange(0u, geometry_size)) { forward_dist_prefix_sum[i] = temp_sum; @@ -135,14 +140,22 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, BOOST_ASSERT(forward_data.distance >= temp_sum); } + BOOST_ASSERT(forward_dist_prefix_sum.back() + forward_geometry.back().second == forward_data.distance); + // v-----------------------------------u + // 0 1 i-1 i n-1 + // x --> x --> ... --> x --> x ... --> x + // |-------------------| + // reverse_dist_prefix_sum[i] temp_sum = 0; for (const auto i : osrm::irange(0u, geometry_size)) { - temp_sum += reverse_geometry[reverse_geometry.size() - 1 - i].second; - reverse_dist_prefix_sum[i] = reverse_data.distance - temp_sum; - // BOOST_ASSERT(reverse_data.distance >= temp_sum); + reverse_dist_prefix_sum[i] = temp_sum; + temp_sum += reverse_geometry[i].second; + + BOOST_ASSERT(reverse_data.distance >= temp_sum); } + BOOST_ASSERT(reverse_dist_prefix_sum.back() + reverse_geometry.back().second == reverse_data.distance); NodeID current_edge_source_coordinate_id = node_u; @@ -168,8 +181,10 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, forward_data.edgeBasedNodeID, reverse_data.edgeBasedNodeID, current_edge_source_coordinate_id, current_edge_target_coordinate_id, forward_data.nameID, forward_geometry[i].second, - reverse_geometry[geometry_size - 1 - i].second, forward_dist_prefix_sum[i], - reverse_dist_prefix_sum[i], m_geometry_compressor.GetPositionForID(edge_id_1), + reverse_geometry[geometry_size - 1 - i].second, + forward_dist_prefix_sum[i], + reverse_dist_prefix_sum[geometry_size - 1 - i], + m_geometry_compressor.GetPositionForID(edge_id_1), component_id, i, forward_data.travel_mode, reverse_data.travel_mode); current_edge_source_coordinate_id = current_edge_target_coordinate_id; From 5c77bb7c677d089b98c584101e9157405558856d Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 17 Jun 2015 19:51:35 +0200 Subject: [PATCH 78/87] Fix inversion of sign --- routing_algorithms/alternative_path.hpp | 4 ++-- routing_algorithms/shortest_path.hpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/routing_algorithms/alternative_path.hpp b/routing_algorithms/alternative_path.hpp index 1dcc02096..327bb329f 100644 --- a/routing_algorithms/alternative_path.hpp +++ b/routing_algorithms/alternative_path.hpp @@ -103,8 +103,8 @@ class AlternativeRouting final int upper_bound_to_shortest_path_distance = INVALID_EDGE_WEIGHT; NodeID middle_node = SPECIAL_NODEID; const EdgeWeight min_edge_offset = - std::min(phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), - phantom_node_pair.source_phantom.GetReverseWeightPlusOffset()); + std::min(-phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), + -phantom_node_pair.source_phantom.GetReverseWeightPlusOffset()); if (phantom_node_pair.source_phantom.forward_node_id != SPECIAL_NODEID) { diff --git a/routing_algorithms/shortest_path.hpp b/routing_algorithms/shortest_path.hpp index 9e6b89045..eaa6d3c53 100644 --- a/routing_algorithms/shortest_path.hpp +++ b/routing_algorithms/shortest_path.hpp @@ -93,8 +93,8 @@ class ShortestPathRouting final const bool allow_u_turn = current_leg > 0 && uturn_indicators.size() > current_leg && uturn_indicators[current_leg - 1]; const EdgeWeight min_edge_offset = - std::min(phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), - phantom_node_pair.source_phantom.GetReverseWeightPlusOffset()); + std::min(-phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), + -phantom_node_pair.source_phantom.GetReverseWeightPlusOffset()); // insert new starting nodes into forward heap, adjusted by previous distances. if ((allow_u_turn || search_from_1st_node) && From ebff45f8037cefb76859e664b43579af92c63e9d Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 18 Jun 2015 00:18:04 +0200 Subject: [PATCH 79/87] Add new regression test for looping bugs --- features/testbot/loop.feature | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/features/testbot/loop.feature b/features/testbot/loop.feature index 72cad33c0..c69e7a251 100644 --- a/features/testbot/loop.feature +++ b/features/testbot/loop.feature @@ -4,6 +4,25 @@ Feature: Avoid weird loops caused by rounding errors Background: Given the profile "testbot" + Scenario: Weired sidestreet loops + Given the node map + | a | 1 | b | 2 | c | 3 | d | + | | | | | | | | + | e | | f | | g | | h | + + And the ways + | nodes | + | aefghd | + | abcd | + | bf | + | cg | + + When I route I should get + | waypoints | route | turns | + | a,1,d | abcd,abcd | head,via,destination | + | a,2,d | abcd,abcd | head,via,destination | + | a,3,d | abcd,abcd | head,via,destination | + Scenario: Avoid weird loops 1 Given the node locations | node | lat | lon | From 86df55f5cca89baf1bfbbe52918828124fd43b7b Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 18 Jun 2015 17:31:49 +0200 Subject: [PATCH 80/87] Add penalty if there is only one lane for both directions --- profiles/car.lua | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/profiles/car.lua b/profiles/car.lua index 912d98cac..8c68bc994 100644 --- a/profiles/car.lua +++ b/profiles/car.lua @@ -241,12 +241,6 @@ function way_function (way, result) return end - local width = math.huge - local width_string = way:get_value_by_key("width") - if width_string and tonumber(width_string:match("%d*")) then - width = tonumber(width_string:match("%d*")) - end - -- Check if we are allowed to access the way local access = find_access_tag(way, access_tags_hierachy) if access_tag_blacklist[access] then @@ -404,12 +398,25 @@ function way_function (way, result) result.ignore_in_grid = true end + local width = math.huge + local lanes = math.huge + if result.forward_speed > 0 or result.backward_speed > 0 then + local width_string = way:get_value_by_key("width") + if width_string and tonumber(width_string:match("%d*")) then + width = tonumber(width_string:match("%d*")) + end + + local lanes_string = way:get_value_by_key("lanes") + if lanes_string and tonumber(lanes_string:match("%d*")) then + lanes = tonumber(lanes_string:match("%d*")) + end + end -- scale speeds to get better avg driving times if result.forward_speed > 0 then local scaled_speed = result.forward_speed*speed_reduction + 11; local penalized_speed = math.huge - if width <= 3 then + if width <= 3 or lanes <= 1 then penalized_speed = result.forward_speed / 2; end result.forward_speed = math.min(penalized_speed, scaled_speed) @@ -418,7 +425,7 @@ function way_function (way, result) if result.backward_speed > 0 then local scaled_speed = result.backward_speed*speed_reduction + 11; local penalized_speed = math.huge - if width <= 3 then + if width <= 3 or lanes <= 1 then penalized_speed = result.backward_speed / 2; end result.backward_speed = math.min(penalized_speed, scaled_speed) From 1445f11c19ab76827a85218765fb004a538e5463 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 18 Jun 2015 17:35:39 +0200 Subject: [PATCH 81/87] Add test for lane penalty --- features/car/maxspeed.feature | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/features/car/maxspeed.feature b/features/car/maxspeed.feature index 781f4c617..84cfa4ff8 100644 --- a/features/car/maxspeed.feature +++ b/features/car/maxspeed.feature @@ -93,3 +93,22 @@ OSRM will use 4/5 of the projected free-flow speed. | primary | 15 | | 30 | 60 | 34 km/h | 59 km/h | | primary | 15 | 3 | 30 | 60 | 15 km/h | 30 km/h | + Scenario: Car - Single lane streets be ignored or incur a penalty + Then routability should be + + | highway | maxspeed | lanes | maxspeed:forward | maxspeed:backward | forw | backw | + | primary | | | | | 63 km/h | 63 km/h | + | primary | | 1 | | | 32 km/h | 32 km/h | + | primary | 60 | | | | 59 km/h | 59 km/h | + | primary | 60 | 1 | | | 30 km/h | 30 km/h | + | primary | | | 60 | | 59 km/h | 63 km/h | + | primary | | 1 | 60 | | 30 km/h | 32 km/h | + | primary | | | | 60 | 63 km/h | 59 km/h | + | primary | | 1 | | 60 | 32 km/h | 30 km/h | + | primary | 15 | | 60 | | 59 km/h | 23 km/h | + | primary | 15 | 1 | 60 | | 30 km/h | 7 km/h | + | primary | 15 | | | 60 | 23 km/h | 59 km/h | + | primary | 15 | 1 | | 60 | 7 km/h | 30 km/h | + | primary | 15 | | 30 | 60 | 34 km/h | 59 km/h | + | primary | 15 | 1 | 30 | 60 | 15 km/h | 30 km/h | + From da38a1367aebebf11af345abcd607183b85a8819 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Thu, 11 Jun 2015 14:17:16 -0700 Subject: [PATCH 82/87] try building with vs2015 as CTP_Nov2013 appears broken now on appveyor --- appveyor.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index afb8dd406..38d07d0e0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,16 +3,12 @@ environment: - configuration: Debug - configuration: Release -# branches to build -branches: - # whitelist - only: - - develop - # scripts that are called at very beginning, before repo cloning init: - git config --global core.autocrlf input +os: Visual Studio 2015 Preview + # clone directory clone_folder: c:\projects\osrm @@ -35,9 +31,11 @@ build_script: - SET P=c:/projects/osrm - set TBB_INSTALL_DIR=%P%/tbb - set TBB_ARCH_PLATFORM=intel64/vc12 - - cmake .. -G "Visual Studio 12 Win64" -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_INSTALL_PREFIX=%P%/libs -DBOOST_ROOT=%P%/boost_min -DBoost_ADDITIONAL_VERSIONS=1.57 -DBoost_USE_STATIC_LIBS=ON -T CTP_Nov2013 - - msbuild /clp:Verbosity=minimal /nologo OSRM.sln - - msbuild /clp:Verbosity=minimal /nologo tests.vcxproj + - cmake .. -G "Visual Studio 14 Win64" -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_INSTALL_PREFIX=%P%/libs -DBOOST_ROOT=%P%/boost_min -DBoost_ADDITIONAL_VERSIONS=1.57 -DBoost_USE_STATIC_LIBS=ON + - SET PLATFORM_TOOLSET=v140 + - SET TOOLS_VERSION=14.0 + - msbuild /p:Platform=x64 /clp:Verbosity=minimal /toolsversion:%TOOLS_VERSION% /p:PlatformToolset=%PLATFORM_TOOLSET% /nologo OSRM.sln + - msbuild /p:Platform=x64 /clp:Verbosity=minimal /toolsversion:%TOOLS_VERSION% /p:PlatformToolset=%PLATFORM_TOOLSET% /nologo tests.vcxproj - cd %Configuration% - if "%APPVEYOR_REPO_BRANCH%"=="develop" (7z a %P%/osrm_%Configuration%.zip *.exe *.pdb %P%/libs/bin/*.dll -tzip) - cd ..\..\profiles From a5fd7cf4e9928b6c91398c04f250b84658490c7a Mon Sep 17 00:00:00 2001 From: bergwerkgis Date: Tue, 16 Jun 2015 21:15:24 +0200 Subject: [PATCH 83/87] make AppVeyor green again --- .gitignore | 4 ++ appveyor-build.bat | 113 +++++++++++++++++++++++++++++++++++++++++++++ appveyor.yml | 39 +++------------- build-local.bat | 19 ++++++++ 4 files changed, 143 insertions(+), 32 deletions(-) create mode 100644 appveyor-build.bat create mode 100644 build-local.bat diff --git a/.gitignore b/.gitignore index 6ed5d2a4e..6264b4294 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# pre compiled dependencies # +############################# +osrm-deps + # Compiled source # ################### *.com diff --git a/appveyor-build.bat b/appveyor-build.bat new file mode 100644 index 000000000..eef29b1ba --- /dev/null +++ b/appveyor-build.bat @@ -0,0 +1,113 @@ +@ECHO OFF +SETLOCAL +SET EL=0 + +ECHO platform^: %platform% +:: HARDCODE "x64" as it is uppercase on AppVeyor and download from S3 is case sensitive +SET DEPSPKG=osrm-deps-win-x64-14.0.7z + +:: local development +IF "%computername%"=="MB" GOTO SKIPDL + +IF EXIST %DEPSPKG% DEL %DEPSPKG% +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +ECHO downloading %DEPSPKG% +powershell Invoke-WebRequest https://mapbox.s3.amazonaws.com/windows-builds/windows-deps/$env:DEPSPKG -OutFile C:\projects\osrm\$env:DEPSPKG +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +7z -y x %DEPSPKG% | %windir%\system32\FIND "ing archive" +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +:SKIPDL + +IF EXIST build rd /s /q build +IF %ERRORLEVEL% NEQ 0 GOTO ERROR +mkdir build +IF %ERRORLEVEL% NEQ 0 GOTO ERROR +cd build +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +SET OSRMDEPSDIR=c:\projects\osrm\osrm-deps +set PREFIX=%OSRMDEPSDIR%/libs +set BOOST_ROOT=%OSRMDEPSDIR%/boost +set TBB_INSTALL_DIR=%OSRMDEPSDIR%/tbb +set TBB_ARCH_PLATFORM=intel64/vc14 + +ECHO calling cmake .... +cmake .. ^ +-G "Visual Studio 14 Win64" ^ +-DBOOST_ROOT=%BOOST_ROOT% ^ +-DBoost_ADDITIONAL_VERSIONS=1.57 ^ +-DBoost_USE_MULTITHREADED=ON ^ +-DBoost_USE_STATIC_LIBS=ON ^ +-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ +-DCMAKE_INSTALL_PREFIX=%PREFIX% +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +ECHO building ... +msbuild OSRM.sln ^ +/p:Configuration=%Configuration% ^ +/p:Platform=x64 ^ +/t:rebuild ^ +/p:BuildInParallel=true ^ +/m:%NUMBER_OF_PROCESSORS% ^ +/toolsversion:14.0 ^ +/p:PlatformToolset=v140 ^ +/clp:Verbosity=normal ^ +/nologo ^ +/flp1:logfile=build_errors.txt;errorsonly ^ +/flp2:logfile=build_warnings.txt;warningsonly +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +ECHO ========= TODO^: CREATE PACKAGES ========== + +CD c:\projects\osrm\build\%Configuration% +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +SET PATH=c:\projects\osrm\osrm-deps\libs\bin;%PATH% + +ECHO running datastructure-tests.exe ... +datastructure-tests.exe +IF %ERRORLEVEL% NEQ 0 GOTO ERROR +ECHO running algorithm-tests.exe ... +algorithm-tests.exe +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +GOTO DONE + +:ERROR +SET EL=%ERRORLEVEL% +ECHO ============== ERROR =============== + +:DONE +ECHO ============= DONE =============== +CD C:\projects\osrm +EXIT /b %EL% + + + + + - cd c:/projects/osrm + - mkdir build + - cd build + - echo Running cmake... + - call "%VS120COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64 + - SET PATH=C:\Program Files (x86)\MSBuild\12.0\bin\;%PATH% + - SET P=c:/projects/osrm + - set TBB_INSTALL_DIR=%P%/tbb + - set TBB_ARCH_PLATFORM=intel64/vc12 + - cmake .. -G "Visual Studio 14 Win64" -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_INSTALL_PREFIX=%P%/libs -DBOOST_ROOT=%P%/boost_min -DBoost_ADDITIONAL_VERSIONS=1.57 -DBoost_USE_STATIC_LIBS=ON + - SET PLATFORM_TOOLSET=v140 + - SET TOOLS_VERSION=14.0 + - msbuild /p:Platform=x64 /clp:Verbosity=minimal /toolsversion:%TOOLS_VERSION% /p:PlatformToolset=%PLATFORM_TOOLSET% /nologo OSRM.sln + - msbuild /p:Platform=x64 /clp:Verbosity=minimal /toolsversion:%TOOLS_VERSION% /p:PlatformToolset=%PLATFORM_TOOLSET% /nologo tests.vcxproj + - cd %Configuration% + - if "%APPVEYOR_REPO_BRANCH%"=="develop" (7z a %P%/osrm_%Configuration%.zip *.exe *.pdb %P%/libs/bin/*.dll -tzip) + - cd ..\..\profiles + - echo disk=c:\temp\stxxl,10000,wincall > .stxxl.txt + - if "%APPVEYOR_REPO_BRANCH%"=="develop" (7z a %P%/osrm_%Configuration%.zip * -tzip) + - set PATH=%PATH%;c:/projects/osrm/libs/bin + - cd c:/projects/osrm/build/%Configuration% + - datastructure-tests.exe + - algorithm-tests.exe diff --git a/appveyor.yml b/appveyor.yml index 38d07d0e0..cf45a3224 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,13 +1,13 @@ environment: matrix: - - configuration: Debug - configuration: Release +# - configuration: Debug # scripts that are called at very beginning, before repo cloning init: - git config --global core.autocrlf input -os: Visual Studio 2015 Preview +os: Visual Studio 2015 RC # clone directory clone_folder: c:\projects\osrm @@ -15,44 +15,19 @@ clone_folder: c:\projects\osrm platform: x64 install: - # by default, all script lines are interpreted as batch - - nuget install protobuf - - cd c:\projects\osrm - - curl -O http://build.project-osrm.org/libs_osrm_%Configuration%.7z - - 7z x libs_osrm_%Configuration%.7z | find ":" + - set PATH=C:\Program Files (x86)\MSBuild\14.0\Bin;%PATH% + - CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 build_script: - - cd c:/projects/osrm - - mkdir build - - cd build - - echo Running cmake... - - call "%VS120COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64 - - SET PATH=C:\Program Files (x86)\MSBuild\12.0\bin\;%PATH% - - SET P=c:/projects/osrm - - set TBB_INSTALL_DIR=%P%/tbb - - set TBB_ARCH_PLATFORM=intel64/vc12 - - cmake .. -G "Visual Studio 14 Win64" -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_INSTALL_PREFIX=%P%/libs -DBOOST_ROOT=%P%/boost_min -DBoost_ADDITIONAL_VERSIONS=1.57 -DBoost_USE_STATIC_LIBS=ON - - SET PLATFORM_TOOLSET=v140 - - SET TOOLS_VERSION=14.0 - - msbuild /p:Platform=x64 /clp:Verbosity=minimal /toolsversion:%TOOLS_VERSION% /p:PlatformToolset=%PLATFORM_TOOLSET% /nologo OSRM.sln - - msbuild /p:Platform=x64 /clp:Verbosity=minimal /toolsversion:%TOOLS_VERSION% /p:PlatformToolset=%PLATFORM_TOOLSET% /nologo tests.vcxproj - - cd %Configuration% - - if "%APPVEYOR_REPO_BRANCH%"=="develop" (7z a %P%/osrm_%Configuration%.zip *.exe *.pdb %P%/libs/bin/*.dll -tzip) - - cd ..\..\profiles - - echo disk=c:\temp\stxxl,10000,wincall > .stxxl.txt - - if "%APPVEYOR_REPO_BRANCH%"=="develop" (7z a %P%/osrm_%Configuration%.zip * -tzip) - - set PATH=%PATH%;c:/projects/osrm/libs/bin - - cd c:/projects/osrm/build/%Configuration% - - datastructure-tests.exe - - algorithm-tests.exe + - CALL appveyor-build.bat test: off artifacts: - - path: osrm_Debug.zip - name: osrm_Debug.zip - path: osrm_Release.zip name: osrm_Release.zip +# - path: osrm_Debug.zip +# name: osrm_Debug.zip deploy: provider: FTP diff --git a/build-local.bat b/build-local.bat new file mode 100644 index 000000000..47a1fb3a6 --- /dev/null +++ b/build-local.bat @@ -0,0 +1,19 @@ +@ECHO OFF + +SET PLATFORM=x64 +SET CONFIGURATION=Release + +WHERE msbuild +IF %ERRORLEVEL% EQU 0 GOTO RUNBUILD + +SET PATH=C:\mb\windows-builds-64\tmp-bin\cmake-3.1.0-win32-x86\bin;%PATH% +SET PATH=C:\Program Files\7-Zip;%PATH% +ECHO activating VS command prompt ... +SET PATH=C:\Program Files (x86)\MSBuild\14.0\Bin;%PATH% +CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 + +:RUNBUILD + +powershell Set-ExecutionPolicy -Scope CurrentUser -ExecutionPolicy Unrestricted -Force +CALL appveyor-build.bat +EXIT /b %ERRORLEVEL% From 5fc0d284cb15c935eaa8627efc0d52d7b49bd099 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 19 Jun 2015 00:33:56 +0200 Subject: [PATCH 84/87] Revert "Simplify offeset calculation logic a little bit" This reverts commit 8ade26b4a4ff2a91c75542629c065da843e794fd. One of the assertions triggers when run on an extract of Serok. Since this code does not fix any bugs, I'll just revert this for now. This definetly needs investigation. --- contractor/edge_based_graph_factory.cpp | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index e76f7be52..dbd73a73c 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -128,11 +128,6 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, // TODO: move to lambda function with C++11 int temp_sum = 0; - // u-----------------------------------v - // 0 1 i-1 i n-1 - // x --> x --> ... --> x --> x ... --> x - // |-------------------| - // forward_dist_prefix_sum[i] for (const auto i : osrm::irange(0u, geometry_size)) { forward_dist_prefix_sum[i] = temp_sum; @@ -140,22 +135,14 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, BOOST_ASSERT(forward_data.distance >= temp_sum); } - BOOST_ASSERT(forward_dist_prefix_sum.back() + forward_geometry.back().second == forward_data.distance); - // v-----------------------------------u - // 0 1 i-1 i n-1 - // x --> x --> ... --> x --> x ... --> x - // |-------------------| - // reverse_dist_prefix_sum[i] temp_sum = 0; for (const auto i : osrm::irange(0u, geometry_size)) { - reverse_dist_prefix_sum[i] = temp_sum; - temp_sum += reverse_geometry[i].second; - - BOOST_ASSERT(reverse_data.distance >= temp_sum); + temp_sum += reverse_geometry[reverse_geometry.size() - 1 - i].second; + reverse_dist_prefix_sum[i] = reverse_data.distance - temp_sum; + // BOOST_ASSERT(reverse_data.distance >= temp_sum); } - BOOST_ASSERT(reverse_dist_prefix_sum.back() + reverse_geometry.back().second == reverse_data.distance); NodeID current_edge_source_coordinate_id = node_u; @@ -181,10 +168,8 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, forward_data.edgeBasedNodeID, reverse_data.edgeBasedNodeID, current_edge_source_coordinate_id, current_edge_target_coordinate_id, forward_data.nameID, forward_geometry[i].second, - reverse_geometry[geometry_size - 1 - i].second, - forward_dist_prefix_sum[i], - reverse_dist_prefix_sum[geometry_size - 1 - i], - m_geometry_compressor.GetPositionForID(edge_id_1), + reverse_geometry[geometry_size - 1 - i].second, forward_dist_prefix_sum[i], + reverse_dist_prefix_sum[i], m_geometry_compressor.GetPositionForID(edge_id_1), component_id, i, forward_data.travel_mode, reverse_data.travel_mode); current_edge_source_coordinate_id = current_edge_target_coordinate_id; From 94b749ab00106e7050751c06efff52dc3bde0eee Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 18 Jun 2015 18:34:25 +0200 Subject: [PATCH 85/87] Fix magic number check for fingerprint --- contractor/processing_chain.cpp | 4 +-- contractor/processing_chain.hpp | 1 - datastore.cpp | 5 ++-- extractor/extraction_containers.cpp | 3 ++ extractor/extraction_containers.hpp | 2 -- util/fingerprint.hpp | 8 ++---- util/fingerprint_impl.hpp.in | 43 +++++++++++++++-------------- util/graph_loader.hpp | 13 +++++---- 8 files changed, 41 insertions(+), 38 deletions(-) diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index b354a62b3..a76bc657f 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -141,9 +141,9 @@ std::size_t Prepare::WriteContractedGraph(unsigned number_of_edge_based_nodes, SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count << " edges"; - FingerPrint fingerprint_orig; + const FingerPrint fingerprint = FingerPrint::GetValid(); boost::filesystem::ofstream hsgr_output_stream(config.graph_output_path, std::ios::binary); - hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint)); + hsgr_output_stream.write((char *)&fingerprint, sizeof(FingerPrint)); const unsigned max_used_node_id = 1 + [&contracted_edge_list] { unsigned tmp_max = 0; diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index 12f74f3fb..281768166 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -33,7 +33,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../data_structures/query_edge.hpp" #include "../data_structures/static_graph.hpp" -class FingerPrint; struct EdgeBasedNode; struct lua_State; diff --git a/datastore.cpp b/datastore.cpp index 3c032a2a9..89efefcb2 100644 --- a/datastore.cpp +++ b/datastore.cpp @@ -264,9 +264,10 @@ int main(const int argc, const char *argv[]) boost::filesystem::ifstream hsgr_input_stream(hsgr_path, std::ios::binary); - FingerPrint fingerprint_loaded, fingerprint_orig; + FingerPrint fingerprint_valid = FingerPrint::GetValid(); + FingerPrint fingerprint_loaded; hsgr_input_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint)); - if (fingerprint_loaded.TestGraphUtil(fingerprint_orig)) + if (fingerprint_loaded.TestGraphUtil(fingerprint_valid)) { SimpleLogger().Write(logDEBUG) << "Fingerprint checked out ok"; } diff --git a/extractor/extraction_containers.cpp b/extractor/extraction_containers.cpp index 7097993f8..caca05ccb 100644 --- a/extractor/extraction_containers.cpp +++ b/extractor/extraction_containers.cpp @@ -35,6 +35,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../util/osrm_exception.hpp" #include "../util/simple_logger.hpp" #include "../util/timing_util.hpp" +#include "../util/fingerprint.hpp" #include #include @@ -81,6 +82,7 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name, { std::ofstream file_out_stream; file_out_stream.open(output_file_name.c_str(), std::ios::binary); + const FingerPrint fingerprint = FingerPrint::GetValid(); file_out_stream.write((char *)&fingerprint, sizeof(FingerPrint)); PrepareNodes(); @@ -471,6 +473,7 @@ void ExtractionContainers::WriteRestrictions(const std::string& path) const std::ofstream restrictions_out_stream; unsigned written_restriction_count = 0; restrictions_out_stream.open(path.c_str(), std::ios::binary); + const FingerPrint fingerprint = FingerPrint::GetValid(); restrictions_out_stream.write((char *)&fingerprint, sizeof(FingerPrint)); const auto count_position = restrictions_out_stream.tellp(); restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned)); diff --git a/extractor/extraction_containers.hpp b/extractor/extraction_containers.hpp index 444f04f02..ef71d059f 100644 --- a/extractor/extraction_containers.hpp +++ b/extractor/extraction_containers.hpp @@ -32,7 +32,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "first_and_last_segment_of_way.hpp" #include "../data_structures/external_memory_node.hpp" #include "../data_structures/restriction.hpp" -#include "../util/fingerprint.hpp" #include #include @@ -75,7 +74,6 @@ class ExtractionContainers STXXLRestrictionsVector restrictions_list; STXXLWayIDStartEndVector way_start_end_id_list; std::unordered_map external_to_internal_node_id_map; - const FingerPrint fingerprint; ExtractionContainers(); diff --git a/util/fingerprint.hpp b/util/fingerprint.hpp index 82eb1364a..5b64ed648 100644 --- a/util/fingerprint.hpp +++ b/util/fingerprint.hpp @@ -34,18 +34,16 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. class FingerPrint { public: - FingerPrint(); - FingerPrint(const FingerPrint &) = delete; - ~FingerPrint(); + static FingerPrint GetValid(); const boost::uuids::uuid &GetFingerPrint() const; - bool IsMagicNumberOK() const; + bool IsMagicNumberOK(const FingerPrint &other) const; bool TestGraphUtil(const FingerPrint &other) const; bool TestPrepare(const FingerPrint &other) const; bool TestRTree(const FingerPrint &other) const; bool TestQueryObjects(const FingerPrint &other) const; private: - const unsigned magic_number; + unsigned magic_number; char md5_prepare[33]; char md5_tree[33]; char md5_graph[33]; diff --git a/util/fingerprint_impl.hpp.in b/util/fingerprint_impl.hpp.in index 9259afd37..98bb122d3 100644 --- a/util/fingerprint_impl.hpp.in +++ b/util/fingerprint_impl.hpp.in @@ -40,35 +40,38 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #cmakedefine MD5GRAPH "${MD5GRAPH}" #cmakedefine MD5OBJECTS "${MD5OBJECTS}" -FingerPrint::FingerPrint() : magic_number(1297240911) +FingerPrint FingerPrint::GetValid() { - md5_prepare[32] = md5_tree[32] = md5_graph[32] = md5_objects[32] = '\0'; + FingerPrint fingerprint; - boost::uuids::name_generator gen(named_uuid); + fingerprint.magic_number = 1297240911; + fingerprint.md5_prepare[32] = fingerprint.md5_tree[32] = fingerprint.md5_graph[32] = fingerprint.md5_objects[32] = '\0'; + + boost::uuids::name_generator gen(fingerprint.named_uuid); std::string temp_string; - std::memcpy(md5_prepare, MD5PREPARE, strlen(MD5PREPARE)); - temp_string += md5_prepare; - std::memcpy(md5_tree, MD5RTREE, 32); - temp_string += md5_tree; - std::memcpy(md5_graph, MD5GRAPH, 32); - temp_string += md5_graph; - std::memcpy(md5_objects, MD5OBJECTS, 32); - temp_string += md5_objects; + std::memcpy(fingerprint.md5_prepare, MD5PREPARE, 32); + temp_string += fingerprint.md5_prepare; + std::memcpy(fingerprint.md5_tree, MD5RTREE, 32); + temp_string += fingerprint.md5_tree; + std::memcpy(fingerprint.md5_graph, MD5GRAPH, 32); + temp_string += fingerprint.md5_graph; + std::memcpy(fingerprint.md5_objects, MD5OBJECTS, 32); + temp_string += fingerprint.md5_objects; - named_uuid = gen(temp_string); - has_64_bits = HAS64BITS; + fingerprint.named_uuid = gen(temp_string); + fingerprint.has_64_bits = HAS64BITS; + + return fingerprint; } -FingerPrint::~FingerPrint() {} - const boost::uuids::uuid &FingerPrint::GetFingerPrint() const { return named_uuid; } -bool FingerPrint::IsMagicNumberOK() const { return 1297240911 == magic_number; } +bool FingerPrint::IsMagicNumberOK(const FingerPrint& other) const { return other.magic_number == magic_number; } bool FingerPrint::TestGraphUtil(const FingerPrint &other) const { - if (!other.IsMagicNumberOK()) + if (!IsMagicNumberOK(other)) { throw osrm::exception("hsgr input file misses magic number. Check or reprocess the file"); } @@ -77,7 +80,7 @@ bool FingerPrint::TestGraphUtil(const FingerPrint &other) const bool FingerPrint::TestPrepare(const FingerPrint &other) const { - if (!other.IsMagicNumberOK()) + if (!IsMagicNumberOK(other)) { throw osrm::exception("osrm input file misses magic number. Check or reprocess the file"); } @@ -86,7 +89,7 @@ bool FingerPrint::TestPrepare(const FingerPrint &other) const bool FingerPrint::TestRTree(const FingerPrint &other) const { - if (!other.IsMagicNumberOK()) + if (!IsMagicNumberOK(other)) { throw osrm::exception("r-tree input file misses magic number. Check or reprocess the file"); } @@ -95,7 +98,7 @@ bool FingerPrint::TestRTree(const FingerPrint &other) const bool FingerPrint::TestQueryObjects(const FingerPrint &other) const { - if (!other.IsMagicNumberOK()) + if (!IsMagicNumberOK(other)) { throw osrm::exception("missing magic number. Check or reprocess the file"); } diff --git a/util/graph_loader.hpp b/util/graph_loader.hpp index 2243538f3..71bc9e9f3 100644 --- a/util/graph_loader.hpp +++ b/util/graph_loader.hpp @@ -60,11 +60,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. unsigned loadRestrictionsFromFile(std::istream& input_stream, std::vector& restriction_list) { - const FingerPrint fingerprint_orig; + const FingerPrint fingerprint_valid = FingerPrint::GetValid(); FingerPrint fingerprint_loaded; unsigned number_of_usable_restrictions = 0; input_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint)); - if (!fingerprint_loaded.TestPrepare(fingerprint_orig)) + if (!fingerprint_loaded.TestPrepare(fingerprint_valid)) { SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n" "Reprocess to get rid of this warning."; @@ -93,11 +93,11 @@ NodeID loadNodesFromFile(std::istream &input_stream, std::vector &traffic_light_node_list, std::vector &node_array) { - const FingerPrint fingerprint_orig; + const FingerPrint fingerprint_valid = FingerPrint::GetValid(); FingerPrint fingerprint_loaded; input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(FingerPrint)); - if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) + if (!fingerprint_loaded.TestPrepare(fingerprint_valid)) { SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n" "Reprocess to get rid of this warning."; @@ -187,9 +187,10 @@ unsigned readHSGRFromStream(const boost::filesystem::path &hsgr_file, boost::filesystem::ifstream hsgr_input_stream(hsgr_file, std::ios::binary); - FingerPrint fingerprint_loaded, fingerprint_orig; + const FingerPrint fingerprint_valid = FingerPrint::GetValid(); + FingerPrint fingerprint_loaded; hsgr_input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(FingerPrint)); - if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) + if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid)) { SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build.\n" "Reprocess to get rid of this warning."; From a17776cb5fbdc6daabd692ead7f544b2016ca083 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 19 Jun 2015 17:48:27 +0200 Subject: [PATCH 86/87] Check if FingerPrint is trivial. TODO: Add this for all other data that is going to be serialized --- util/fingerprint.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/util/fingerprint.hpp b/util/fingerprint.hpp index 5b64ed648..3894fc80f 100644 --- a/util/fingerprint.hpp +++ b/util/fingerprint.hpp @@ -29,6 +29,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define FINGERPRINT_H #include +#include // implements a singleton, i.e. there is one and only one conviguration object class FingerPrint @@ -52,6 +53,10 @@ class FingerPrint // initialize to {6ba7b810-9dad-11d1-80b4-00c04fd430c8} boost::uuids::uuid named_uuid; bool has_64_bits; + }; +static_assert(std::is_trivial::value, "FingerPrint needs to be trivial."); + + #endif /* FingerPrint_H */ From 1cb72acd27d91a2fd2a920e3d9e9ea10ad3d7784 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 22 Jun 2015 08:36:21 +0200 Subject: [PATCH 87/87] Remove unused header --- contractor/contractor_options.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/contractor/contractor_options.cpp b/contractor/contractor_options.cpp index 175613f03..eaa1ba997 100644 --- a/contractor/contractor_options.cpp +++ b/contractor/contractor_options.cpp @@ -28,7 +28,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "contractor_options.hpp" #include "../util/git_sha.hpp" -#include "../util/ini_file.hpp" #include "../util/simple_logger.hpp" #include