diff --git a/features/car/traffic.feature b/features/car/traffic.feature new file mode 100644 index 000000000..6d1a5d4e0 --- /dev/null +++ b/features/car/traffic.feature @@ -0,0 +1,47 @@ +@routing @speed @traffic +Feature: Traffic - speeds + + Background: Use specific speeds + Given the node locations + | node | lat | lon | + | a | 0.1 | 0.1 | + | b | .05 | 0.1 | + | c | 0.0 | 0.1 | + | d | .05 | .03 | + | e | .05 | .066 | + | f | .075 | .066 | + | g | .075 | 0.1 | + And the ways + | nodes | highway | + | ab | primary | + | ad | primary | + | bc | primary | + | dc | primary | + | de | primary | + | eb | primary | + | df | primary | + | fb | primary | + And the speed file + """ + 1,2,27 + 2,1,27 + 2,3,27 + 3,2,27 + 1,4,27 + 4,1,27 + """ + + Scenario: Weighting not based on raster sources + Given the profile "testbot" + Given the extract extra arguments "--generate-edge-lookup" + Given the prepare extra arguments "--segment-speed-file speeds.csv" + And I route I should get + | from | to | route | speed | + | a | b | ab | 27 km/h | + | a | c | ab,bc | 27 km/h | + | b | c | bc | 27 km/h | + | a | d | ad | 27 km/h | + | d | c | dc | 36 km/h | + | g | b | ab | 27 km/h | + | a | g | ab | 27 km/h | + diff --git a/features/step_definitions/data.rb b/features/step_definitions/data.rb index 3e4ccc91c..6d6172bc9 100644 --- a/features/step_definitions/data.rb +++ b/features/step_definitions/data.rb @@ -10,6 +10,10 @@ Given /^the extract extra arguments "(.*?)"$/ do |args| set_extract_args args end +Given /^the prepare extra arguments "(.*?)"$/ do |args| + set_prepare_args args +end + Given /^a grid size of (\d+) meters$/ do |meters| set_grid_size meters end @@ -146,6 +150,12 @@ Given /^the raster source$/ do |data| end end +Given /^the speed file$/ do |data| + Dir.chdir TEST_FOLDER do + File.open("speeds.csv", "w") {|f| f.write(data)} + end +end + Given /^the data has been saved to disk$/ do begin write_input_data diff --git a/features/support/config.rb b/features/support/config.rb index 434b4ec9b..5730baf2b 100644 --- a/features/support/config.rb +++ b/features/support/config.rb @@ -14,3 +14,7 @@ end def set_extract_args args @extract_args = args end + +def set_prepare_args args + @prepare_args = args +end diff --git a/features/support/data.rb b/features/support/data.rb index c0f46967e..f23c6d8aa 100644 --- a/features/support/data.rb +++ b/features/support/data.rb @@ -258,6 +258,7 @@ def extract_data Dir.chdir TEST_FOLDER do log_preprocess_info log "== Extracting #{osm_file}.osm...", :preprocess + log "#{LOAD_LIBRARIES}#{BIN_PATH}/osrm-extract #{osm_file}.osm #{@extract_args} --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1" unless system "#{LOAD_LIBRARIES}#{BIN_PATH}/osrm-extract #{osm_file}.osm #{@extract_args} --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1" log "*** Exited with code #{$?.exitstatus}.", :preprocess raise ExtractError.new $?.exitstatus, "osrm-extract exited with code #{$?.exitstatus}." @@ -270,6 +271,16 @@ def extract_data rescue Exception => e raise FileError.new nil, "failed to rename data file after extracting." end + begin + ["osrm.edge_segment_lookup","osrm.edge_penalties"].each do |file| + if File.exists?("#{osm_file}.#{file}") + log "Renaming #{osm_file}.#{file} to #{extracted_file}.#{file}", :preprocess + File.rename "#{osm_file}.#{file}", "#{extracted_file}.#{file}" + end + end + rescue Exception => e + raise FileError.new nil, "failed to rename data file after extracting." + end end end @@ -277,7 +288,8 @@ def prepare_data Dir.chdir TEST_FOLDER do log_preprocess_info log "== Preparing #{extracted_file}.osm...", :preprocess - unless system "#{LOAD_LIBRARIES}#{BIN_PATH}/osrm-contract #{extracted_file}.osrm >>#{PREPROCESS_LOG_FILE} 2>&1" + log "#{LOAD_LIBRARIES}#{BIN_PATH}/osrm-contract #{@prepare_args} #{extracted_file}.osrm >>#{PREPROCESS_LOG_FILE} 2>&1" + unless system "#{LOAD_LIBRARIES}#{BIN_PATH}/osrm-contract #{@prepare_args} #{extracted_file}.osrm >>#{PREPROCESS_LOG_FILE} 2>&1" log "*** Exited with code #{$?.exitstatus}.", :preprocess raise PrepareError.new $?.exitstatus, "osrm-contract exited with code #{$?.exitstatus}." end diff --git a/include/contractor/contractor.hpp b/include/contractor/contractor.hpp index 64d3c0491..4398da894 100644 --- a/include/contractor/contractor.hpp +++ b/include/contractor/contractor.hpp @@ -7,6 +7,7 @@ #include "contractor/contractor_config.hpp" #include "contractor/query_edge.hpp" #include "extractor/edge_based_edge.hpp" +#include "extractor/edge_based_node.hpp" #include "util/static_graph.hpp" #include "util/deallocating_vector.hpp" #include "util/node_based_graph.hpp" @@ -22,7 +23,6 @@ namespace osrm namespace extractor { struct SpeedProfileProperties; -struct EdgeBasedNode; struct EdgeBasedEdge; } namespace contractor @@ -66,7 +66,10 @@ class Contractor util::DeallocatingVector &edge_based_edge_list, const std::string &edge_segment_lookup_path, const std::string &edge_penalty_path, - const std::string &segment_speed_path); + const std::string &segment_speed_path, + const std::string &nodes_filename, + const std::string &geometry_filename, + const std::string &rtree_leaf_filename); }; } } diff --git a/include/contractor/contractor_config.hpp b/include/contractor/contractor_config.hpp index 846a99e55..be9e620bd 100644 --- a/include/contractor/contractor_config.hpp +++ b/include/contractor/contractor_config.hpp @@ -24,6 +24,8 @@ struct ContractorConfig edge_segment_lookup_path = osrm_input_path.string() + ".edge_segment_lookup"; edge_penalty_path = osrm_input_path.string() + ".edge_penalties"; node_based_graph_path = osrm_input_path.string() + ".nodes"; + geometry_path = osrm_input_path.string() + ".geometry"; + rtree_leaf_path = osrm_input_path.string() + ".fileIndex"; } boost::filesystem::path config_file_path; @@ -37,6 +39,8 @@ struct ContractorConfig std::string edge_segment_lookup_path; std::string edge_penalty_path; std::string node_based_graph_path; + std::string geometry_path; + std::string rtree_leaf_path; bool use_cached_priority; unsigned requested_num_threads; diff --git a/include/engine/datafacade/datafacade_base.hpp b/include/engine/datafacade/datafacade_base.hpp index 6c0fa6c57..231c2a019 100644 --- a/include/engine/datafacade/datafacade_base.hpp +++ b/include/engine/datafacade/datafacade_base.hpp @@ -69,8 +69,13 @@ template class BaseDataFacade virtual unsigned GetGeometryIndexForEdgeID(const unsigned id) const = 0; - virtual void GetUncompressedGeometry(const unsigned id, - std::vector &result_nodes) const = 0; + virtual void GetUncompressedGeometry(const EdgeID id, + std::vector &result_nodes) const = 0; + + // Gets the weight values for each segment in an uncompressed geometry. + // Should always be 1 shorter than GetUncompressedGeometry + virtual void GetUncompressedWeights(const EdgeID id, + std::vector &result_weights) const = 0; virtual extractor::TurnInstruction GetTurnInstructionForEdgeID(const unsigned id) const = 0; diff --git a/include/engine/datafacade/internal_datafacade.hpp b/include/engine/datafacade/internal_datafacade.hpp index ffa851706..b49bc463c 100644 --- a/include/engine/datafacade/internal_datafacade.hpp +++ b/include/engine/datafacade/internal_datafacade.hpp @@ -16,6 +16,7 @@ #include "util/graph_loader.hpp" #include "util/simple_logger.hpp" #include "util/rectangle.hpp" +#include "extractor/compressed_edge_container.hpp" #include "osrm/coordinate.hpp" @@ -55,7 +56,7 @@ template class InternalDataFacade final : public BaseDataFacad using RTreeLeaf = typename super::RTreeLeaf; using InternalRTree = util::StaticRTree::vector, false>; - using InternalGeospatialQuery = GeospatialQuery; + using InternalGeospatialQuery = GeospatialQuery>; InternalDataFacade() {} @@ -72,8 +73,9 @@ template class InternalDataFacade final : public BaseDataFacad util::ShM::vector m_names_char_list; util::ShM::vector m_edge_is_compressed; util::ShM::vector m_geometry_indices; - util::ShM::vector m_geometry_list; + util::ShM::vector m_geometry_list; util::ShM::vector m_is_core_node; + util::ShM::vector m_segment_weights; boost::thread_specific_ptr m_static_rtree; boost::thread_specific_ptr m_geospatial_query; @@ -220,7 +222,7 @@ template class InternalDataFacade final : public BaseDataFacad if (number_of_compressed_geometries > 0) { geometry_stream.read((char *)&(m_geometry_list[0]), - number_of_compressed_geometries * sizeof(unsigned)); + number_of_compressed_geometries * sizeof(extractor::CompressedEdgeContainer::CompressedEdge)); } geometry_stream.close(); } @@ -230,7 +232,7 @@ template class InternalDataFacade final : public BaseDataFacad BOOST_ASSERT_MSG(!m_coordinate_list->empty(), "coordinates must be loaded before r-tree"); m_static_rtree.reset(new InternalRTree(ram_index_path, file_index_path, m_coordinate_list)); - m_geospatial_query.reset(new InternalGeospatialQuery(*m_static_rtree, m_coordinate_list)); + m_geospatial_query.reset(new InternalGeospatialQuery(*m_static_rtree, m_coordinate_list, *this)); } void LoadStreetNames(const boost::filesystem::path &names_file) @@ -464,15 +466,27 @@ template class InternalDataFacade final : public BaseDataFacad } } - virtual void GetUncompressedGeometry(const unsigned id, - std::vector &result_nodes) const override final + virtual void GetUncompressedGeometry(const EdgeID id, + std::vector &result_nodes) const override final { const unsigned begin = m_geometry_indices.at(id); const unsigned end = m_geometry_indices.at(id + 1); result_nodes.clear(); - result_nodes.insert(result_nodes.begin(), m_geometry_list.begin() + begin, - m_geometry_list.begin() + end); + result_nodes.reserve(end - begin); + std::for_each(m_geometry_list.begin() + begin, m_geometry_list.begin() + end, [&](const osrm::extractor::CompressedEdgeContainer::CompressedEdge &edge){ result_nodes.emplace_back(edge.node_id); }); + } + + virtual void GetUncompressedWeights(const EdgeID id, + std::vector &result_weights) const override final + { + const unsigned begin = m_geometry_indices.at(id); + const unsigned end = m_geometry_indices.at(id + 1); + + result_weights.clear(); + result_weights.reserve(end - begin); + std::for_each(m_geometry_list.begin() + begin, m_geometry_list.begin() + end, [&](const osrm::extractor::CompressedEdgeContainer::CompressedEdge &edge){ result_weights.emplace_back(edge.weight); }); + } std::string GetTimestamp() const override final { return m_timestamp; } diff --git a/include/engine/datafacade/shared_datafacade.hpp b/include/engine/datafacade/shared_datafacade.hpp index b6ed20ff1..c6915e931 100644 --- a/include/engine/datafacade/shared_datafacade.hpp +++ b/include/engine/datafacade/shared_datafacade.hpp @@ -50,7 +50,7 @@ template class SharedDataFacade final : public BaseDataFacade< using RTreeLeaf = typename super::RTreeLeaf; using SharedRTree = util::StaticRTree::vector, true>; - using SharedGeospatialQuery = GeospatialQuery; + using SharedGeospatialQuery = GeospatialQuery>; using TimeStampedRTreePair = std::pair>; using RTreeNode = typename SharedRTree::TreeNode; @@ -77,7 +77,7 @@ template class SharedDataFacade final : public BaseDataFacade< util::ShM::vector m_name_begin_indices; util::ShM::vector m_edge_is_compressed; util::ShM::vector m_geometry_indices; - util::ShM::vector m_geometry_list; + util::ShM::vector m_geometry_list; util::ShM::vector m_is_core_node; boost::thread_specific_ptr>> m_static_rtree; @@ -115,7 +115,7 @@ template class SharedDataFacade final : public BaseDataFacade< tree_ptr, data_layout->num_entries[storage::SharedDataLayout::R_SEARCH_TREE], file_index_path, m_coordinate_list))); m_geospatial_query.reset( - new SharedGeospatialQuery(*m_static_rtree->second, m_coordinate_list)); + new SharedGeospatialQuery(*m_static_rtree->second, m_coordinate_list, *this)); } void LoadGraph() @@ -221,9 +221,10 @@ template class SharedDataFacade final : public BaseDataFacade< data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_INDEX]); m_geometry_indices = std::move(geometry_begin_indices); - auto geometries_list_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::GEOMETRIES_LIST); - typename util::ShM::vector geometry_list( + auto geometries_list_ptr = + data_layout->GetBlockPtr( + shared_memory, storage::SharedDataLayout::GEOMETRIES_LIST); + typename util::ShM::vector geometry_list( geometries_list_ptr, data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_LIST]); m_geometry_list = std::move(geometry_list); @@ -382,15 +383,27 @@ template class SharedDataFacade final : public BaseDataFacade< return m_edge_is_compressed.at(id); } - virtual void GetUncompressedGeometry(const unsigned id, - std::vector &result_nodes) const override final + virtual void GetUncompressedGeometry(const EdgeID id, + std::vector &result_nodes) const override final { const unsigned begin = m_geometry_indices.at(id); const unsigned end = m_geometry_indices.at(id + 1); result_nodes.clear(); - result_nodes.insert(result_nodes.begin(), m_geometry_list.begin() + begin, - m_geometry_list.begin() + end); + result_nodes.reserve(end - begin); + std::for_each(m_geometry_list.begin() + begin, m_geometry_list.begin() + end, [&](const osrm::extractor::CompressedEdgeContainer::CompressedEdge &edge){ result_nodes.emplace_back(edge.node_id); }); + } + + virtual void GetUncompressedWeights(const EdgeID id, + std::vector &result_weights) const override final + { + const unsigned begin = m_geometry_indices.at(id); + const unsigned end = m_geometry_indices.at(id + 1); + + result_weights.clear(); + result_weights.reserve(end - begin); + std::for_each(m_geometry_list.begin() + begin, m_geometry_list.begin() + end, [&](const osrm::extractor::CompressedEdgeContainer::CompressedEdge &edge){ result_weights.emplace_back(edge.weight); }); + } virtual unsigned GetGeometryIndexForEdgeID(const unsigned id) const override final diff --git a/include/engine/geospatial_query.hpp b/include/engine/geospatial_query.hpp index 3cc0dada4..16adcbf52 100644 --- a/include/engine/geospatial_query.hpp +++ b/include/engine/geospatial_query.hpp @@ -22,14 +22,14 @@ namespace engine // Implements complex queries on top of an RTree and builds PhantomNodes from it. // // Only holds a weak reference on the RTree! -template class GeospatialQuery +template class GeospatialQuery { using EdgeData = typename RTreeT::EdgeData; using CoordinateList = typename RTreeT::CoordinateList; public: - GeospatialQuery(RTreeT &rtree_, std::shared_ptr coordinates_) - : rtree(rtree_), coordinates(std::move(coordinates_)) + GeospatialQuery(RTreeT &rtree_, std::shared_ptr coordinates_, DataFacadeT &datafacade_) + : rtree(rtree_), coordinates(std::move(coordinates_)), datafacade(datafacade_) { } @@ -150,19 +150,51 @@ template class GeospatialQuery coordinates->at(data.u), coordinates->at(data.v), input_coordinate, point_on_segment, ratio); - auto transformed = PhantomNodeWithDistance{PhantomNode{data, point_on_segment}, - current_perpendicular_distance}; + // Find the node-based-edge that this belongs to, and directly + // calculate the forward_weight, forward_offset, reverse_weight, reverse_offset + + int forward_offset = 0, forward_weight = 0; + int reverse_offset = 0, reverse_weight = 0; + + if (data.forward_packed_geometry_id != SPECIAL_EDGEID) { + std::vector forward_weight_vector; + datafacade.GetUncompressedWeights(data.forward_packed_geometry_id, + forward_weight_vector); + for (std::size_t i = 0; i < data.fwd_segment_position; i++) + { + forward_offset += forward_weight_vector[i]; + } + forward_weight = forward_weight_vector[data.fwd_segment_position]; + } + + if (data.reverse_packed_geometry_id != SPECIAL_EDGEID) { + std::vector reverse_weight_vector; + datafacade.GetUncompressedWeights(data.reverse_packed_geometry_id, + reverse_weight_vector); + + //BOOST_ASSERT(reverse_weight_vector.size() == forward_weight_vector.size()); + BOOST_ASSERT(data.fwd_segment_position < reverse_weight_vector.size()); + + for (std::size_t i = 0; i < reverse_weight_vector.size() - data.fwd_segment_position - 1; i++) + { + reverse_offset += reverse_weight_vector[i]; + } + reverse_weight = reverse_weight_vector[reverse_weight_vector.size() - + data.fwd_segment_position - 1]; + } ratio = std::min(1.0, std::max(0.0, ratio)); + if (SPECIAL_NODEID != data.forward_edge_based_node_id) { + forward_weight *= ratio; + } + if (SPECIAL_NODEID != data.reverse_edge_based_node_id) { + reverse_weight *= 1.0 - ratio; + } + + auto transformed = PhantomNodeWithDistance{PhantomNode{data, forward_weight, forward_offset, + reverse_weight, reverse_offset, point_on_segment}, + current_perpendicular_distance}; - if (SPECIAL_NODEID != transformed.phantom_node.forward_node_id) - { - transformed.phantom_node.forward_weight *= ratio; - } - if (SPECIAL_NODEID != transformed.phantom_node.reverse_node_id) - { - transformed.phantom_node.reverse_weight *= 1.0 - ratio; - } return transformed; } @@ -190,6 +222,7 @@ template class GeospatialQuery RTreeT &rtree; const std::shared_ptr coordinates; + DataFacadeT &datafacade; }; } } diff --git a/include/engine/phantom_node.hpp b/include/engine/phantom_node.hpp index 3cb43c019..ecb30d696 100644 --- a/include/engine/phantom_node.hpp +++ b/include/engine/phantom_node.hpp @@ -24,7 +24,8 @@ struct PhantomNode int reverse_weight, int forward_offset, int reverse_offset, - unsigned packed_geometry_id, + unsigned forward_packed_geometry_id_, + unsigned reverse_packed_geometry_id_, bool is_tiny_component, unsigned component_id, util::FixedPointCoordinate location, @@ -34,7 +35,9 @@ struct PhantomNode : forward_node_id(forward_node_id), reverse_node_id(reverse_node_id), name_id(name_id), forward_weight(forward_weight), reverse_weight(reverse_weight), forward_offset(forward_offset), reverse_offset(reverse_offset), - packed_geometry_id(packed_geometry_id), component{component_id, is_tiny_component}, + forward_packed_geometry_id(forward_packed_geometry_id_), + reverse_packed_geometry_id(reverse_packed_geometry_id_), + component{component_id, is_tiny_component}, location(std::move(location)), fwd_segment_position(fwd_segment_position), forward_travel_mode(forward_travel_mode), backward_travel_mode(backward_travel_mode) { @@ -44,7 +47,9 @@ struct PhantomNode : forward_node_id(SPECIAL_NODEID), reverse_node_id(SPECIAL_NODEID), name_id(std::numeric_limits::max()), forward_weight(INVALID_EDGE_WEIGHT), reverse_weight(INVALID_EDGE_WEIGHT), forward_offset(0), reverse_offset(0), - packed_geometry_id(SPECIAL_EDGEID), component{INVALID_COMPONENTID, false}, + forward_packed_geometry_id(SPECIAL_EDGEID), + reverse_packed_geometry_id(SPECIAL_EDGEID), + component{INVALID_COMPONENTID, false}, fwd_segment_position(0), forward_travel_mode(TRAVEL_MODE_INACCESSIBLE), backward_travel_mode(TRAVEL_MODE_INACCESSIBLE) { @@ -89,19 +94,20 @@ struct PhantomNode bool operator==(const PhantomNode &other) const { return location == other.location; } template - PhantomNode(const OtherT &other, const util::FixedPointCoordinate foot_point) + PhantomNode(const OtherT &other, int forward_weight_, int forward_offset_, int reverse_weight_, int reverse_offset_, const util::FixedPointCoordinate foot_point) { forward_node_id = other.forward_edge_based_node_id; reverse_node_id = other.reverse_edge_based_node_id; name_id = other.name_id; - forward_weight = other.forward_weight; - reverse_weight = other.reverse_weight; + forward_weight = forward_weight_; + reverse_weight = reverse_weight_; - forward_offset = other.forward_offset; - reverse_offset = other.reverse_offset; + forward_offset = forward_offset_; + reverse_offset = reverse_offset_; - packed_geometry_id = other.packed_geometry_id; + forward_packed_geometry_id = other.forward_packed_geometry_id; + reverse_packed_geometry_id = other.reverse_packed_geometry_id; component.id = other.component.id; component.is_tiny = other.component.is_tiny; @@ -120,7 +126,8 @@ struct PhantomNode int reverse_weight; int forward_offset; int reverse_offset; - unsigned packed_geometry_id; + unsigned forward_packed_geometry_id; + unsigned reverse_packed_geometry_id; struct ComponentType { uint32_t id : 31; @@ -139,7 +146,7 @@ struct PhantomNode }; #ifndef _MSC_VER -static_assert(sizeof(PhantomNode) == 48, "PhantomNode has more padding then expected"); +static_assert(sizeof(PhantomNode) == 52, "PhantomNode has more padding then expected"); #endif using PhantomNodePair = std::pair; @@ -172,7 +179,8 @@ inline std::ostream &operator<<(std::ostream &out, const PhantomNode &pn) << "rev-w: " << pn.reverse_weight << ", " << "fwd-o: " << pn.forward_offset << ", " << "rev-o: " << pn.reverse_offset << ", " - << "geom: " << pn.packed_geometry_id << ", " + << "fwd_geom: " << pn.forward_packed_geometry_id << ", " + << "rev_geom: " << pn.reverse_packed_geometry_id << ", " << "comp: " << pn.component.is_tiny << " / " << pn.component.id << ", " << "pos: " << pn.fwd_segment_position << ", " << "loc: " << pn.location; diff --git a/include/engine/plugins/tile.hpp b/include/engine/plugins/tile.hpp index c6b1cb4b7..acaea5ae1 100644 --- a/include/engine/plugins/tile.hpp +++ b/include/engine/plugins/tile.hpp @@ -244,12 +244,34 @@ template class TilePlugin final : public BasePlugin // Get coordinates for start/end nodes of segmet (NodeIDs u and v) const auto a = facade->GetCoordinateOfNode(edge.u); const auto b = facade->GetCoordinateOfNode(edge.v); - // Calculate the length in meters - const double length = osrm::util::coordinate_calculation::haversineDistance( - a.lon, a.lat, b.lon, b.lat); + // Calculate the length in meters, using the same calculation used to set the + // weight, so we can back-calculate the speed value that was set. + const double length = osrm::util::coordinate_calculation::greatCircleDistance( + a.lat, a.lon, b.lat, b.lon); + + int forward_weight = 0; + int reverse_weight = 0; + + if (edge.forward_packed_geometry_id != SPECIAL_EDGEID) { + std::vector forward_weight_vector; + facade->GetUncompressedWeights(edge.forward_packed_geometry_id, + forward_weight_vector); + forward_weight = forward_weight_vector[edge.fwd_segment_position]; + } + + if (edge.reverse_packed_geometry_id != SPECIAL_EDGEID) { + std::vector reverse_weight_vector; + facade->GetUncompressedWeights(edge.reverse_packed_geometry_id, + reverse_weight_vector); + + BOOST_ASSERT(edge.fwd_segment_position < reverse_weight_vector.size()); + + reverse_weight = reverse_weight_vector[reverse_weight_vector.size() - + edge.fwd_segment_position - 1]; + } // If this is a valid forward edge, go ahead and add it to the tile - if (edge.forward_weight != 0 && + if (forward_weight != 0 && edge.forward_edge_based_node_id != SPECIAL_NODEID) { std::int32_t start_x = 0; @@ -263,7 +285,7 @@ template class TilePlugin final : public BasePlugin // Calculate the speed for this line std::uint32_t speed = static_cast( - round(length / edge.forward_weight * 10 * 3.6)); + round(length / forward_weight * 10 * 3.6)); line_type tile_line; for (auto const &pt : geo_line) @@ -315,7 +337,7 @@ template class TilePlugin final : public BasePlugin // Repeat the above for the coordinates reversed and using the `reverse` // properties - if (edge.reverse_weight != 0 && + if (reverse_weight != 0 && edge.reverse_edge_based_node_id != SPECIAL_NODEID) { std::int32_t start_x = 0; @@ -328,7 +350,7 @@ template class TilePlugin final : public BasePlugin a.lat / COORDINATE_PRECISION); const auto speed = static_cast( - round(length / edge.forward_weight * 10 * 3.6)); + round(length / reverse_weight * 10 * 3.6)); line_type tile_line; for (auto const &pt : geo_line) diff --git a/include/engine/routing_algorithms/routing_base.hpp b/include/engine/routing_algorithms/routing_base.hpp index 50c4b284c..796733d52 100644 --- a/include/engine/routing_algorithms/routing_base.hpp +++ b/include/engine/routing_algorithms/routing_base.hpp @@ -17,6 +17,7 @@ #include #include #include +#include namespace osrm { @@ -302,10 +303,20 @@ template class BasicRoutingInterface } else { - std::vector id_vector; + std::vector id_vector; facade->GetUncompressedGeometry(facade->GetGeometryIndexForEdgeID(ed.id), id_vector); + std::vector weight_vector; + facade->GetUncompressedWeights(facade->GetGeometryIndexForEdgeID(ed.id), + weight_vector); + + int total_weight = std::accumulate(weight_vector.begin(), weight_vector.end(), 0); + + BOOST_ASSERT(weight_vector.size() == id_vector.size()); + // ed.distance should be total_weight + penalties (turn, stop, etc) + BOOST_ASSERT(ed.distance >= total_weight); + const std::size_t start_index = (unpacked_path.empty() ? ((start_traversed_in_reverse) @@ -320,58 +331,57 @@ template class BasicRoutingInterface for (std::size_t i = start_index; i < end_index; ++i) { unpacked_path.emplace_back(id_vector[i], name_index, - extractor::TurnInstruction::NoTurn, 0, + extractor::TurnInstruction::NoTurn, weight_vector[i], travel_mode); } unpacked_path.back().turn_instruction = turn_instruction; - unpacked_path.back().segment_duration = ed.distance; + unpacked_path.back().segment_duration += (ed.distance - total_weight); } } } - if (SPECIAL_EDGEID != phantom_node_pair.target_phantom.packed_geometry_id) + std::vector id_vector; + facade->GetUncompressedGeometry(phantom_node_pair.target_phantom.forward_packed_geometry_id, + id_vector); + const bool is_local_path = (phantom_node_pair.source_phantom.forward_packed_geometry_id == + phantom_node_pair.target_phantom.forward_packed_geometry_id) && + unpacked_path.empty(); + + std::cout << "Got id vector of size " << id_vector.size() << "\n"; + + std::size_t start_index = 0; + if (is_local_path) { - std::vector id_vector; - facade->GetUncompressedGeometry(phantom_node_pair.target_phantom.packed_geometry_id, - id_vector); - const bool is_local_path = (phantom_node_pair.source_phantom.packed_geometry_id == - phantom_node_pair.target_phantom.packed_geometry_id) && - unpacked_path.empty(); - - std::size_t start_index = 0; - if (is_local_path) - { - start_index = phantom_node_pair.source_phantom.fwd_segment_position; - if (target_traversed_in_reverse) - { - start_index = - id_vector.size() - phantom_node_pair.source_phantom.fwd_segment_position; - } - } - - std::size_t end_index = phantom_node_pair.target_phantom.fwd_segment_position; + start_index = phantom_node_pair.source_phantom.fwd_segment_position; if (target_traversed_in_reverse) { - std::reverse(id_vector.begin(), id_vector.end()); - end_index = - id_vector.size() - phantom_node_pair.target_phantom.fwd_segment_position; + start_index = + id_vector.size() - phantom_node_pair.source_phantom.fwd_segment_position; } + } - if (start_index > end_index) - { - start_index = std::min(start_index, id_vector.size() - 1); - } + std::size_t end_index = phantom_node_pair.target_phantom.fwd_segment_position; + if (target_traversed_in_reverse) + { + std::reverse(id_vector.begin(), id_vector.end()); + end_index = + id_vector.size() - phantom_node_pair.target_phantom.fwd_segment_position; + } - for (std::size_t i = start_index; i != end_index; (start_index < end_index ? ++i : --i)) - { - BOOST_ASSERT(i < id_vector.size()); - BOOST_ASSERT(phantom_node_pair.target_phantom.forward_travel_mode > 0); - unpacked_path.emplace_back( - PathData{id_vector[i], phantom_node_pair.target_phantom.name_id, - extractor::TurnInstruction::NoTurn, 0, - target_traversed_in_reverse - ? phantom_node_pair.target_phantom.backward_travel_mode - : phantom_node_pair.target_phantom.forward_travel_mode}); - } + if (start_index > end_index) + { + start_index = std::min(start_index, id_vector.size() - 1); + } + + for (std::size_t i = start_index; i != end_index; (start_index < end_index ? ++i : --i)) + { + BOOST_ASSERT(i < id_vector.size()); + BOOST_ASSERT(phantom_node_pair.target_phantom.forward_travel_mode > 0); + unpacked_path.emplace_back( + PathData{id_vector[i], phantom_node_pair.target_phantom.name_id, + extractor::TurnInstruction::NoTurn, 0, + target_traversed_in_reverse + ? phantom_node_pair.target_phantom.backward_travel_mode + : phantom_node_pair.target_phantom.forward_travel_mode}); } // there is no equivalent to a node-based node in an edge-expanded graph. diff --git a/include/extractor/compressed_edge_container.hpp b/include/extractor/compressed_edge_container.hpp index ed9122dd2..0ab194bc7 100644 --- a/include/extractor/compressed_edge_container.hpp +++ b/include/extractor/compressed_edge_container.hpp @@ -16,8 +16,13 @@ namespace extractor class CompressedEdgeContainer { public: - using CompressedNode = std::pair; - using EdgeBucket = std::vector; + struct CompressedEdge + { + public: + NodeID node_id; // refers to an internal node-based-node + EdgeWeight weight; // the weight of the edge leading to this node + }; + using EdgeBucket = std::vector; CompressedEdgeContainer(); void CompressEdge(const EdgeID surviving_edge_id, @@ -27,6 +32,10 @@ class CompressedEdgeContainer const EdgeWeight weight1, const EdgeWeight weight2); + void AddUncompressedEdge(const EdgeID edgei_id, + const NodeID target_node, + const EdgeWeight weight); + bool HasEntryForID(const EdgeID edge_id) const; void PrintStatistics() const; void SerializeInternalVector(const std::string &path) const; diff --git a/include/extractor/edge_based_graph_factory.hpp b/include/extractor/edge_based_graph_factory.hpp index 6dfbca3ff..6b89169a0 100644 --- a/include/extractor/edge_based_graph_factory.hpp +++ b/include/extractor/edge_based_graph_factory.hpp @@ -43,7 +43,7 @@ class EdgeBasedGraphFactory EdgeBasedGraphFactory &operator=(const EdgeBasedGraphFactory &) = delete; explicit EdgeBasedGraphFactory(std::shared_ptr node_based_graph, - const CompressedEdgeContainer &compressed_edge_container, + CompressedEdgeContainer &compressed_edge_container, const std::unordered_set &barrier_nodes, const std::unordered_set &traffic_lights, std::shared_ptr restriction_map, @@ -99,7 +99,7 @@ class EdgeBasedGraphFactory const std::unordered_set &m_barrier_nodes; const std::unordered_set &m_traffic_lights; - const CompressedEdgeContainer &m_compressed_edge_container; + CompressedEdgeContainer &m_compressed_edge_container; SpeedProfileProperties speed_profile; diff --git a/include/extractor/edge_based_node.hpp b/include/extractor/edge_based_node.hpp index 4c821d275..c383d932e 100644 --- a/include/extractor/edge_based_node.hpp +++ b/include/extractor/edge_based_node.hpp @@ -22,8 +22,8 @@ struct EdgeBasedNode EdgeBasedNode() : forward_edge_based_node_id(SPECIAL_NODEID), reverse_edge_based_node_id(SPECIAL_NODEID), u(SPECIAL_NODEID), v(SPECIAL_NODEID), name_id(0), - forward_weight(INVALID_EDGE_WEIGHT >> 1), reverse_weight(INVALID_EDGE_WEIGHT >> 1), - forward_offset(0), reverse_offset(0), packed_geometry_id(SPECIAL_EDGEID), + forward_packed_geometry_id(SPECIAL_EDGEID), + reverse_packed_geometry_id(SPECIAL_EDGEID), component{INVALID_COMPONENTID, false}, fwd_segment_position(std::numeric_limits::max()), forward_travel_mode(TRAVEL_MODE_INACCESSIBLE), @@ -36,11 +36,8 @@ struct EdgeBasedNode NodeID u, NodeID v, unsigned name_id, - int forward_weight, - int reverse_weight, - int forward_offset, - int reverse_offset, - unsigned packed_geometry_id, + unsigned forward_weight_or_packed_geometry_id_, + unsigned reverse_weight_or_packed_geometry_id_, bool is_tiny_component, unsigned component_id, unsigned short fwd_segment_position, @@ -48,9 +45,9 @@ struct EdgeBasedNode TravelMode backward_travel_mode) : forward_edge_based_node_id(forward_edge_based_node_id), reverse_edge_based_node_id(reverse_edge_based_node_id), u(u), v(v), name_id(name_id), - forward_weight(forward_weight), reverse_weight(reverse_weight), - forward_offset(forward_offset), reverse_offset(reverse_offset), - packed_geometry_id(packed_geometry_id), component{component_id, is_tiny_component}, + forward_packed_geometry_id(forward_weight_or_packed_geometry_id_), + reverse_packed_geometry_id(reverse_weight_or_packed_geometry_id_), + component{component_id, is_tiny_component}, fwd_segment_position(fwd_segment_position), forward_travel_mode(forward_travel_mode), backward_travel_mode(backward_travel_mode) { @@ -68,18 +65,14 @@ struct EdgeBasedNode return centroid; } - bool IsCompressed() const { return packed_geometry_id != SPECIAL_EDGEID; } - NodeID forward_edge_based_node_id; // needed for edge-expanded graph NodeID reverse_edge_based_node_id; // needed for edge-expanded graph NodeID u; // indices into the coordinates array NodeID v; // indices into the coordinates array unsigned name_id; // id of the edge name - int forward_weight; // weight of the edge - int reverse_weight; // weight in the other direction (may be different) - int forward_offset; // prefix sum of the weight up the edge TODO: short must suffice - int reverse_offset; // prefix sum of the weight from the edge TODO: short must suffice - unsigned packed_geometry_id; // if set, then the edge represents a packed geometry + + unsigned forward_packed_geometry_id; + unsigned reverse_packed_geometry_id; struct { unsigned id : 31; diff --git a/include/util/static_rtree.hpp b/include/util/static_rtree.hpp index f01decf15..94b8ea360 100644 --- a/include/util/static_rtree.hpp +++ b/include/util/static_rtree.hpp @@ -59,6 +59,13 @@ class StaticRTree std::uint32_t children[BRANCHING_FACTOR]; }; + struct LeafNode + { + LeafNode() : object_count(0), objects() {} + uint32_t object_count; + std::array objects; + }; + private: struct WrappedInputElement { @@ -79,13 +86,6 @@ class StaticRTree } }; - struct LeafNode - { - LeafNode() : object_count(0), objects() {} - std::uint32_t object_count; - std::array objects; - }; - using QueryNodeType = mapbox::util::variant; struct QueryCandidate { diff --git a/src/benchmarks/static_rtree.cpp b/src/benchmarks/static_rtree.cpp index 54126c805..d8bbac128 100644 --- a/src/benchmarks/static_rtree.cpp +++ b/src/benchmarks/static_rtree.cpp @@ -3,6 +3,8 @@ #include "extractor/edge_based_node.hpp" #include "engine/geospatial_query.hpp" #include "util/timing_util.hpp" +#include "engine/datafacade/datafacade_base.hpp" +#include "contractor/query_edge.hpp" #include "osrm/coordinate.hpp" @@ -14,6 +16,84 @@ namespace osrm namespace benchmarks { +template class MockDataFacadeT final : public osrm::engine::datafacade::BaseDataFacade +{ + private: + EdgeDataT foo; + public: + unsigned GetNumberOfNodes() const { return 0; } + unsigned GetNumberOfEdges() const { return 0; } + unsigned GetOutDegree(const NodeID /* n */) const { return 0; } + NodeID GetTarget(const EdgeID /* e */) const { return SPECIAL_NODEID; } + const EdgeDataT &GetEdgeData(const EdgeID /* e */) const { + return foo; + } + EdgeID BeginEdges(const NodeID /* n */) const { return SPECIAL_EDGEID; } + EdgeID EndEdges(const NodeID /* n */) const { return SPECIAL_EDGEID; } + osrm::engine::datafacade::EdgeRange GetAdjacentEdgeRange(const NodeID /* node */) const { + return util::irange(static_cast(0),static_cast(0)); + } + EdgeID FindEdge(const NodeID /* from */, const NodeID /* to */) const { return SPECIAL_EDGEID; } + EdgeID FindEdgeInEitherDirection(const NodeID /* from */, const NodeID /* to */) const { return SPECIAL_EDGEID; } + EdgeID + FindEdgeIndicateIfReverse(const NodeID /* from */, const NodeID /* to */, bool & /* result */) const { return SPECIAL_EDGEID; } + util::FixedPointCoordinate GetCoordinateOfNode(const unsigned /* id */) const { + FixedPointCoordinate foo(0,0); + return foo; + } + bool EdgeIsCompressed(const unsigned /* id */) const { return false; } + unsigned GetGeometryIndexForEdgeID(const unsigned /* id */) const { return SPECIAL_NODEID; } + void GetUncompressedGeometry(const EdgeID /* id */, + std::vector &/* result_nodes */) const {} + void GetUncompressedWeights(const EdgeID /* id */, + std::vector & /* result_weights */) const {} + extractor::TurnInstruction GetTurnInstructionForEdgeID(const unsigned /* id */) const { + return osrm::extractor::TurnInstruction::NoTurn; + } + extractor::TravelMode GetTravelModeForEdgeID(const unsigned /* id */) const + { + return TRAVEL_MODE_DEFAULT; + } + std::vector::RTreeLeaf> GetEdgesInBox(const util::FixedPointCoordinate & /* south_west */, + const util::FixedPointCoordinate & /*north_east */) { + std::vector::RTreeLeaf> foo; + return foo; + } + std::vector + NearestPhantomNodesInRange(const util::FixedPointCoordinate /* input_coordinate */, + const float /* max_distance */, + const int /* bearing = 0 */, + const int /* bearing_range = 180 */) { + std::vector foo; + return foo; + } + std::vector + NearestPhantomNodes(const util::FixedPointCoordinate /* input_coordinate */, + const unsigned /* max_results */, + const int /* bearing = 0 */, + const int /* bearing_range = 180 */) { + std::vector foo; + return foo; + } + std::pair NearestPhantomNodeWithAlternativeFromBigComponent( + const util::FixedPointCoordinate /* input_coordinate */, + const int /* bearing = 0 */, + const int /* bearing_range = 180 */) { + std::pair foo; + return foo; + } + unsigned GetCheckSum() const { return 0; } + bool IsCoreNode(const NodeID /* id */) const { return false; } + unsigned GetNameIndexFromEdgeID(const unsigned /* id */) const { return 0; } + std::string get_name_for_id(const unsigned /* name_id */) const { return ""; } + std::size_t GetCoreSize() const { return 0; } + std::string GetTimestamp() const { return ""; } + +}; + +using MockDataFacade = MockDataFacadeT; + + // Choosen by a fair W20 dice roll (this value is completely arbitrary) constexpr unsigned RANDOM_SEED = 13; constexpr int32_t WORLD_MIN_LAT = -90 * COORDINATE_PRECISION; @@ -25,7 +105,7 @@ using RTreeLeaf = extractor::EdgeBasedNode; using FixedPointCoordinateListPtr = std::shared_ptr>; using BenchStaticRTree = util::StaticRTree::vector, false>; -using BenchQuery = engine::GeospatialQuery; +using BenchQuery = engine::GeospatialQuery; FixedPointCoordinateListPtr loadCoordinates(const boost::filesystem::path &nodes_file) { @@ -128,7 +208,8 @@ int main(int argc, char **argv) auto coords = osrm::benchmarks::loadCoordinates(nodes_path); osrm::benchmarks::BenchStaticRTree rtree(ram_path, file_path, coords); - osrm::benchmarks::BenchQuery query(rtree, coords); + std::unique_ptr mockfacade_ptr(new osrm::benchmarks::MockDataFacade); + osrm::benchmarks::BenchQuery query(rtree, coords, *mockfacade_ptr); osrm::benchmarks::benchmark(rtree, query, 10000); diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp index b84feb126..a3831ce3d 100644 --- a/src/contractor/contractor.cpp +++ b/src/contractor/contractor.cpp @@ -2,6 +2,9 @@ #include "contractor/graph_contractor.hpp" #include "extractor/edge_based_edge.hpp" +#include "extractor/compressed_edge_container.hpp" + +#include "util/static_rtree.hpp" #include "util/deallocating_vector.hpp" @@ -31,6 +34,7 @@ #include #include #include +#include namespace std { @@ -74,8 +78,10 @@ int Contractor::Run() util::DeallocatingVector edge_based_edge_list; std::size_t max_edge_id = LoadEdgeExpandedGraph( - config.edge_based_graph_path, edge_based_edge_list, config.edge_segment_lookup_path, - config.edge_penalty_path, config.segment_speed_lookup_path); + config.edge_based_graph_path, edge_based_edge_list, + config.edge_segment_lookup_path, config.edge_penalty_path, + config.segment_speed_lookup_path, config.node_based_graph_path, + config.geometry_path, config.rtree_leaf_path); // Contracting the edge-expanded graph @@ -130,7 +136,10 @@ std::size_t Contractor::LoadEdgeExpandedGraph( util::DeallocatingVector &edge_based_edge_list, const std::string &edge_segment_lookup_filename, const std::string &edge_penalty_filename, - const std::string &segment_speed_filename) + const std::string &segment_speed_filename, + const std::string &nodes_filename, + const std::string &geometry_filename, + const std::string &rtree_leaf_filename) { util::SimpleLogger().Write() << "Opening " << edge_based_graph_filename; boost::filesystem::ifstream input_stream(edge_based_graph_filename, std::ios::binary); @@ -184,6 +193,170 @@ std::size_t Contractor::LoadEdgeExpandedGraph( segment_speed_lookup[std::make_pair(OSMNodeID(from_node_id), OSMNodeID(to_node_id))] = speed; } + + std::vector internal_to_external_node_map; + + // Here, we have to update the compressed geometry weights + // First, we need the external-to-internal node lookup table + { + boost::filesystem::ifstream nodes_input_stream(nodes_filename, std::ios::binary); + + if (!nodes_input_stream) + { + throw util::exception("Failed to open "+nodes_filename); + } + + unsigned number_of_nodes = 0; + nodes_input_stream.read((char *)&number_of_nodes, sizeof(unsigned)); + internal_to_external_node_map.resize(number_of_nodes); + + // Load all the query nodes into a vector + nodes_input_stream.read(reinterpret_cast(&(internal_to_external_node_map[0])), number_of_nodes * sizeof(extractor::QueryNode)); + nodes_input_stream.close(); + } + + std::vector m_geometry_indices; + std::vector m_geometry_list; + + { + std::ifstream geometry_stream(geometry_filename, std::ios::binary); + if (!geometry_stream) + { + throw util::exception("Failed to open "+geometry_filename); + } + unsigned number_of_indices = 0; + unsigned number_of_compressed_geometries = 0; + + geometry_stream.read((char *)&number_of_indices, sizeof(unsigned)); + + m_geometry_indices.resize(number_of_indices); + if (number_of_indices > 0) + { + geometry_stream.read((char *)&(m_geometry_indices[0]), + number_of_indices * sizeof(unsigned)); + } + + geometry_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned)); + + BOOST_ASSERT(m_geometry_indices.back() == number_of_compressed_geometries); + m_geometry_list.resize(number_of_compressed_geometries); + + if (number_of_compressed_geometries > 0) + { + geometry_stream.read((char *)&(m_geometry_list[0]), + number_of_compressed_geometries * sizeof(extractor::CompressedEdgeContainer::CompressedEdge)); + } + geometry_stream.close(); + } + + // Now, we iterate over all the segments stored in the StaticRTree, updating + // the packed geometry weights in the `.geometries` file (note: we do not + // update the RTree itself, we just use the leaf nodes to iterate over all segments) + { + + using LeafNode = util::StaticRTree::LeafNode; + + // Open file for reading *and* writing + std::ifstream leaf_node_file(rtree_leaf_filename, std::ios::binary | std::ios::in); + if (!leaf_node_file) + { + throw util::exception("Failed to open "+rtree_leaf_filename); + } + uint64_t m_element_count; + leaf_node_file.read((char *)&m_element_count, sizeof(uint64_t)); + + LeafNode current_node; + while (m_element_count > 0) + { + leaf_node_file.read(reinterpret_cast(¤t_node), sizeof(current_node)); + + for (size_t i=0; i< current_node.object_count; i++) + { + auto & leaf_object = current_node.objects[i]; + extractor::QueryNode *u; + extractor::QueryNode *v; + + if (leaf_object.forward_packed_geometry_id != SPECIAL_EDGEID) + { + const unsigned forward_begin = m_geometry_indices.at(leaf_object.forward_packed_geometry_id); + + if (leaf_object.fwd_segment_position == 0) + { + u = &(internal_to_external_node_map[leaf_object.u]); + v = &(internal_to_external_node_map[m_geometry_list[forward_begin].node_id]); + } + else + { + u = &(internal_to_external_node_map[m_geometry_list[forward_begin + leaf_object.fwd_segment_position - 1].node_id]); + v = &(internal_to_external_node_map[m_geometry_list[forward_begin + leaf_object.fwd_segment_position].node_id]); + } + const double segment_length = + util::coordinate_calculation::greatCircleDistance( + u->lat, u->lon, v->lat, v->lon); + + auto forward_speed_iter = segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id)); + if (forward_speed_iter != segment_speed_lookup.end()) + { + int new_segment_weight = + std::max(1, static_cast(std::floor( + (segment_length * 10.) / (forward_speed_iter->second / 3.6) + .5))); + m_geometry_list[forward_begin + leaf_object.fwd_segment_position].weight = new_segment_weight; + } + } + if (leaf_object.reverse_packed_geometry_id != SPECIAL_EDGEID) + { + const unsigned reverse_begin = m_geometry_indices.at(leaf_object.reverse_packed_geometry_id); + const unsigned reverse_end = m_geometry_indices.at(leaf_object.reverse_packed_geometry_id + 1); + + int rev_segment_position = (reverse_end - reverse_begin) - leaf_object.fwd_segment_position - 1; + if (rev_segment_position == 0) + { + u = &(internal_to_external_node_map[leaf_object.v]); + v = &(internal_to_external_node_map[m_geometry_list[reverse_begin].node_id]); + } + else + { + u = &(internal_to_external_node_map[m_geometry_list[reverse_begin + rev_segment_position - 1].node_id]); + v = &(internal_to_external_node_map[m_geometry_list[reverse_begin + rev_segment_position].node_id]); + } + const double segment_length = + util::coordinate_calculation::greatCircleDistance( + u->lat, u->lon, v->lat, v->lon); + + auto reverse_speed_iter = segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id)); + if (reverse_speed_iter != segment_speed_lookup.end()) + { + int new_segment_weight = + std::max(1, static_cast(std::floor( + (segment_length * 10.) / (reverse_speed_iter->second / 3.6) + .5))); + m_geometry_list[reverse_begin + rev_segment_position].weight = new_segment_weight; + } + } + } + m_element_count -= current_node.object_count; + + } + leaf_node_file.close(); + + } + + // Now save out the updated compressed geometries + { + std::ofstream geometry_stream(geometry_filename, std::ios::binary); + if (!geometry_stream) + { + throw util::exception("Failed to open "+geometry_filename+" for writing"); + } + const unsigned number_of_indices = m_geometry_indices.size(); + const unsigned number_of_compressed_geometries = m_geometry_list.size(); + geometry_stream.write(reinterpret_cast(&number_of_indices), sizeof(unsigned)); + geometry_stream.write(reinterpret_cast(&(m_geometry_indices[0])), number_of_indices * sizeof(unsigned)); + geometry_stream.write(reinterpret_cast(&number_of_compressed_geometries), sizeof(unsigned)); + geometry_stream.write(reinterpret_cast(&(m_geometry_list[0])), number_of_compressed_geometries * sizeof(extractor::CompressedEdgeContainer::CompressedEdge)); + geometry_stream.close(); + } + + } // TODO: can we read this in bulk? util::DeallocatingVector isn't necessarily diff --git a/src/extractor/compressed_edge_container.cpp b/src/extractor/compressed_edge_container.cpp index e34bb424e..7eb65f3f9 100644 --- a/src/extractor/compressed_edge_container.cpp +++ b/src/extractor/compressed_edge_container.cpp @@ -59,7 +59,7 @@ void CompressedEdgeContainer::SerializeInternalVector(const std::string &path) c { geometry_out_stream.write((char *)&prefix_sum_of_list_indices, sizeof(unsigned)); - const std::vector ¤t_vector = elem; + const std::vector ¤t_vector = elem; const unsigned unpacked_size = current_vector.size(); BOOST_ASSERT(std::numeric_limits::max() != unpacked_size); prefix_sum_of_list_indices += unpacked_size; @@ -74,13 +74,13 @@ void CompressedEdgeContainer::SerializeInternalVector(const std::string &path) c // write compressed geometries for (auto &elem : m_compressed_geometries) { - const std::vector ¤t_vector = elem; + const std::vector ¤t_vector = elem; const unsigned unpacked_size = current_vector.size(); control_sum += unpacked_size; BOOST_ASSERT(std::numeric_limits::max() != unpacked_size); - for (const CompressedNode current_node : current_vector) + for (const auto & current_node : current_vector) { - geometry_out_stream.write((char *)&(current_node.first), sizeof(NodeID)); + geometry_out_stream.write((char *)&(current_node), sizeof(CompressedEdge)); } } BOOST_ASSERT(control_sum == prefix_sum_of_list_indices); @@ -88,6 +88,14 @@ void CompressedEdgeContainer::SerializeInternalVector(const std::string &path) c geometry_out_stream.close(); } +// Adds info for a compressed edge to the container. edge_id_2 +// has been removed from the graph, so we have to save These edges/nodes +// have already been trimmed from the graph, this function just stores +// the original data for unpacking later. +// +// edge_id_1 edge_id_2 +// ----------> via_node_id -----------> target_node_id +// weight_1 weight_2 void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1, const EdgeID edge_id_2, const NodeID via_node_id, @@ -131,13 +139,13 @@ void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1, BOOST_ASSERT(edge_bucket_id1 == GetPositionForID(edge_id_1)); BOOST_ASSERT(edge_bucket_id1 < m_compressed_geometries.size()); - std::vector &edge_bucket_list1 = m_compressed_geometries[edge_bucket_id1]; + std::vector &edge_bucket_list1 = m_compressed_geometries[edge_bucket_id1]; // note we don't save the start coordinate: it is implicitly given by edge 1 // weight1 is the distance to the (currently) last coordinate in the bucket if (edge_bucket_list1.empty()) { - edge_bucket_list1.emplace_back(via_node_id, weight1); + edge_bucket_list1.emplace_back(CompressedEdge { via_node_id, weight1 }); } BOOST_ASSERT(0 < edge_bucket_list1.size()); @@ -149,7 +157,7 @@ void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1, const unsigned list_to_remove_index = GetPositionForID(edge_id_2); BOOST_ASSERT(list_to_remove_index < m_compressed_geometries.size()); - std::vector &edge_bucket_list2 = + std::vector &edge_bucket_list2 = m_compressed_geometries[list_to_remove_index]; // found an existing list, append it to the list of edge_id_1 @@ -168,10 +176,57 @@ void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1, else { // we are certain that the second edge is atomic. - edge_bucket_list1.emplace_back(target_node_id, weight2); + edge_bucket_list1.emplace_back( CompressedEdge { target_node_id, weight2 }); } } +void CompressedEdgeContainer::AddUncompressedEdge(const EdgeID edge_id, + const NodeID target_node_id, + const EdgeWeight weight) +{ + // remove super-trivial geometries + BOOST_ASSERT(SPECIAL_EDGEID != edge_id); + BOOST_ASSERT(SPECIAL_NODEID != target_node_id); + BOOST_ASSERT(INVALID_EDGE_WEIGHT != weight); + + // There should be no entry for uncompressed edges + BOOST_ASSERT(!HasEntryForID(edge_id)); + + // Add via node id. List is created if it does not exist + if (!HasEntryForID(edge_id)) + { + // create a new entry in the map + if (0 == m_free_list.size()) + { + // make sure there is a place to put the entries + IncreaseFreeList(); + } + BOOST_ASSERT(!m_free_list.empty()); + m_edge_id_to_list_index_map[edge_id] = m_free_list.back(); + m_free_list.pop_back(); + } + + // find bucket index + const auto iter = m_edge_id_to_list_index_map.find(edge_id); + BOOST_ASSERT(iter != m_edge_id_to_list_index_map.end()); + const unsigned edge_bucket_id = iter->second; + BOOST_ASSERT(edge_bucket_id == GetPositionForID(edge_id)); + BOOST_ASSERT(edge_bucket_id < m_compressed_geometries.size()); + + std::vector &edge_bucket_list = m_compressed_geometries[edge_bucket_id]; + + // We're adding uncompressed edges, there should be no entry + BOOST_ASSERT(edge_bucket_list.empty()); + // note we don't save the start coordinate: it is implicitly given by edge_id + // weight is the distance to the (currently) last coordinate in the bucket + if (edge_bucket_list.empty()) + { + edge_bucket_list.emplace_back(CompressedEdge { target_node_id, weight }); + } +} + + + void CompressedEdgeContainer::PrintStatistics() const { const uint64_t compressed_edges = m_compressed_geometries.size(); @@ -180,7 +235,7 @@ void CompressedEdgeContainer::PrintStatistics() const uint64_t compressed_geometries = 0; uint64_t longest_chain_length = 0; - for (const std::vector ¤t_vector : m_compressed_geometries) + for (const std::vector ¤t_vector : m_compressed_geometries) { compressed_geometries += current_vector.size(); longest_chain_length = std::max(longest_chain_length, (uint64_t)current_vector.size()); @@ -207,13 +262,13 @@ NodeID CompressedEdgeContainer::GetFirstEdgeTargetID(const EdgeID edge_id) const { const auto &bucket = GetBucketReference(edge_id); BOOST_ASSERT(bucket.size() >= 2); - return bucket.front().first; + return bucket.front().node_id; } NodeID CompressedEdgeContainer::GetLastEdgeSourceID(const EdgeID edge_id) const { const auto &bucket = GetBucketReference(edge_id); BOOST_ASSERT(bucket.size() >= 2); - return bucket[bucket.size() - 2].first; + return bucket[bucket.size() - 2].node_id; } } } diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index 20c08dfce..9628ac5d6 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -45,7 +45,7 @@ const double constexpr DESIRED_SEGMENT_LENGTH = 10.; EdgeBasedGraphFactory::EdgeBasedGraphFactory( std::shared_ptr node_based_graph, - const CompressedEdgeContainer &compressed_edge_container, + CompressedEdgeContainer &compressed_edge_container, const std::unordered_set &barrier_nodes, const std::unordered_set &traffic_lights, std::shared_ptr restriction_map, @@ -138,8 +138,8 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI // reconstruct bidirectional edge with individual weights and put each into the NN index - std::vector forward_dist_prefix_sum(forward_geometry.size(), 0); - std::vector reverse_dist_prefix_sum(reverse_geometry.size(), 0); + std::vector forward_offsets(forward_geometry.size(), 0); + std::vector reverse_offsets(reverse_geometry.size(), 0); // quick'n'dirty prefix sum as std::partial_sum needs addtional casts // TODO: move to lambda function with C++11 @@ -147,8 +147,8 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI for (const auto i : util::irange(0u, geometry_size)) { - forward_dist_prefix_sum[i] = temp_sum; - temp_sum += forward_geometry[i].second; + forward_offsets[i] = temp_sum; + temp_sum += forward_geometry[i].weight; BOOST_ASSERT(forward_data.distance >= temp_sum); } @@ -156,8 +156,8 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI temp_sum = 0; for (const auto i : util::irange(0u, geometry_size)) { - temp_sum += reverse_geometry[reverse_geometry.size() - 1 - i].second; - reverse_dist_prefix_sum[i] = reverse_data.distance - temp_sum; + temp_sum += reverse_geometry[reverse_geometry.size() - 1 - i].weight; + reverse_offsets[i] = reverse_data.distance - temp_sum; // BOOST_ASSERT(reverse_data.distance >= temp_sum); } @@ -167,24 +167,22 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI for (const auto i : util::irange(0u, geometry_size)) { BOOST_ASSERT(current_edge_source_coordinate_id == - reverse_geometry[geometry_size - 1 - i].first); - const NodeID current_edge_target_coordinate_id = forward_geometry[i].first; + reverse_geometry[geometry_size - 1 - i].node_id); + const NodeID current_edge_target_coordinate_id = forward_geometry[i].node_id; BOOST_ASSERT(current_edge_target_coordinate_id != current_edge_source_coordinate_id); // build edges m_edge_based_node_list.emplace_back( forward_data.edge_id, reverse_data.edge_id, current_edge_source_coordinate_id, - current_edge_target_coordinate_id, forward_data.name_id, forward_geometry[i].second, - reverse_geometry[geometry_size - 1 - i].second, forward_dist_prefix_sum[i], - reverse_dist_prefix_sum[i], m_compressed_edge_container.GetPositionForID(edge_id_1), + current_edge_target_coordinate_id, forward_data.name_id, + m_compressed_edge_container.GetPositionForID(edge_id_1), + m_compressed_edge_container.GetPositionForID(edge_id_2), false, INVALID_COMPONENTID, i, forward_data.travel_mode, reverse_data.travel_mode); m_edge_based_node_is_startpoint.push_back(forward_data.startpoint || reverse_data.startpoint); current_edge_source_coordinate_id = current_edge_target_coordinate_id; - BOOST_ASSERT(m_edge_based_node_list.back().IsCompressed()); - BOOST_ASSERT(node_u != m_edge_based_node_list.back().u || node_v != m_edge_based_node_list.back().v); @@ -193,7 +191,6 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI } BOOST_ASSERT(current_edge_source_coordinate_id == node_v); - BOOST_ASSERT(m_edge_based_node_list.back().IsCompressed()); } else { @@ -220,13 +217,16 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI BOOST_ASSERT(forward_data.edge_id != SPECIAL_NODEID || reverse_data.edge_id != SPECIAL_NODEID); + m_compressed_edge_container.AddUncompressedEdge(edge_id_1, node_v, forward_data.distance); + m_compressed_edge_container.AddUncompressedEdge(edge_id_2, node_u, reverse_data.distance); + m_edge_based_node_list.emplace_back( forward_data.edge_id, reverse_data.edge_id, node_u, node_v, forward_data.name_id, - forward_data.distance, reverse_data.distance, 0, 0, SPECIAL_EDGEID, false, - INVALID_COMPONENTID, 0, forward_data.travel_mode, reverse_data.travel_mode); + m_compressed_edge_container.GetPositionForID(edge_id_1), + m_compressed_edge_container.GetPositionForID(edge_id_2), + false, INVALID_COMPONENTID, 0, forward_data.travel_mode, reverse_data.travel_mode); m_edge_based_node_is_startpoint.push_back(forward_data.startpoint || reverse_data.startpoint); - BOOST_ASSERT(!m_edge_based_node_list.back().IsCompressed()); } } @@ -500,7 +500,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( for (auto target_node : node_based_edges) { const QueryNode &from = m_node_info_list[previous]; - const QueryNode &to = m_node_info_list[target_node.first]; + const QueryNode &to = m_node_info_list[target_node.node_id]; const double segment_length = util::coordinate_calculation::greatCircleDistance( from.lat, from.lon, to.lat, to.lon); @@ -510,9 +510,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( edge_segment_file.write(reinterpret_cast(&segment_length), sizeof(segment_length)); edge_segment_file.write( - reinterpret_cast(&target_node.second), - sizeof(target_node.second)); - previous = target_node.first; + reinterpret_cast(&target_node.weight), + sizeof(target_node.weight)); + previous = target_node.node_id; } } else @@ -1131,17 +1131,17 @@ QueryNode EdgeBasedGraphFactory::getRepresentativeCoordinate(const NodeID src, for (auto itr = geometry.rbegin(), end = geometry.rend(); itr != end; ++itr) { const auto compressed_node = *itr; - cur = util::FixedPointCoordinate(m_node_info_list[compressed_node.first].lat, - m_node_info_list[compressed_node.first].lon); + cur = util::FixedPointCoordinate(m_node_info_list[compressed_node.node_id].lat, + m_node_info_list[compressed_node.node_id].lon); this_dist = util::coordinate_calculation::haversineDistance(prev, cur); if (dist + this_dist > DESIRED_SEGMENT_LENGTH) { - return selectBestCandidate(compressed_node.first, dist + this_dist, prev_id, + return selectBestCandidate(compressed_node.node_id, dist + this_dist, prev_id, dist); } dist += this_dist; prev = cur; - prev_id = compressed_node.first; + prev_id = compressed_node.node_id; } cur = util::FixedPointCoordinate(m_node_info_list[src].lat, m_node_info_list[src].lon); this_dist = util::coordinate_calculation::haversineDistance(prev, cur); @@ -1152,17 +1152,17 @@ QueryNode EdgeBasedGraphFactory::getRepresentativeCoordinate(const NodeID src, for (auto itr = geometry.begin(), end = geometry.end(); itr != end; ++itr) { const auto compressed_node = *itr; - cur = util::FixedPointCoordinate(m_node_info_list[compressed_node.first].lat, - m_node_info_list[compressed_node.first].lon); + cur = util::FixedPointCoordinate(m_node_info_list[compressed_node.node_id].lat, + m_node_info_list[compressed_node.node_id].lon); this_dist = util::coordinate_calculation::haversineDistance(prev, cur); if (dist + this_dist > DESIRED_SEGMENT_LENGTH) { - return selectBestCandidate(compressed_node.first, dist + this_dist, prev_id, + return selectBestCandidate(compressed_node.node_id, dist + this_dist, prev_id, dist); } dist += this_dist; prev = cur; - prev_id = compressed_node.first; + prev_id = compressed_node.node_id; } cur = util::FixedPointCoordinate(m_node_info_list[tgt].lat, m_node_info_list[tgt].lon); this_dist = util::coordinate_calculation::haversineDistance(prev, cur); diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 77baeea43..be441d464 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -522,7 +522,6 @@ Extractor::BuildEdgeExpandedGraph(std::vector &internal_to_external_n std::const_pointer_cast(restriction_map), internal_to_external_node_map, speed_profile); - compressed_edge_container.SerializeInternalVector(config.geometry_output_path); edge_based_graph_factory.Run(config.edge_output_path, lua_state, config.edge_segment_lookup_path, config.edge_penalty_path, @@ -532,6 +531,12 @@ Extractor::BuildEdgeExpandedGraph(std::vector &internal_to_external_n config.debug_turns_path #endif ); + + // Note: this needs to be done *after* the edge-based-graph-factory runs, + // becase it will insert all the uncompressable segments into the compressed + // edge container (necessary for later use) + compressed_edge_container.SerializeInternalVector(config.geometry_output_path); + lua_close(lua_state); edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list); diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 663b3a688..0bcd1a094 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -2,6 +2,7 @@ #include "util/range_table.hpp" #include "contractor/query_edge.hpp" #include "extractor/query_node.hpp" +#include "extractor/compressed_edge_container.hpp" #include "util/shared_memory_vector_wrapper.hpp" #include "util/static_graph.hpp" #include "util/static_rtree.hpp" @@ -326,7 +327,7 @@ int Storage::Run() boost::iostreams::seek(geometry_input_stream, number_of_geometries_indices * sizeof(unsigned), BOOST_IOS::cur); geometry_input_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned)); - shared_layout_ptr->SetBlockSize(SharedDataLayout::GEOMETRIES_LIST, + shared_layout_ptr->SetBlockSize(SharedDataLayout::GEOMETRIES_LIST, number_of_compressed_geometries); // allocate shared memory block util::SimpleLogger().Write() << "allocating shared memory of " @@ -445,7 +446,7 @@ int Storage::Run() (char *)geometries_index_ptr, shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_INDEX)); } - unsigned *geometries_list_ptr = shared_layout_ptr->GetBlockPtr( + extractor::CompressedEdgeContainer::CompressedEdge *geometries_list_ptr = shared_layout_ptr->GetBlockPtr( shared_memory_ptr, SharedDataLayout::GEOMETRIES_LIST); geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned)); diff --git a/unit_tests/util/static_rtree.cpp b/unit_tests/util/static_rtree.cpp index 7b976e4d6..6d4cc1171 100644 --- a/unit_tests/util/static_rtree.cpp +++ b/unit_tests/util/static_rtree.cpp @@ -7,6 +7,9 @@ #include "util/rectangle.hpp" #include "util/exception.hpp" +#include "engine/datafacade/datafacade_base.hpp" +#include "contractor/query_edge.hpp" + #include #include #include @@ -24,6 +27,7 @@ #include #include + BOOST_AUTO_TEST_SUITE(static_rtree) using namespace osrm; @@ -40,6 +44,83 @@ using TestStaticRTree = StaticRTree; using MiniStaticRTree = StaticRTree, false, 2, 3>; +template class MockDataFacadeT final : public osrm::engine::datafacade::BaseDataFacade +{ + private: + EdgeDataT foo; + public: + unsigned GetNumberOfNodes() const { return 0; } + unsigned GetNumberOfEdges() const { return 0; } + unsigned GetOutDegree(const NodeID /* n */) const { return 0; } + NodeID GetTarget(const EdgeID /* e */) const { return SPECIAL_NODEID; } + const EdgeDataT &GetEdgeData(const EdgeID /* e */) const { + return foo; + } + EdgeID BeginEdges(const NodeID /* n */) const { return SPECIAL_EDGEID; } + EdgeID EndEdges(const NodeID /* n */) const { return SPECIAL_EDGEID; } + osrm::engine::datafacade::EdgeRange GetAdjacentEdgeRange(const NodeID /* node */) const { + return irange(static_cast(0),static_cast(0)); + } + EdgeID FindEdge(const NodeID /* from */, const NodeID /* to */) const { return SPECIAL_EDGEID; } + EdgeID FindEdgeInEitherDirection(const NodeID /* from */, const NodeID /* to */) const { return SPECIAL_EDGEID; } + EdgeID + FindEdgeIndicateIfReverse(const NodeID /* from */, const NodeID /* to */, bool & /* result */) const { return SPECIAL_EDGEID; } + util::FixedPointCoordinate GetCoordinateOfNode(const unsigned /* id */) const { + FixedPointCoordinate foo(0,0); + return foo; + } + bool EdgeIsCompressed(const unsigned /* id */) const { return false; } + unsigned GetGeometryIndexForEdgeID(const unsigned /* id */) const { return SPECIAL_NODEID; } + void GetUncompressedGeometry(const EdgeID /* id */, + std::vector &/* result_nodes */) const {} + void GetUncompressedWeights(const EdgeID /* id */, + std::vector & /* result_weights */) const {} + extractor::TurnInstruction GetTurnInstructionForEdgeID(const unsigned /* id */) const { + return osrm::extractor::TurnInstruction::NoTurn; + } + extractor::TravelMode GetTravelModeForEdgeID(const unsigned /* id */) const + { + return TRAVEL_MODE_DEFAULT; + } + std::vector::RTreeLeaf> GetEdgesInBox(const util::FixedPointCoordinate & /* south_west */, + const util::FixedPointCoordinate & /*north_east */) { + std::vector::RTreeLeaf> foo; + return foo; + } + std::vector + NearestPhantomNodesInRange(const util::FixedPointCoordinate /* input_coordinate */, + const float /* max_distance */, + const int /* bearing = 0 */, + const int /* bearing_range = 180 */) { + std::vector foo; + return foo; + } + std::vector + NearestPhantomNodes(const util::FixedPointCoordinate /* input_coordinate */, + const unsigned /* max_results */, + const int /* bearing = 0 */, + const int /* bearing_range = 180 */) { + std::vector foo; + return foo; + } + std::pair NearestPhantomNodeWithAlternativeFromBigComponent( + const util::FixedPointCoordinate /* input_coordinate */, + const int /* bearing = 0 */, + const int /* bearing_range = 180 */) { + std::pair foo; + return foo; + } + unsigned GetCheckSum() const { return 0; } + bool IsCoreNode(const NodeID /* id */) const { return false; } + unsigned GetNameIndexFromEdgeID(const unsigned /* id */) const { return 0; } + std::string get_name_for_id(const unsigned /* name_id */) const { return ""; } + std::size_t GetCoreSize() const { return 0; } + std::string GetTimestamp() const { return ""; } + +}; + +using MockDataFacade = MockDataFacadeT; + // Choosen by a fair W20 dice roll (this value is completely arbitrary) constexpr unsigned RANDOM_SEED = 42; static const int32_t WORLD_MIN_LAT = -90 * COORDINATE_PRECISION; @@ -417,7 +498,8 @@ BOOST_AUTO_TEST_CASE(bearing_tests) std::string nodes_path; build_rtree("test_bearing", &fixture, leaves_path, nodes_path); MiniStaticRTree rtree(nodes_path, leaves_path, fixture.coords); - engine::GeospatialQuery query(rtree, fixture.coords); + std::unique_ptr mockfacade_ptr(new MockDataFacade); + engine::GeospatialQuery query(rtree, fixture.coords, *mockfacade_ptr); FixedPointCoordinate input(5.0 * COORDINATE_PRECISION, 5.1 * COORDINATE_PRECISION); @@ -477,7 +559,8 @@ BOOST_AUTO_TEST_CASE(bbox_search_tests) std::string nodes_path; build_rtree("test_bbox", &fixture, leaves_path, nodes_path); MiniStaticRTree rtree(nodes_path, leaves_path, fixture.coords); - engine::GeospatialQuery query(rtree, fixture.coords); + std::unique_ptr mockfacade_ptr(new MockDataFacade); + engine::GeospatialQuery query(rtree, fixture.coords, *mockfacade_ptr); { RectangleInt2D bbox = {static_cast(0.5 * COORDINATE_PRECISION),