diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 08b56be31..2bd535093 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -125,11 +125,11 @@ int Prepare::Run() namespace std { -template <> struct hash> +template <> struct hash> { - std::size_t operator()(const std::pair &k) const + std::size_t operator()(const std::pair &k) const { - return k.first ^ (k.second << 12); + return OSMNodeID_to_uint64_t(k.first) ^ (OSMNodeID_to_uint64_t(k.second) << 12); } }; } @@ -172,7 +172,7 @@ std::size_t Prepare::LoadEdgeExpandedGraph(std::string const &edge_based_graph_f edge_based_edge_list.resize(number_of_edges); SimpleLogger().Write() << "Reading " << number_of_edges << " edges from the edge based graph"; - std::unordered_map, unsigned> segment_speed_lookup; + std::unordered_map, unsigned> segment_speed_lookup; if (update_edge_weights) { @@ -180,12 +180,12 @@ std::size_t Prepare::LoadEdgeExpandedGraph(std::string const &edge_based_graph_f << segment_speed_filename; io::CSVReader<3> csv_in(segment_speed_filename); csv_in.set_header("from_node", "to_node", "speed"); - unsigned from_node_id; - unsigned to_node_id; + uint64_t from_node_id; + uint64_t to_node_id; unsigned speed; while (csv_in.read_row(from_node_id, to_node_id, speed)) { - segment_speed_lookup[std::pair(from_node_id, to_node_id)] = speed; + segment_speed_lookup[std::make_pair(OSMNodeID(from_node_id), OSMNodeID(to_node_id))] = speed; } } @@ -210,10 +210,10 @@ std::size_t Prepare::LoadEdgeExpandedGraph(std::string const &edge_based_graph_f unsigned num_osm_nodes = 0; edge_segment_input_stream.read(reinterpret_cast(&num_osm_nodes), sizeof(num_osm_nodes)); - NodeID previous_osm_node_id; + OSMNodeID previous_osm_node_id; edge_segment_input_stream.read(reinterpret_cast(&previous_osm_node_id), sizeof(previous_osm_node_id)); - NodeID this_osm_node_id; + OSMNodeID this_osm_node_id; double segment_length; int segment_weight; --num_osm_nodes; @@ -227,7 +227,7 @@ std::size_t Prepare::LoadEdgeExpandedGraph(std::string const &edge_based_graph_f sizeof(segment_weight)); auto speed_iter = segment_speed_lookup.find( - std::pair(previous_osm_node_id, this_osm_node_id)); + std::make_pair(previous_osm_node_id, this_osm_node_id)); if (speed_iter != segment_speed_lookup.end()) { // This sets the segment weight using the same formula as the diff --git a/data_structures/external_memory_node.cpp b/data_structures/external_memory_node.cpp index 72b8198f9..d144f52a9 100644 --- a/data_structures/external_memory_node.cpp +++ b/data_structures/external_memory_node.cpp @@ -31,7 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include ExternalMemoryNode::ExternalMemoryNode( - int lat, int lon, unsigned int node_id, bool barrier, bool traffic_lights) + int lat, int lon, OSMNodeID node_id, bool barrier, bool traffic_lights) : QueryNode(lat, lon, node_id), barrier(barrier), traffic_lights(traffic_lights) { } @@ -40,13 +40,13 @@ ExternalMemoryNode::ExternalMemoryNode() : barrier(false), traffic_lights(false) ExternalMemoryNode ExternalMemoryNode::min_value() { - return ExternalMemoryNode(0, 0, 0, false, false); + return ExternalMemoryNode(0, 0, MIN_OSM_NODEID, false, false); } ExternalMemoryNode ExternalMemoryNode::max_value() { return ExternalMemoryNode(std::numeric_limits::max(), std::numeric_limits::max(), - std::numeric_limits::max(), false, false); + MAX_OSM_NODEID, false, false); } bool ExternalMemoryNodeSTXXLCompare::operator()(const ExternalMemoryNode &left, diff --git a/data_structures/external_memory_node.hpp b/data_structures/external_memory_node.hpp index 83b88e759..a48d1a13a 100644 --- a/data_structures/external_memory_node.hpp +++ b/data_structures/external_memory_node.hpp @@ -34,7 +34,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. struct ExternalMemoryNode : QueryNode { - ExternalMemoryNode(int lat, int lon, NodeID id, bool barrier, bool traffic_light); + ExternalMemoryNode(int lat, int lon, OSMNodeID id, bool barrier, bool traffic_light); ExternalMemoryNode(); diff --git a/data_structures/import_edge.hpp b/data_structures/import_edge.hpp index 8c5325cfa..af792ee3f 100644 --- a/data_structures/import_edge.hpp +++ b/data_structures/import_edge.hpp @@ -59,6 +59,25 @@ struct NodeBasedEdge TravelMode travel_mode : 4; }; +struct NodeBasedEdgeWithOSM : NodeBasedEdge +{ + explicit NodeBasedEdgeWithOSM(OSMNodeID source, + OSMNodeID target, + NodeID name_id, + EdgeWeight weight, + bool forward, + bool backward, + bool roundabout, + bool access_restricted, + TravelMode travel_mode, + bool is_split) + : NodeBasedEdge(SPECIAL_NODEID, SPECIAL_NODEID, name_id, weight, forward, backward, roundabout, access_restricted, travel_mode, is_split), + osm_source_id(source), osm_target_id(target) {} + + OSMNodeID osm_source_id; + OSMNodeID osm_target_id; +}; + struct EdgeBasedEdge { diff --git a/data_structures/node_based_graph.hpp b/data_structures/node_based_graph.hpp index efdacb998..998e64196 100644 --- a/data_structures/node_based_graph.hpp +++ b/data_structures/node_based_graph.hpp @@ -75,7 +75,7 @@ using NodeBasedDynamicGraph = DynamicGraph; /// The since DynamicGraph expects directed edges, we need to insert /// two edges for undirected edges. inline std::shared_ptr -NodeBasedDynamicGraphFromEdges(int number_of_nodes, const std::vector &input_edge_list) +NodeBasedDynamicGraphFromEdges(std::size_t number_of_nodes, const std::vector &input_edge_list) { auto edges_list = directedEdgesFromCompressed(input_edge_list, [](NodeBasedDynamicGraph::InputEdge& output_edge, const NodeBasedEdge& input_edge) diff --git a/data_structures/node_id.hpp b/data_structures/node_id.hpp index 4d6fff0b2..0671a4d9d 100644 --- a/data_structures/node_id.hpp +++ b/data_structures/node_id.hpp @@ -32,10 +32,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. struct Cmp { - using value_type = NodeID; - bool operator()(const NodeID left, const NodeID right) const { return left < right; } - value_type max_value() { return 0xffffffff; } - value_type min_value() { return 0x0; } + using value_type = OSMNodeID; + bool operator()(const value_type left, const value_type right) const { return left < right; } + value_type max_value() { return MAX_OSM_NODEID; } + value_type min_value() { return MIN_OSM_NODEID; } }; #endif // NODE_ID_HPP diff --git a/data_structures/query_node.hpp b/data_structures/query_node.hpp index f3e990476..0f32a536f 100644 --- a/data_structures/query_node.hpp +++ b/data_structures/query_node.hpp @@ -38,32 +38,32 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. struct QueryNode { - using key_type = NodeID; // type of NodeID + using key_type = OSMNodeID; // type of NodeID using value_type = int; // type of lat,lons - explicit QueryNode(int lat, int lon, NodeID node_id) : lat(lat), lon(lon), node_id(node_id) {} + explicit QueryNode(int lat, int lon, OSMNodeID node_id) : lat(lat), lon(lon), node_id(node_id) {} QueryNode() : lat(std::numeric_limits::max()), lon(std::numeric_limits::max()), - node_id(std::numeric_limits::max()) + node_id(SPECIAL_OSM_NODEID) { } int lat; int lon; - NodeID node_id; + OSMNodeID node_id; static QueryNode min_value() { return QueryNode(static_cast(-90 * COORDINATE_PRECISION), static_cast(-180 * COORDINATE_PRECISION), - std::numeric_limits::min()); + MIN_OSM_NODEID); } static QueryNode max_value() { return QueryNode(static_cast(90 * COORDINATE_PRECISION), static_cast(180 * COORDINATE_PRECISION), - std::numeric_limits::max()); + MAX_OSM_NODEID); } value_type operator[](const std::size_t n) const diff --git a/data_structures/restriction.hpp b/data_structures/restriction.hpp index a59fb1e26..ecab4f921 100644 --- a/data_structures/restriction.hpp +++ b/data_structures/restriction.hpp @@ -36,8 +36,8 @@ struct TurnRestriction { union WayOrNode { - NodeID node; - EdgeID way; + OSMNodeID_weak node; + OSMEdgeID_weak way; }; WayOrNode via; WayOrNode from; diff --git a/data_structures/restriction_map.cpp b/data_structures/restriction_map.cpp index 017ee3251..eb685be80 100644 --- a/data_structures/restriction_map.cpp +++ b/data_structures/restriction_map.cpp @@ -33,10 +33,16 @@ RestrictionMap::RestrictionMap(const std::vector &restriction_l // a pair of starting edge and a list of all end nodes for (auto &restriction : restriction_list) { + // This downcasting is OK because when this is called, the node IDs have been + // renumbered into internal values, which should be well under 2^32 + // This will be a problem if we have more than 2^32 actual restrictions + BOOST_ASSERT(restriction.from.node < std::numeric_limits::max()); + BOOST_ASSERT(restriction.via.node < std::numeric_limits::max()); m_restriction_start_nodes.insert(restriction.from.node); m_no_turn_via_node_set.insert(restriction.via.node); - RestrictionSource restriction_source = {restriction.from.node, restriction.via.node}; + // This explicit downcasting is also OK for the same reason. + RestrictionSource restriction_source = {static_cast(restriction.from.node), static_cast(restriction.via.node)}; std::size_t index; auto restriction_iter = m_restriction_map.find(restriction_source); @@ -62,6 +68,7 @@ RestrictionMap::RestrictionMap(const std::vector &restriction_l } } ++m_count; + BOOST_ASSERT(restriction.to.node < std::numeric_limits::max()); m_restriction_bucket_list.at(index) .emplace_back(restriction.to.node, restriction.flags.is_only); } diff --git a/extractor/edge_based_graph_factory.cpp b/extractor/edge_based_graph_factory.cpp index 8eaf5cb9c..165bf2bd3 100644 --- a/extractor/edge_based_graph_factory.cpp +++ b/extractor/edge_based_graph_factory.cpp @@ -518,6 +518,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( BOOST_ASSERT(SPECIAL_NODEID != edge_data1.edge_id); BOOST_ASSERT(SPECIAL_NODEID != edge_data2.edge_id); + + // NOTE: potential overflow here if we hit 2^32 routable edges + BOOST_ASSERT(m_edge_based_edge_list.size() <= std::numeric_limits::max()); m_edge_based_edge_list.emplace_back(edge_data1.edge_id, edge_data2.edge_id, m_edge_based_edge_list.size(), distance, true, false); diff --git a/extractor/extraction_containers.cpp b/extractor/extraction_containers.cpp index 10b9c29fb..725ae25ce 100644 --- a/extractor/extraction_containers.cpp +++ b/extractor/extraction_containers.cpp @@ -42,6 +42,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include #include @@ -171,7 +172,11 @@ void ExtractionContainers::PrepareNodes() auto ref_iter = used_node_id_list.begin(); const auto all_nodes_list_end = all_nodes_list.end(); const auto used_node_id_list_end = used_node_id_list.end(); - auto internal_id = 0u; + // Note: despite being able to handle 64 bit OSM node ids, we can't + // handle > uint32_t actual usable nodes. This should be OK for a while + // because we usually route on a *lot* less than 2^32 of the OSM + // graph nodes. + std::size_t internal_id = 0; // compute the intersection of nodes that were referenced and nodes we actually have while (node_iter != all_nodes_list_end && ref_iter != used_node_id_list_end) @@ -187,11 +192,15 @@ void ExtractionContainers::PrepareNodes() continue; } BOOST_ASSERT(node_iter->node_id == *ref_iter); - external_to_internal_node_id_map[*ref_iter] = internal_id++; + external_to_internal_node_id_map[*ref_iter] = static_cast(internal_id++); node_iter++; ref_iter++; } - max_internal_node_id = internal_id; + if (internal_id > std::numeric_limits::max()) + { + throw osrm::exception("There are too many nodes remaining after filtering, OSRM only supports 2^32 unique nodes"); + } + max_internal_node_id = boost::numeric_cast(internal_id); TIMER_STOP(id_map); std::cout << "ok, after " << TIMER_SEC(id_map) << "s" << std::endl; @@ -202,7 +211,7 @@ void ExtractionContainers::PrepareEdges(lua_State *segment_state) // Sort edges by start. std::cout << "[extractor] Sorting edges by start ... " << std::flush; TIMER_START(sort_edges_by_start); - stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), stxxl_memory); + stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByOSMStartID(), stxxl_memory); TIMER_STOP(sort_edges_by_start); std::cout << "ok, after " << TIMER_SEC(sort_edges_by_start) << "s" << std::endl; @@ -217,21 +226,21 @@ void ExtractionContainers::PrepareEdges(lua_State *segment_state) while (edge_iterator != all_edges_list_end && node_iterator != all_nodes_list_end) { - if (edge_iterator->result.source < node_iterator->node_id) + if (edge_iterator->result.osm_source_id < node_iterator->node_id) { SimpleLogger().Write(LogLevel::logWARNING) << "Found invalid node reference " << edge_iterator->result.source; edge_iterator->result.source = SPECIAL_NODEID; ++edge_iterator; continue; } - if (edge_iterator->result.source > node_iterator->node_id) + if (edge_iterator->result.osm_source_id > node_iterator->node_id) { node_iterator++; continue; } // remove loops - if (edge_iterator->result.source == edge_iterator->result.target) + if (edge_iterator->result.osm_source_id == edge_iterator->result.osm_target_id) { edge_iterator->result.source = SPECIAL_NODEID; edge_iterator->result.target = SPECIAL_NODEID; @@ -239,7 +248,7 @@ void ExtractionContainers::PrepareEdges(lua_State *segment_state) continue; } - BOOST_ASSERT(edge_iterator->result.source == node_iterator->node_id); + BOOST_ASSERT(edge_iterator->result.osm_source_id == node_iterator->node_id); // assign new node id auto id_iter = external_to_internal_node_id_map.find(node_iterator->node_id); @@ -258,6 +267,7 @@ void ExtractionContainers::PrepareEdges(lua_State *segment_state) SimpleLogger().Write(LogLevel::logWARNING) << "Found invalid node reference " << edge.result.source; edge.result.source = SPECIAL_NODEID; + edge.result.osm_source_id = SPECIAL_OSM_NODEID; }; std::for_each(edge_iterator, all_edges_list_end, markSourcesInvalid); TIMER_STOP(set_start_coords); @@ -266,7 +276,7 @@ void ExtractionContainers::PrepareEdges(lua_State *segment_state) // Sort Edges by target std::cout << "[extractor] Sorting edges by target ... " << std::flush; TIMER_START(sort_edges_by_target); - stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(), + stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByOSMTargetID(), stxxl_memory); TIMER_STOP(sort_edges_by_target); std::cout << "ok, after " << TIMER_SEC(sort_edges_by_target) << "s" << std::endl; @@ -288,20 +298,20 @@ void ExtractionContainers::PrepareEdges(lua_State *segment_state) continue; } - if (edge_iterator->result.target < node_iterator->node_id) + if (edge_iterator->result.osm_target_id < node_iterator->node_id) { - SimpleLogger().Write(LogLevel::logWARNING) << "Found invalid node reference " << edge_iterator->result.target; + SimpleLogger().Write(LogLevel::logWARNING) << "Found invalid node reference " << OSMNodeID_to_uint64_t(edge_iterator->result.osm_target_id); edge_iterator->result.target = SPECIAL_NODEID; ++edge_iterator; continue; } - if (edge_iterator->result.target > node_iterator->node_id) + if (edge_iterator->result.osm_target_id > node_iterator->node_id) { ++node_iterator; continue; } - BOOST_ASSERT(edge_iterator->result.target == node_iterator->node_id); + BOOST_ASSERT(edge_iterator->result.osm_target_id == node_iterator->node_id); BOOST_ASSERT(edge_iterator->weight_data.speed >= 0); BOOST_ASSERT(edge_iterator->source_coordinate.lat != std::numeric_limits::min()); BOOST_ASSERT(edge_iterator->source_coordinate.lon != std::numeric_limits::min()); @@ -373,7 +383,7 @@ void ExtractionContainers::PrepareEdges(lua_State *segment_state) // Sort edges by start. std::cout << "[extractor] Sorting edges by renumbered start ... " << std::flush; TIMER_START(sort_edges_by_renumbered_start); - stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartThenTargetID(), stxxl_memory); + stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByInternalStartThenInternalTargetID(), stxxl_memory); TIMER_STOP(sort_edges_by_renumbered_start); std::cout << "ok, after " << TIMER_SEC(sort_edges_by_renumbered_start) << "s" << std::endl; @@ -464,13 +474,14 @@ void ExtractionContainers::PrepareEdges(lua_State *segment_state) void ExtractionContainers::WriteEdges(std::ofstream& file_out_stream) const { - std::cout << "[extractor] Writing used egdes ... " << std::flush; + std::cout << "[extractor] Writing used edges ... " << std::flush; TIMER_START(write_edges); // Traverse list of edges and nodes in parallel and set target coord - unsigned number_of_used_edges = 0; + std::size_t used_edges_counter = 0; + unsigned used_edges_counter_buffer = 0; auto start_position = file_out_stream.tellp(); - file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned)); + file_out_stream.write((char *)&used_edges_counter_buffer, sizeof(unsigned)); for (const auto& edge : all_edges_list) { @@ -479,18 +490,29 @@ void ExtractionContainers::WriteEdges(std::ofstream& file_out_stream) const continue; } - file_out_stream.write((char*) &edge.result, sizeof(NodeBasedEdge)); - number_of_used_edges++; + // IMPORTANT: here, we're using slicing to only write the data from the base + // class of NodeBasedEdgeWithOSM + NodeBasedEdge tmp = edge.result; + file_out_stream.write((char*) &tmp, sizeof(NodeBasedEdge)); + used_edges_counter++; + } + + if (used_edges_counter > std::numeric_limits::max()) + { + throw osrm::exception("There are too many edges, OSRM only supports 2^32"); } TIMER_STOP(write_edges); std::cout << "ok, after " << TIMER_SEC(write_edges) << "s" << std::endl; std::cout << "[extractor] setting number of edges ... " << std::flush; + + used_edges_counter_buffer = boost::numeric_cast(used_edges_counter); + file_out_stream.seekp(start_position); - file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned)); + file_out_stream.write((char *)&used_edges_counter_buffer, sizeof(unsigned)); std::cout << "ok" << std::endl; - SimpleLogger().Write() << "Processed " << number_of_used_edges << " edges"; + SimpleLogger().Write() << "Processed " << used_edges_counter << " edges"; } void ExtractionContainers::WriteNodes(std::ofstream& file_out_stream) const @@ -589,13 +611,13 @@ void ExtractionContainers::PrepareRestrictions() while (way_start_and_end_iterator != way_start_end_id_list_end && restrictions_iterator != restrictions_list_end) { - if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.from.way) + if (way_start_and_end_iterator->way_id < OSMWayID(restrictions_iterator->restriction.from.way)) { ++way_start_and_end_iterator; continue; } - if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.from.way) + if (way_start_and_end_iterator->way_id > OSMWayID(restrictions_iterator->restriction.from.way)) { SimpleLogger().Write(LogLevel::logDEBUG) << "Restriction references invalid way: " << restrictions_iterator->restriction.from.way; restrictions_iterator->restriction.from.node = SPECIAL_NODEID; @@ -604,9 +626,9 @@ void ExtractionContainers::PrepareRestrictions() } BOOST_ASSERT(way_start_and_end_iterator->way_id == - restrictions_iterator->restriction.from.way); + OSMWayID(restrictions_iterator->restriction.from.way)); // we do not remap the via id yet, since we will need it for the to node as well - const NodeID via_node_id = restrictions_iterator->restriction.via.node; + const OSMNodeID via_node_id = OSMNodeID(restrictions_iterator->restriction.via.node); // check if via is actually valid, if not invalidate auto via_id_iter = external_to_internal_node_id_map.find(via_node_id); @@ -618,19 +640,19 @@ void ExtractionContainers::PrepareRestrictions() continue; } - if (way_start_and_end_iterator->first_segment_source_id == via_node_id) + if (OSMNodeID(way_start_and_end_iterator->first_segment_source_id) == via_node_id) { // assign new from node id auto id_iter = external_to_internal_node_id_map.find( - way_start_and_end_iterator->first_segment_target_id); + OSMNodeID(way_start_and_end_iterator->first_segment_target_id)); BOOST_ASSERT(id_iter != external_to_internal_node_id_map.end()); restrictions_iterator->restriction.from.node = id_iter->second; } - else if (way_start_and_end_iterator->last_segment_target_id == via_node_id) + else if (OSMNodeID(way_start_and_end_iterator->last_segment_target_id) == via_node_id) { // assign new from node id auto id_iter = external_to_internal_node_id_map.find( - way_start_and_end_iterator->last_segment_source_id); + OSMNodeID(way_start_and_end_iterator->last_segment_source_id)); BOOST_ASSERT(id_iter != external_to_internal_node_id_map.end()); restrictions_iterator->restriction.from.node = id_iter->second; } @@ -657,7 +679,7 @@ void ExtractionContainers::PrepareRestrictions() while (way_start_and_end_iterator != way_start_end_id_list_end_ && restrictions_iterator != restrictions_list_end_) { - if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.to.way) + if (way_start_and_end_iterator->way_id < OSMWayID(restrictions_iterator->restriction.to.way)) { ++way_start_and_end_iterator; continue; @@ -668,7 +690,7 @@ void ExtractionContainers::PrepareRestrictions() ++restrictions_iterator; continue; } - if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.to.way) + if (way_start_and_end_iterator->way_id > OSMWayID(restrictions_iterator->restriction.to.way)) { SimpleLogger().Write(LogLevel::logDEBUG) << "Restriction references invalid way: " << restrictions_iterator->restriction.to.way; restrictions_iterator->restriction.to.way = SPECIAL_NODEID; @@ -676,25 +698,25 @@ void ExtractionContainers::PrepareRestrictions() continue; } BOOST_ASSERT(way_start_and_end_iterator->way_id == - restrictions_iterator->restriction.to.way); - const NodeID via_node_id = restrictions_iterator->restriction.via.node; + OSMWayID(restrictions_iterator->restriction.to.way)); + const OSMNodeID via_node_id = OSMNodeID(restrictions_iterator->restriction.via.node); // assign new via node id auto via_id_iter = external_to_internal_node_id_map.find(via_node_id); BOOST_ASSERT(via_id_iter != external_to_internal_node_id_map.end()); restrictions_iterator->restriction.via.node = via_id_iter->second; - if (way_start_and_end_iterator->first_segment_source_id == via_node_id) + if (OSMNodeID(way_start_and_end_iterator->first_segment_source_id) == via_node_id) { auto to_id_iter = external_to_internal_node_id_map.find( - way_start_and_end_iterator->first_segment_target_id); + OSMNodeID(way_start_and_end_iterator->first_segment_target_id)); BOOST_ASSERT(to_id_iter != external_to_internal_node_id_map.end()); restrictions_iterator->restriction.to.node = to_id_iter->second; } - else if (way_start_and_end_iterator->last_segment_target_id == via_node_id) + else if (OSMNodeID(way_start_and_end_iterator->last_segment_target_id) == via_node_id) { auto to_id_iter = external_to_internal_node_id_map.find( - way_start_and_end_iterator->last_segment_source_id); + OSMNodeID(way_start_and_end_iterator->last_segment_source_id)); BOOST_ASSERT(to_id_iter != external_to_internal_node_id_map.end()); restrictions_iterator->restriction.to.node = to_id_iter->second; } diff --git a/extractor/extraction_containers.hpp b/extractor/extraction_containers.hpp index 8b7a82930..09a5ef189 100644 --- a/extractor/extraction_containers.hpp +++ b/extractor/extraction_containers.hpp @@ -61,7 +61,7 @@ class ExtractionContainers void WriteEdges(std::ofstream& file_out_stream) const; void WriteNames(const std::string& names_file_name) const; public: - using STXXLNodeIDVector = stxxl::vector; + using STXXLNodeIDVector = stxxl::vector; using STXXLNodeVector = stxxl::vector; using STXXLEdgeVector = stxxl::vector; using STXXLStringVector = stxxl::vector; @@ -74,7 +74,7 @@ class ExtractionContainers STXXLStringVector name_list; STXXLRestrictionsVector restrictions_list; STXXLWayIDStartEndVector way_start_end_id_list; - std::unordered_map external_to_internal_node_id_map; + std::unordered_map external_to_internal_node_id_map; unsigned max_internal_node_id; ExtractionContainers(); diff --git a/extractor/extractor_callbacks.cpp b/extractor/extractor_callbacks.cpp index 14c195bb4..4b0ac6b22 100644 --- a/extractor/extractor_callbacks.cpp +++ b/extractor/extractor_callbacks.cpp @@ -63,7 +63,7 @@ void ExtractorCallbacks::ProcessNode(const osmium::Node &input_node, external_memory.all_nodes_list.push_back( {static_cast(input_node.location().lat() * COORDINATE_PRECISION), static_cast(input_node.location().lon() * COORDINATE_PRECISION), - static_cast(input_node.id()), + OSMNodeID(input_node.id()), result_node.barrier, result_node.traffic_lights}); } @@ -175,7 +175,7 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti std::back_inserter(external_memory.used_node_id_list), [](const osmium::NodeRef &ref) { - return ref.ref(); + return OSMNodeID(ref.ref()); }); const bool is_opposite_way = TRAVEL_MODE_INACCESSIBLE == parsed_way.forward_travel_mode; @@ -189,18 +189,18 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) { external_memory.all_edges_list.push_back(InternalExtractorEdge( - first_node.ref(), last_node.ref(), name_id, + OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), name_id, backward_weight_data, true, false, parsed_way.roundabout, parsed_way.is_access_restricted, parsed_way.backward_travel_mode, false)); }); external_memory.way_start_end_id_list.push_back( - {static_cast(input_way.id()), - static_cast(input_way.nodes().back().ref()), - static_cast(input_way.nodes()[input_way.nodes().size() - 2].ref()), - static_cast(input_way.nodes()[1].ref()), - static_cast(input_way.nodes()[0].ref())}); + {OSMWayID(input_way.id()), + OSMNodeID(input_way.nodes().back().ref()), + OSMNodeID(input_way.nodes()[input_way.nodes().size() - 2].ref()), + OSMNodeID(input_way.nodes()[1].ref()), + OSMNodeID(input_way.nodes()[0].ref())}); } else { @@ -210,7 +210,7 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) { external_memory.all_edges_list.push_back(InternalExtractorEdge( - first_node.ref(), last_node.ref(), name_id, forward_weight_data, + OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), name_id, forward_weight_data, true, !forward_only, parsed_way.roundabout, parsed_way.is_access_restricted, parsed_way.forward_travel_mode, split_edge)); @@ -223,17 +223,17 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) { external_memory.all_edges_list.push_back(InternalExtractorEdge( - first_node.ref(), last_node.ref(), name_id, backward_weight_data, false, + OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), name_id, backward_weight_data, false, true, parsed_way.roundabout, parsed_way.is_access_restricted, parsed_way.backward_travel_mode, true)); }); } external_memory.way_start_end_id_list.push_back( - {static_cast(input_way.id()), - static_cast(input_way.nodes().back().ref()), - static_cast(input_way.nodes()[input_way.nodes().size() - 2].ref()), - static_cast(input_way.nodes()[1].ref()), - static_cast(input_way.nodes()[0].ref())}); + {OSMWayID(input_way.id()), + OSMNodeID(input_way.nodes().back().ref()), + OSMNodeID(input_way.nodes()[input_way.nodes().size() - 2].ref()), + OSMNodeID(input_way.nodes()[1].ref()), + OSMNodeID(input_way.nodes()[0].ref())}); } } diff --git a/extractor/first_and_last_segment_of_way.hpp b/extractor/first_and_last_segment_of_way.hpp index 3a26be7ce..ead8b4c5c 100644 --- a/extractor/first_and_last_segment_of_way.hpp +++ b/extractor/first_and_last_segment_of_way.hpp @@ -36,21 +36,22 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. struct FirstAndLastSegmentOfWay { - EdgeID way_id; - NodeID first_segment_source_id; - NodeID first_segment_target_id; - NodeID last_segment_source_id; - NodeID last_segment_target_id; + OSMWayID way_id; + OSMNodeID first_segment_source_id; + OSMNodeID first_segment_target_id; + OSMNodeID last_segment_source_id; + OSMNodeID last_segment_target_id; + FirstAndLastSegmentOfWay() - : way_id(std::numeric_limits::max()), - first_segment_source_id(std::numeric_limits::max()), - first_segment_target_id(std::numeric_limits::max()), - last_segment_source_id(std::numeric_limits::max()), - last_segment_target_id(std::numeric_limits::max()) + : way_id(SPECIAL_OSM_WAYID), + first_segment_source_id(SPECIAL_OSM_NODEID), + first_segment_target_id(SPECIAL_OSM_NODEID), + last_segment_source_id(SPECIAL_OSM_NODEID), + last_segment_target_id(SPECIAL_OSM_NODEID) { } - FirstAndLastSegmentOfWay(EdgeID w, NodeID fs, NodeID ft, NodeID ls, NodeID lt) + FirstAndLastSegmentOfWay(OSMWayID w, OSMNodeID fs, OSMNodeID ft, OSMNodeID ls, OSMNodeID lt) : way_id(w), first_segment_source_id(fs), first_segment_target_id(ft), last_segment_source_id(ls), last_segment_target_id(lt) { @@ -58,19 +59,19 @@ struct FirstAndLastSegmentOfWay static FirstAndLastSegmentOfWay min_value() { - return {std::numeric_limits::min(), - std::numeric_limits::min(), - std::numeric_limits::min(), - std::numeric_limits::min(), - std::numeric_limits::min()}; + return {MIN_OSM_WAYID, + MIN_OSM_NODEID, + MIN_OSM_NODEID, + MIN_OSM_NODEID, + MIN_OSM_NODEID}; } static FirstAndLastSegmentOfWay max_value() { - return {std::numeric_limits::max(), - std::numeric_limits::max(), - std::numeric_limits::max(), - std::numeric_limits::max(), - std::numeric_limits::max()}; + return {MAX_OSM_WAYID, + MAX_OSM_NODEID, + MAX_OSM_NODEID, + MAX_OSM_NODEID, + MAX_OSM_NODEID}; } }; diff --git a/extractor/internal_extractor_edge.hpp b/extractor/internal_extractor_edge.hpp index 2dc3c3488..bc8842614 100644 --- a/extractor/internal_extractor_edge.hpp +++ b/extractor/internal_extractor_edge.hpp @@ -63,13 +63,13 @@ struct InternalExtractorEdge }; explicit InternalExtractorEdge() - : result(0, 0, 0, 0, false, false, false, false, + : result(MIN_OSM_NODEID, MIN_OSM_NODEID, 0, 0, false, false, false, false, TRAVEL_MODE_INACCESSIBLE, false) { } - explicit InternalExtractorEdge(NodeID source, - NodeID target, + explicit InternalExtractorEdge(OSMNodeID source, + OSMNodeID target, NodeID name_id, WeightData weight_data, bool forward, @@ -78,8 +78,8 @@ struct InternalExtractorEdge bool access_restricted, TravelMode travel_mode, bool is_split) - : result(source, - target, + : result(OSMNodeID(source), + OSMNodeID(target), name_id, 0, forward, @@ -93,7 +93,7 @@ struct InternalExtractorEdge } // data that will be written to disk - NodeBasedEdge result; + NodeBasedEdgeWithOSM result; // intermediate edge weight WeightData weight_data; // coordinate of the source node @@ -101,19 +101,35 @@ struct InternalExtractorEdge // necessary static util functions for stxxl's sorting - static InternalExtractorEdge min_value() + static InternalExtractorEdge min_osm_value() { - return InternalExtractorEdge(0, 0, 0, WeightData(), false, false, false, + return InternalExtractorEdge(MIN_OSM_NODEID, MIN_OSM_NODEID, 0, WeightData(), false, false, false, false, TRAVEL_MODE_INACCESSIBLE, false); } - static InternalExtractorEdge max_value() + static InternalExtractorEdge max_osm_value() { - return InternalExtractorEdge(SPECIAL_NODEID, SPECIAL_NODEID, 0, WeightData(), false, + return InternalExtractorEdge(MAX_OSM_NODEID, MAX_OSM_NODEID, 0, WeightData(), false, false, false, false, TRAVEL_MODE_INACCESSIBLE, false); } + + static InternalExtractorEdge min_internal_value() + { + auto v = min_osm_value(); + v.result.source = 0; + v.result.target = 0; + return v; + } + static InternalExtractorEdge max_internal_value() + { + auto v = max_osm_value(); + v.result.source = std::numeric_limits::max(); + v.result.target = std::numeric_limits::max(); + return v; + } + }; -struct CmpEdgeByStartThenTargetID +struct CmpEdgeByInternalStartThenInternalTargetID { using value_type = InternalExtractorEdge; bool operator()(const InternalExtractorEdge &lhs, const InternalExtractorEdge &rhs) const @@ -123,32 +139,32 @@ struct CmpEdgeByStartThenTargetID (lhs.result.target < rhs.result.target)); } - value_type max_value() { return InternalExtractorEdge::max_value(); } - value_type min_value() { return InternalExtractorEdge::min_value(); } + value_type max_value() { return InternalExtractorEdge::max_internal_value(); } + value_type min_value() { return InternalExtractorEdge::min_internal_value(); } }; -struct CmpEdgeByStartID +struct CmpEdgeByOSMStartID { using value_type = InternalExtractorEdge; bool operator()(const InternalExtractorEdge &lhs, const InternalExtractorEdge &rhs) const { - return lhs.result.source < rhs.result.source; + return lhs.result.osm_source_id < rhs.result.osm_source_id; } - value_type max_value() { return InternalExtractorEdge::max_value(); } - value_type min_value() { return InternalExtractorEdge::min_value(); } + value_type max_value() { return InternalExtractorEdge::max_osm_value(); } + value_type min_value() { return InternalExtractorEdge::min_osm_value(); } }; -struct CmpEdgeByTargetID +struct CmpEdgeByOSMTargetID { using value_type = InternalExtractorEdge; bool operator()(const InternalExtractorEdge &lhs, const InternalExtractorEdge &rhs) const { - return lhs.result.target < rhs.result.target; + return lhs.result.osm_target_id < rhs.result.osm_target_id; } - value_type max_value() { return InternalExtractorEdge::max_value(); } - value_type min_value() { return InternalExtractorEdge::min_value(); } + value_type max_value() { return InternalExtractorEdge::max_osm_value(); } + value_type min_value() { return InternalExtractorEdge::min_osm_value(); } }; #endif // INTERNAL_EXTRACTOR_EDGE_HPP diff --git a/features/step_definitions/data.rb b/features/step_definitions/data.rb index 0f3384323..13ed209bc 100644 --- a/features/step_definitions/data.rb +++ b/features/step_definitions/data.rb @@ -28,7 +28,7 @@ Given /^the node map$/ do |table| raise "*** invalid node name '#{name}', must me alphanumeric" unless name.match /[a-z0-9]/ if name.match /[a-z]/ raise "*** duplicate node '#{name}'" if name_node_hash[name] - add_osm_node name, *table_coord_to_lonlat(ci,ri) + add_osm_node name, *table_coord_to_lonlat(ci,ri), nil else raise "*** duplicate node '#{name}'" if location_hash[name] add_location name, *table_coord_to_lonlat(ci,ri) @@ -43,7 +43,9 @@ Given /^the node locations$/ do |table| name = row['node'] raise "*** duplicate node '#{name}'" if find_node_by_name name if name.match /[a-z]/ - add_osm_node name, row['lon'].to_f, row['lat'].to_f + id = row['id'] + id = id.to_i if id + add_osm_node name, row['lon'].to_f, row['lat'].to_f, id else add_location name, row['lon'].to_f, row['lat'].to_f end diff --git a/features/support/data.rb b/features/support/data.rb index 020a8c6c4..253fe6044 100644 --- a/features/support/data.rb +++ b/features/support/data.rb @@ -123,8 +123,9 @@ def table_coord_to_lonlat ci,ri [@origin[0]+ci*@zoom, @origin[1]-ri*@zoom] end -def add_osm_node name,lon,lat - node = OSM::Node.new make_osm_id, OSM_USER, OSM_TIMESTAMP, lon, lat +def add_osm_node name,lon,lat,id + id = make_osm_id if id == nil + node = OSM::Node.new id, OSM_USER, OSM_TIMESTAMP, lon, lat node << { :name => name } node.uid = OSM_UID osm_db << node diff --git a/features/testbot/64bit.feature b/features/testbot/64bit.feature new file mode 100644 index 000000000..d2d0c8a7c --- /dev/null +++ b/features/testbot/64bit.feature @@ -0,0 +1,23 @@ +@testbot +Feature: Support 64bit node IDs + + # Without 64bit support, this test should fail + Scenario: 64bit overflow conflicts + Given the node locations + | node | lat | lon | id | + | a | 55.660778 | 12.573909 | 1 | + | b | 55.660672 | 12.573693 | 2 | + | c | 55.660128 | 12.572546 | 3 | + | d | 55.660015 | 12.572476 | 4294967297 | + | e | 55.660119 | 12.572325 | 4294967298 | + | x | 55.660818 | 12.574051 | 4294967299 | + | y | 55.660073 | 12.574067 | 4294967300 | + + And the ways + | nodes | + | abc | + | cdec | + + When I route I should get + | from | to | route | turns | + | x | y | abc | head,destination | diff --git a/include/osrm/strong_typedef.hpp b/include/osrm/strong_typedef.hpp new file mode 100644 index 000000000..c2364b053 --- /dev/null +++ b/include/osrm/strong_typedef.hpp @@ -0,0 +1,68 @@ +#ifndef OSRM_STRONG_TYPEDEF_HPP +#define OSRM_STRONG_TYPEDEF_HPP + +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include +#include + +/* Creates strongly typed wrappers around scalar types. + * Useful for stopping accidental assignment of lats to lons, + * etc. Also clarifies what this random "int" value is + * being used for. + */ +#define OSRM_STRONG_TYPEDEF(From, To) \ + class To final { \ + static_assert(std::is_arithmetic(), ""); \ + From x; \ + \ + public: \ + To() = default; \ + explicit To(const From x_) : x(x_) {} \ + explicit operator From&() { return x; } \ + explicit operator const From&() const { return x; } \ + bool operator <(const To &z_) const { return x < static_cast(z_) ; } \ + bool operator >(const To &z_) const { return x > static_cast(z_) ; } \ + bool operator <=(const To &z_) const { return x <= static_cast(z_) ; } \ + bool operator >=(const To &z_) const { return x >= static_cast(z_) ; } \ + bool operator ==(const To &z_) const { return x == static_cast(z_) ; } \ + bool operator !=(const To &z_) const { return x != static_cast(z_) ; } \ + }; \ + inline From To##_to_##From(To to) { return static_cast(to); } \ + namespace std { \ + template <> \ + struct hash \ + { \ + std::size_t operator()(const To& k) const \ + { \ + return std::hash()(static_cast(k)); \ + } \ + }; \ + } + +#endif // OSRM_STRONG_TYPEDEF_HPP diff --git a/typedefs.h b/typedefs.h index 242c4c55d..dc77f39c7 100644 --- a/typedefs.h +++ b/typedefs.h @@ -29,6 +29,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define TYPEDEFS_H #include +#include +#include // Necessary workaround for Windows as VS doesn't implement C99 #ifdef _MSC_VER @@ -38,6 +40,21 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif +// OpenStreetMap node ids are higher than 2^32 +OSRM_STRONG_TYPEDEF(uint64_t, OSMNodeID) +OSRM_STRONG_TYPEDEF(uint32_t, OSMWayID) + +static const OSMNodeID SPECIAL_OSM_NODEID = OSMNodeID(std::numeric_limits::max()); +static const OSMWayID SPECIAL_OSM_WAYID = OSMWayID(std::numeric_limits::max()); + +static const OSMNodeID MAX_OSM_NODEID = OSMNodeID(std::numeric_limits::max()); +static const OSMNodeID MIN_OSM_NODEID = OSMNodeID(std::numeric_limits::min()); +static const OSMWayID MAX_OSM_WAYID = OSMWayID(std::numeric_limits::max()); +static const OSMWayID MIN_OSM_WAYID = OSMWayID(std::numeric_limits::min()); + +using OSMNodeID_weak = std::uint64_t; +using OSMEdgeID_weak = std::uint64_t; + using NodeID = unsigned int; using EdgeID = unsigned int; using EdgeWeight = int; diff --git a/unit_tests/data_structures/static_rtree.cpp b/unit_tests/data_structures/static_rtree.cpp index ed8619617..62295407d 100644 --- a/unit_tests/data_structures/static_rtree.cpp +++ b/unit_tests/data_structures/static_rtree.cpp @@ -211,7 +211,7 @@ template struct RandomGraphFixture { int lat = lat_udist(g); int lon = lon_udist(g); - nodes.emplace_back(QueryNode(lat, lon, i)); + nodes.emplace_back(QueryNode(lat, lon, OSMNodeID(i))); coords->emplace_back(FixedPointCoordinate(lat, lon)); } @@ -251,7 +251,7 @@ struct GraphFixture FixedPointCoordinate c(input_coords[i].first * COORDINATE_PRECISION, input_coords[i].second * COORDINATE_PRECISION); coords->emplace_back(c); - nodes.emplace_back(QueryNode(c.lat, c.lon, i)); + nodes.emplace_back(QueryNode(c.lat, c.lon, OSMNodeID(i))); } for (const auto &pair : input_edges) diff --git a/util/debug_geometry.hpp b/util/debug_geometry.hpp index bfe4afa37..daa7e1074 100644 --- a/util/debug_geometry.hpp +++ b/util/debug_geometry.hpp @@ -35,7 +35,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. inline void DEBUG_GEOMETRY_START(ContractorConfig & /* config */) {} inline void DEBUG_GEOMETRY_EDGE(int /* new_segment_weight */ , double /* segment_length */, - NodeID /* previous_osm_node_id */, NodeID /* this_osm_node_id */) {} + OSMNodeID /* previous_osm_node_id */, OSMNodeID /* this_osm_node_id */) {} inline void DEBUG_GEOMETRY_STOP() {} inline void DEBUG_TURNS_START(const std::string & /* debug_turns_filename */) {} @@ -86,7 +86,7 @@ inline void DEBUG_GEOMETRY_START(const ContractorConfig &config) } } -inline void DEBUG_GEOMETRY_EDGE(int new_segment_weight, double segment_length, NodeID previous_osm_node_id, NodeID this_osm_node_id) +inline void DEBUG_GEOMETRY_EDGE(int new_segment_weight, double segment_length, OSMNodeID previous_osm_node_id, OSMNodeID this_osm_node_id) { if (dg_output_debug_geometry) {