diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp index 5492af933..f95e41154 100644 --- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp +++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp @@ -16,6 +16,7 @@ #include "extractor/profile_properties.hpp" #include "extractor/segment_data_container.hpp" #include "extractor/turn_data_container.hpp" +#include "extractor/packed_osm_ids.hpp" #include "contractor/query_graph.hpp" @@ -214,7 +215,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade unsigned m_check_sum; util::vector_view m_coordinate_list; - util::PackedVectorView m_osmnodeid_list; + extractor::PackedOSMIDsView m_osmnodeid_list; util::NameTable m_names_table; util::vector_view m_lane_description_offsets; util::vector_view m_lane_description_masks; diff --git a/include/extractor/edge_based_graph_factory.hpp b/include/extractor/edge_based_graph_factory.hpp index 30f84310a..842b84b10 100644 --- a/include/extractor/edge_based_graph_factory.hpp +++ b/include/extractor/edge_based_graph_factory.hpp @@ -15,6 +15,7 @@ #include "extractor/profile_properties.hpp" #include "extractor/query_node.hpp" #include "extractor/restriction_map.hpp" +#include "extractor/packed_osm_ids.hpp" #include "util/deallocating_vector.hpp" #include "util/guidance/bearing_class.hpp" @@ -74,7 +75,7 @@ class EdgeBasedGraphFactory const std::unordered_set &traffic_lights, std::shared_ptr restriction_map, const std::vector &coordinates, - const util::PackedVector &osm_node_ids, + const extractor::PackedOSMIDs &osm_node_ids, ProfileProperties profile_properties, const util::NameTable &name_table, std::vector &turn_lane_offsets, @@ -131,7 +132,7 @@ class EdgeBasedGraphFactory EdgeID m_max_edge_id; const std::vector &m_coordinates; - const util::PackedVector &m_osm_node_ids; + const extractor::PackedOSMIDs &m_osm_node_ids; std::shared_ptr m_node_based_graph; std::shared_ptr m_restriction_map; diff --git a/include/extractor/extractor.hpp b/include/extractor/extractor.hpp index bb9b5527d..fc49caddb 100644 --- a/include/extractor/extractor.hpp +++ b/include/extractor/extractor.hpp @@ -59,7 +59,7 @@ class Extractor std::pair BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment, std::vector &coordinates, - util::PackedVector &osm_node_ids, + extractor::PackedOSMIDs &osm_node_ids, std::vector &node_based_edge_list, std::vector &node_is_startpoint, std::vector &edge_based_node_weights, @@ -78,7 +78,7 @@ class Extractor LoadNodeBasedGraph(std::unordered_set &barrier_nodes, std::unordered_set &traffic_lights, std::vector &coordinates, - util::PackedVector &osm_node_ids); + extractor::PackedOSMIDs &osm_node_ids); void WriteEdgeBasedGraph(const std::string &output_file_filename, const EdgeID max_edge_id, diff --git a/include/util/graph_loader.hpp b/include/util/graph_loader.hpp index 01e246b73..f570db717 100644 --- a/include/util/graph_loader.hpp +++ b/include/util/graph_loader.hpp @@ -5,6 +5,7 @@ #include "extractor/node_based_edge.hpp" #include "extractor/query_node.hpp" #include "extractor/restriction.hpp" +#include "extractor/packed_osm_ids.hpp" #include "storage/io.hpp" #include "util/exception.hpp" #include "util/fingerprint.hpp" @@ -59,7 +60,7 @@ NodeID loadNodesFromFile(storage::io::FileReader &file_reader, BarrierOutIter barriers, TrafficSignalsOutIter traffic_signals, std::vector &coordinates, - util::PackedVector &osm_node_ids) + extractor::PackedOSMIDs &osm_node_ids) { auto number_of_nodes = file_reader.ReadElementCount64(); Log() << "Importing number_of_nodes new = " << number_of_nodes << " nodes "; diff --git a/include/util/packed_vector.hpp b/include/util/packed_vector.hpp index 4fa77bf48..d35879757 100644 --- a/include/util/packed_vector.hpp +++ b/include/util/packed_vector.hpp @@ -16,33 +16,28 @@ namespace util { namespace detail { -template class PackedVector; +template class PackedVector; } namespace serialization { -template -inline void read(storage::io::FileReader &reader, detail::PackedVector &vec); +template +inline void read(storage::io::FileReader &reader, detail::PackedVector &vec); -template -inline void write(storage::io::FileWriter &writer, const detail::PackedVector &vec); +template +inline void write(storage::io::FileWriter &writer, + const detail::PackedVector &vec); } namespace detail { -/** - * Since OSM node IDs are (at the time of writing) not quite yet overflowing 32 bits, and - * will predictably be containable within 33 bits for a long time, the following packs - * 64-bit OSM IDs as 33-bit numbers within a 64-bit vector. - * - * NOTE: this type is templated for future use, but will require a slight refactor to - * configure BITSIZE and ELEMSIZE - */ -template class PackedVector +template class PackedVector { - static const constexpr std::size_t BITSIZE = 33; - static const constexpr std::size_t ELEMSIZE = 64; - static const constexpr std::size_t PACKSIZE = BITSIZE * ELEMSIZE; + static_assert(sizeof(T) <= sizeof(std::uint64_t), "Maximum size of type T is 8 bytes"); + static_assert(Bits <= sizeof(std::uint64_t)*CHAR_BIT, "Maximum number of bits it 64."); + + static const constexpr std::size_t ELEMSIZE = sizeof(std::uint64_t)*CHAR_BIT; + static const constexpr std::size_t PACKSIZE = Bits * ELEMSIZE; public: using value_type = T; @@ -54,31 +49,31 @@ template class PackedVector */ inline static std::size_t elements_to_blocks(std::size_t elements) { - return std::ceil(static_cast(elements) * BITSIZE / ELEMSIZE); + return std::ceil(static_cast(elements) * Bits / ELEMSIZE); } - void push_back(T incoming_node_id) + void push_back(T data) { - std::uint64_t node_id = static_cast(incoming_node_id); + std::uint64_t node_id = static_cast(data); // mask incoming values, just in case they are > bitsize - const std::uint64_t incoming_mask = static_cast(pow(2, BITSIZE)) - 1; + const std::uint64_t incoming_mask = static_cast(pow(2, Bits)) - 1; node_id = node_id & incoming_mask; - const std::size_t available = (PACKSIZE - BITSIZE * num_elements) % ELEMSIZE; + const std::size_t available = (PACKSIZE - Bits * num_elements) % ELEMSIZE; if (available == 0) { // insert ID at the left side of this element - std::uint64_t at_left = node_id << (ELEMSIZE - BITSIZE); + std::uint64_t at_left = node_id << (ELEMSIZE - Bits); add_last_elem(at_left); } - else if (available >= BITSIZE) + else if (available >= Bits) { // insert ID somewhere in the middle of this element; ID can be contained // entirely within one element - const std::uint64_t shifted = node_id << (available - BITSIZE); + const std::uint64_t shifted = node_id << (available - Bits); replace_last_elem(vec_back() | shifted); } @@ -86,9 +81,9 @@ template class PackedVector { // ID will be split between the end of this element and the beginning // of the next element - const std::uint64_t left = node_id >> (BITSIZE - available); + const std::uint64_t left = node_id >> (Bits - available); - std::uint64_t right = node_id << (ELEMSIZE - (BITSIZE - available)); + std::uint64_t right = node_id << (ELEMSIZE - (Bits - available)); replace_last_elem(vec_back() | left); add_last_elem(right); @@ -105,11 +100,11 @@ template class PackedVector const std::size_t pack_group = trunc(a_index / ELEMSIZE); const std::size_t pack_index = (a_index + ELEMSIZE) % ELEMSIZE; - const std::size_t left_index = (PACKSIZE - BITSIZE * pack_index) % ELEMSIZE; + const std::size_t left_index = (PACKSIZE - Bits * pack_index) % ELEMSIZE; - const bool back_half = pack_index >= BITSIZE; - const std::size_t index = pack_group * BITSIZE + trunc(pack_index / BITSIZE) + - trunc((pack_index - back_half) / 2); + const bool back_half = pack_index >= Bits; + const std::size_t index = + pack_group * Bits + trunc(pack_index / Bits) + trunc((pack_index - back_half) / 2); BOOST_ASSERT(index < vec.size()); const std::uint64_t elem = static_cast(vec.at(index)); @@ -117,25 +112,25 @@ template class PackedVector if (left_index == 0) { // ID is at the far left side of this element - return T{elem >> (ELEMSIZE - BITSIZE)}; + return T{elem >> (ELEMSIZE - Bits)}; } - else if (left_index >= BITSIZE) + else if (left_index >= Bits) { // ID is entirely contained within this element - const std::uint64_t at_right = elem >> (left_index - BITSIZE); - const std::uint64_t left_mask = static_cast(pow(2, BITSIZE)) - 1; + const std::uint64_t at_right = elem >> (left_index - Bits); + const std::uint64_t left_mask = static_cast(pow(2, Bits)) - 1; return T{at_right & left_mask}; } else { // ID is split between this and the next element const std::uint64_t left_mask = static_cast(pow(2, left_index)) - 1; - const std::uint64_t left_side = (elem & left_mask) << (BITSIZE - left_index); + const std::uint64_t left_side = (elem & left_mask) << (Bits - left_index); BOOST_ASSERT(index < vec.size() - 1); const std::uint64_t next_elem = static_cast(vec.at(index + 1)); - const std::uint64_t right_side = next_elem >> (ELEMSIZE - (BITSIZE - left_index)); + const std::uint64_t right_side = next_elem >> (ELEMSIZE - (Bits - left_index)); return T{left_side | right_side}; } } @@ -163,14 +158,14 @@ template class PackedVector std::size_t capacity() const { - return std::floor(static_cast(vec.capacity()) * ELEMSIZE / BITSIZE); + return std::floor(static_cast(vec.capacity()) * ELEMSIZE / Bits); } - friend void serialization::read(storage::io::FileReader &reader, - detail::PackedVector &vec); + friend void serialization::read(storage::io::FileReader &reader, + PackedVector &vec); - friend void serialization::write(storage::io::FileWriter &writer, - const detail::PackedVector &vec); + friend void serialization::write(storage::io::FileWriter &writer, + const PackedVector &vec); private: util::ViewOrVector vec; @@ -218,8 +213,8 @@ template class PackedVector }; } -template using PackedVector = detail::PackedVector; -template using PackedVectorView = detail::PackedVector; +template using PackedVector = detail::PackedVector; +template using PackedVectorView = detail::PackedVector; } } diff --git a/include/util/serialization.hpp b/include/util/serialization.hpp index f0c084a2b..95fba929a 100644 --- a/include/util/serialization.hpp +++ b/include/util/serialization.hpp @@ -14,15 +14,15 @@ namespace util { namespace serialization { -template -inline void read(storage::io::FileReader &reader, detail::PackedVector &vec) +template +inline void read(storage::io::FileReader &reader, detail::PackedVector &vec) { vec.num_elements = reader.ReadOne(); storage::serialization::read(reader, vec.vec); } -template -inline void write(storage::io::FileWriter &writer, const detail::PackedVector &vec) +template +inline void write(storage::io::FileWriter &writer, const detail::PackedVector &vec) { writer.WriteOne(vec.num_elements); storage::serialization::write(writer, vec.vec); diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index d4f016410..d47bb6b1f 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -42,7 +42,7 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory( const std::unordered_set &traffic_lights, std::shared_ptr restriction_map, const std::vector &coordinates, - const util::PackedVector &osm_node_ids, + const extractor::PackedOSMIDs &osm_node_ids, ProfileProperties profile_properties, const util::NameTable &name_table, std::vector &turn_lane_offsets, diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 7a0972395..6f1a41290 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -263,7 +263,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) std::vector node_is_startpoint; std::vector edge_based_node_weights; std::vector coordinates; - util::PackedVector osm_node_ids; + extractor::PackedOSMIDs osm_node_ids; auto graph_size = BuildEdgeExpandedGraph(scripting_environment, coordinates, @@ -407,7 +407,7 @@ std::shared_ptr Extractor::LoadNodeBasedGraph(std::unordered_set &barriers, std::unordered_set &traffic_signals, std::vector &coordiantes, - util::PackedVector &osm_node_ids) + extractor::PackedOSMIDs &osm_node_ids) { storage::io::FileReader file_reader(config.output_file_name, storage::io::FileReader::VerifyFingerprint); @@ -439,7 +439,7 @@ Extractor::LoadNodeBasedGraph(std::unordered_set &barriers, std::pair Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment, std::vector &coordinates, - util::PackedVector &osm_node_ids, + extractor::PackedOSMIDs &osm_node_ids, std::vector &node_based_edge_list, std::vector &node_is_startpoint, std::vector &edge_based_node_weights, diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 13e72983f..b4c84fc44 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -19,6 +19,7 @@ #include "extractor/profile_properties.hpp" #include "extractor/query_node.hpp" #include "extractor/travel_mode.hpp" +#include "extractor/packed_osm_ids.hpp" #include "partition/cell_storage.hpp" #include "partition/edge_based_graph_reader.hpp" @@ -326,7 +327,7 @@ void Storage::PopulateLayout(DataLayout &layout) // number of items: layout.SetBlockSize( DataLayout::OSM_NODE_ID_LIST, - util::PackedVectorView::elements_to_blocks(coordinate_list_size)); + extractor::PackedOSMIDsView::elements_to_blocks(coordinate_list_size)); } // load geometries sizes @@ -694,7 +695,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) layout.GetBlockPtr(memory_ptr, DataLayout::OSM_NODE_ID_LIST); util::vector_view coordinates( coordinates_ptr, layout.num_entries[DataLayout::COORDINATE_LIST]); - util::PackedVectorView osm_node_ids; + extractor::PackedOSMIDsView osm_node_ids; osm_node_ids.reset(osmnodeid_ptr, layout.num_entries[DataLayout::OSM_NODE_ID_LIST]); extractor::files::readNodes(config.nodes_data_path, coordinates, osm_node_ids); diff --git a/src/tools/components.cpp b/src/tools/components.cpp index d6cc239aa..fc9ccdb47 100644 --- a/src/tools/components.cpp +++ b/src/tools/components.cpp @@ -33,7 +33,7 @@ using TarjanEdge = util::static_graph_details::SortableEdgeWithData; std::size_t loadGraph(const std::string &path, std::vector &coordinate_list, - util::PackedVector &osm_node_ids, + extractor::PackedOSMIDs &osm_node_ids, std::vector &graph_edge_list) { storage::io::FileReader file_reader(path, storage::io::FileReader::VerifyFingerprint); @@ -142,7 +142,7 @@ int main(int argc, char *argv[]) std::vector graph_edge_list; std::vector coordinate_list; - util::PackedVector osm_node_ids; + extractor::PackedOSMIDs osm_node_ids; auto number_of_nodes = tools::loadGraph(inpath, coordinate_list, osm_node_ids, graph_edge_list); tbb::parallel_sort(graph_edge_list.begin(), graph_edge_list.end()); diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index f28487a70..d8b1d643a 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -134,7 +134,7 @@ updateSegmentData(const UpdaterConfig &config, extractor::SegmentDataContainer &segment_data) { std::vector coordinates; - util::PackedVector osm_node_ids; + extractor::PackedOSMIDs osm_node_ids; extractor::files::readNodes(config.node_based_graph_path, coordinates, osm_node_ids); // vector to count used speeds for logging diff --git a/unit_tests/util/packed_vector.cpp b/unit_tests/util/packed_vector.cpp index e4310a9b1..f7d8ddaaa 100644 --- a/unit_tests/util/packed_vector.cpp +++ b/unit_tests/util/packed_vector.cpp @@ -12,7 +12,7 @@ using namespace osrm::util; // Verify that the packed vector behaves as expected BOOST_AUTO_TEST_CASE(insert_and_retrieve_packed_test) { - PackedVector packed_ids; + PackedVector packed_ids; std::vector original_ids; const constexpr std::size_t num_test_cases = 399; @@ -33,7 +33,7 @@ BOOST_AUTO_TEST_CASE(insert_and_retrieve_packed_test) BOOST_AUTO_TEST_CASE(packed_vector_capacity_test) { - PackedVector packed_vec; + PackedVector packed_vec; const std::size_t original_size = packed_vec.capacity(); std::vector dummy_vec;