Make PackedVector generic

include/engine/datafacade/contiguous_internalmem_datafacade.hpp

@@ -16,6 +16,7 @@
 #include "extractor/profile_properties.hpp"
 #include "extractor/segment_data_container.hpp"
 #include "extractor/turn_data_container.hpp"
+#include "extractor/packed_osm_ids.hpp"
 
 #include "contractor/query_graph.hpp"
 
@@ -214,7 +215,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     unsigned m_check_sum;
     util::vector_view<util::Coordinate> m_coordinate_list;
-    util::PackedVectorView<OSMNodeID> m_osmnodeid_list;
+    extractor::PackedOSMIDsView m_osmnodeid_list;
     util::NameTable m_names_table;
     util::vector_view<std::uint32_t> m_lane_description_offsets;
     util::vector_view<extractor::guidance::TurnLaneType::Mask> m_lane_description_masks;

include/extractor/edge_based_graph_factory.hpp

@@ -15,6 +15,7 @@
 #include "extractor/profile_properties.hpp"
 #include "extractor/query_node.hpp"
 #include "extractor/restriction_map.hpp"
+#include "extractor/packed_osm_ids.hpp"
 
 #include "util/deallocating_vector.hpp"
 #include "util/guidance/bearing_class.hpp"
@@ -74,7 +75,7 @@ class EdgeBasedGraphFactory
                                    const std::unordered_set<NodeID> &traffic_lights,
                                    std::shared_ptr<const RestrictionMap> restriction_map,
                                    const std::vector<util::Coordinate> &coordinates,
-                                   const util::PackedVector<OSMNodeID> &osm_node_ids,
+                                   const extractor::PackedOSMIDs &osm_node_ids,
                                    ProfileProperties profile_properties,
                                    const util::NameTable &name_table,
                                    std::vector<std::uint32_t> &turn_lane_offsets,
@@ -131,7 +132,7 @@ class EdgeBasedGraphFactory
     EdgeID m_max_edge_id;
 
     const std::vector<util::Coordinate> &m_coordinates;
-    const util::PackedVector<OSMNodeID> &m_osm_node_ids;
+    const extractor::PackedOSMIDs &m_osm_node_ids;
     std::shared_ptr<util::NodeBasedDynamicGraph> m_node_based_graph;
     std::shared_ptr<RestrictionMap const> m_restriction_map;
 

include/extractor/extractor.hpp

@@ -59,7 +59,7 @@ class Extractor
     std::pair<std::size_t, EdgeID>
     BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
                            std::vector<util::Coordinate> &coordinates,
-                           util::PackedVector<OSMNodeID> &osm_node_ids,
+                           extractor::PackedOSMIDs &osm_node_ids,
                            std::vector<EdgeBasedNode> &node_based_edge_list,
                            std::vector<bool> &node_is_startpoint,
                            std::vector<EdgeWeight> &edge_based_node_weights,
@@ -78,7 +78,7 @@ class Extractor
     LoadNodeBasedGraph(std::unordered_set<NodeID> &barrier_nodes,
                        std::unordered_set<NodeID> &traffic_lights,
                        std::vector<util::Coordinate> &coordinates,
-                       util::PackedVector<OSMNodeID> &osm_node_ids);
+                       extractor::PackedOSMIDs &osm_node_ids);
 
     void WriteEdgeBasedGraph(const std::string &output_file_filename,
                              const EdgeID max_edge_id,

include/util/graph_loader.hpp

@@ -5,6 +5,7 @@
 #include "extractor/node_based_edge.hpp"
 #include "extractor/query_node.hpp"
 #include "extractor/restriction.hpp"
+#include "extractor/packed_osm_ids.hpp"
 #include "storage/io.hpp"
 #include "util/exception.hpp"
 #include "util/fingerprint.hpp"
@@ -59,7 +60,7 @@ NodeID loadNodesFromFile(storage::io::FileReader &file_reader,
                          BarrierOutIter barriers,
                          TrafficSignalsOutIter traffic_signals,
                          std::vector<util::Coordinate> &coordinates,
-                         util::PackedVector<OSMNodeID> &osm_node_ids)
+                         extractor::PackedOSMIDs &osm_node_ids)
 {
     auto number_of_nodes = file_reader.ReadElementCount64();
     Log() << "Importing number_of_nodes new = " << number_of_nodes << " nodes ";

include/util/packed_vector.hpp

@@ -16,33 +16,28 @@ namespace util
 {
 namespace detail
 {
-template <typename T, storage::Ownership Ownership> class PackedVector;
+template <typename T, std::size_t Bits, storage::Ownership Ownership> class PackedVector;
 }
 
 namespace serialization
 {
-template <typename T, storage::Ownership Ownership>
+template <typename T, std::size_t Bits, storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Ownership> &vec);
+inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Bits, Ownership> &vec);
 
-template <typename T, storage::Ownership Ownership>
+template <typename T, std::size_t Bits, storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer, const detail::PackedVector<T, Ownership> &vec);
+inline void write(storage::io::FileWriter &writer,
+                  const detail::PackedVector<T, Bits, Ownership> &vec);
 }
 
 namespace detail
 {
-/**
+template <typename T, std::size_t Bits, storage::Ownership Ownership> class PackedVector
- * Since OSM node IDs are (at the time of writing) not quite yet overflowing 32 bits, and
- * will predictably be containable within 33 bits for a long time, the following packs
- * 64-bit OSM IDs as 33-bit numbers within a 64-bit vector.
- *
- * NOTE: this type is templated for future use, but will require a slight refactor to
- * configure BITSIZE and ELEMSIZE
- */
-template <typename T, storage::Ownership Ownership> class PackedVector
 {
-    static const constexpr std::size_t BITSIZE = 33;
+    static_assert(sizeof(T) <= sizeof(std::uint64_t), "Maximum size of type T is 8 bytes");
-    static const constexpr std::size_t ELEMSIZE = 64;
+    static_assert(Bits <= sizeof(std::uint64_t)*CHAR_BIT, "Maximum number of bits it 64.");
-    static const constexpr std::size_t PACKSIZE = BITSIZE * ELEMSIZE;
+
+    static const constexpr std::size_t ELEMSIZE = sizeof(std::uint64_t)*CHAR_BIT;
+    static const constexpr std::size_t PACKSIZE = Bits * ELEMSIZE;
 
   public:
     using value_type = T;
@@ -54,31 +49,31 @@ template <typename T, storage::Ownership Ownership> class PackedVector
      */
     inline static std::size_t elements_to_blocks(std::size_t elements)
     {
-        return std::ceil(static_cast<double>(elements) * BITSIZE / ELEMSIZE);
+        return std::ceil(static_cast<double>(elements) * Bits / ELEMSIZE);
     }
 
-    void push_back(T incoming_node_id)
+    void push_back(T data)
     {
-        std::uint64_t node_id = static_cast<std::uint64_t>(incoming_node_id);
+        std::uint64_t node_id = static_cast<std::uint64_t>(data);
 
         // mask incoming values, just in case they are > bitsize
-        const std::uint64_t incoming_mask = static_cast<std::uint64_t>(pow(2, BITSIZE)) - 1;
+        const std::uint64_t incoming_mask = static_cast<std::uint64_t>(pow(2, Bits)) - 1;
         node_id = node_id & incoming_mask;
 
-        const std::size_t available = (PACKSIZE - BITSIZE * num_elements) % ELEMSIZE;
+        const std::size_t available = (PACKSIZE - Bits * num_elements) % ELEMSIZE;
 
         if (available == 0)
         {
             // insert ID at the left side of this element
-            std::uint64_t at_left = node_id << (ELEMSIZE - BITSIZE);
+            std::uint64_t at_left = node_id << (ELEMSIZE - Bits);
 
             add_last_elem(at_left);
         }
-        else if (available >= BITSIZE)
+        else if (available >= Bits)
         {
             // insert ID somewhere in the middle of this element; ID can be contained
             // entirely within one element
-            const std::uint64_t shifted = node_id << (available - BITSIZE);
+            const std::uint64_t shifted = node_id << (available - Bits);
 
             replace_last_elem(vec_back() | shifted);
         }
@@ -86,9 +81,9 @@ template <typename T, storage::Ownership Ownership> class PackedVector
         {
             // ID will be split between the end of this element and the beginning
             // of the next element
-            const std::uint64_t left = node_id >> (BITSIZE - available);
+            const std::uint64_t left = node_id >> (Bits - available);
 
-            std::uint64_t right = node_id << (ELEMSIZE - (BITSIZE - available));
+            std::uint64_t right = node_id << (ELEMSIZE - (Bits - available));
 
             replace_last_elem(vec_back() | left);
             add_last_elem(right);
@@ -105,11 +100,11 @@ template <typename T, storage::Ownership Ownership> class PackedVector
 
         const std::size_t pack_group = trunc(a_index / ELEMSIZE);
         const std::size_t pack_index = (a_index + ELEMSIZE) % ELEMSIZE;
-        const std::size_t left_index = (PACKSIZE - BITSIZE * pack_index) % ELEMSIZE;
+        const std::size_t left_index = (PACKSIZE - Bits * pack_index) % ELEMSIZE;
 
-        const bool back_half = pack_index >= BITSIZE;
+        const bool back_half = pack_index >= Bits;
-        const std::size_t index = pack_group * BITSIZE + trunc(pack_index / BITSIZE) +
+        const std::size_t index =
-                                  trunc((pack_index - back_half) / 2);
+            pack_group * Bits + trunc(pack_index / Bits) + trunc((pack_index - back_half) / 2);
 
         BOOST_ASSERT(index < vec.size());
         const std::uint64_t elem = static_cast<std::uint64_t>(vec.at(index));
@@ -117,25 +112,25 @@ template <typename T, storage::Ownership Ownership> class PackedVector
         if (left_index == 0)
         {
             // ID is at the far left side of this element
-            return T{elem >> (ELEMSIZE - BITSIZE)};
+            return T{elem >> (ELEMSIZE - Bits)};
         }
-        else if (left_index >= BITSIZE)
+        else if (left_index >= Bits)
         {
             // ID is entirely contained within this element
-            const std::uint64_t at_right = elem >> (left_index - BITSIZE);
+            const std::uint64_t at_right = elem >> (left_index - Bits);
-            const std::uint64_t left_mask = static_cast<std::uint64_t>(pow(2, BITSIZE)) - 1;
+            const std::uint64_t left_mask = static_cast<std::uint64_t>(pow(2, Bits)) - 1;
             return T{at_right & left_mask};
         }
         else
         {
             // ID is split between this and the next element
             const std::uint64_t left_mask = static_cast<std::uint64_t>(pow(2, left_index)) - 1;
-            const std::uint64_t left_side = (elem & left_mask) << (BITSIZE - left_index);
+            const std::uint64_t left_side = (elem & left_mask) << (Bits - left_index);
 
             BOOST_ASSERT(index < vec.size() - 1);
             const std::uint64_t next_elem = static_cast<std::uint64_t>(vec.at(index + 1));
 
-            const std::uint64_t right_side = next_elem >> (ELEMSIZE - (BITSIZE - left_index));
+            const std::uint64_t right_side = next_elem >> (ELEMSIZE - (Bits - left_index));
             return T{left_side | right_side};
         }
     }
@@ -163,14 +158,14 @@ template <typename T, storage::Ownership Ownership> class PackedVector
 
     std::size_t capacity() const
     {
-        return std::floor(static_cast<double>(vec.capacity()) * ELEMSIZE / BITSIZE);
+        return std::floor(static_cast<double>(vec.capacity()) * ELEMSIZE / Bits);
     }
 
-    friend void serialization::read<T, Ownership>(storage::io::FileReader &reader,
+    friend void serialization::read<T, Bits, Ownership>(storage::io::FileReader &reader,
-                                                  detail::PackedVector<T, Ownership> &vec);
+                                                        PackedVector &vec);
 
-    friend void serialization::write<T, Ownership>(storage::io::FileWriter &writer,
+    friend void serialization::write<T, Bits, Ownership>(storage::io::FileWriter &writer,
-                                                   const detail::PackedVector<T, Ownership> &vec);
+                                                         const PackedVector &vec);
 
   private:
     util::ViewOrVector<std::uint64_t, Ownership> vec;
@@ -218,8 +213,8 @@ template <typename T, storage::Ownership Ownership> class PackedVector
 };
 }
 
-template <typename T> using PackedVector = detail::PackedVector<T, storage::Ownership::Container>;
+template <typename T, std::size_t Bits> using PackedVector = detail::PackedVector<T, Bits, storage::Ownership::Container>;
-template <typename T> using PackedVectorView = detail::PackedVector<T, storage::Ownership::View>;
+template <typename T, std::size_t Bits> using PackedVectorView = detail::PackedVector<T, Bits, storage::Ownership::View>;
 }
 }
 

include/util/serialization.hpp

@@ -14,15 +14,15 @@ namespace util
 {
 namespace serialization
 {
-template <typename T, storage::Ownership Ownership>
+template <typename T, std::size_t Bits, storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Ownership> &vec)
+inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Bits, Ownership> &vec)
 {
     vec.num_elements = reader.ReadOne<std::uint64_t>();
     storage::serialization::read(reader, vec.vec);
 }
 
-template <typename T, storage::Ownership Ownership>
+template <typename T, std::size_t Bits, storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer, const detail::PackedVector<T, Ownership> &vec)
+inline void write(storage::io::FileWriter &writer, const detail::PackedVector<T, Bits, Ownership> &vec)
 {
     writer.WriteOne(vec.num_elements);
     storage::serialization::write(writer, vec.vec);

src/extractor/edge_based_graph_factory.cpp

@@ -42,7 +42,7 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory(
     const std::unordered_set<NodeID> &traffic_lights,
     std::shared_ptr<const RestrictionMap> restriction_map,
     const std::vector<util::Coordinate> &coordinates,
-    const util::PackedVector<OSMNodeID> &osm_node_ids,
+    const extractor::PackedOSMIDs &osm_node_ids,
     ProfileProperties profile_properties,
     const util::NameTable &name_table,
     std::vector<std::uint32_t> &turn_lane_offsets,

src/extractor/extractor.cpp

@@ -263,7 +263,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
         std::vector<bool> node_is_startpoint;
         std::vector<EdgeWeight> edge_based_node_weights;
         std::vector<util::Coordinate> coordinates;
-        util::PackedVector<OSMNodeID> osm_node_ids;
+        extractor::PackedOSMIDs osm_node_ids;
 
         auto graph_size = BuildEdgeExpandedGraph(scripting_environment,
                                                  coordinates,
@@ -407,7 +407,7 @@ std::shared_ptr<util::NodeBasedDynamicGraph>
 Extractor::LoadNodeBasedGraph(std::unordered_set<NodeID> &barriers,
                               std::unordered_set<NodeID> &traffic_signals,
                               std::vector<util::Coordinate> &coordiantes,
-                              util::PackedVector<OSMNodeID> &osm_node_ids)
+                              extractor::PackedOSMIDs &osm_node_ids)
 {
     storage::io::FileReader file_reader(config.output_file_name,
                                         storage::io::FileReader::VerifyFingerprint);
@@ -439,7 +439,7 @@ Extractor::LoadNodeBasedGraph(std::unordered_set<NodeID> &barriers,
 std::pair<std::size_t, EdgeID>
 Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
                                   std::vector<util::Coordinate> &coordinates,
-                                  util::PackedVector<OSMNodeID> &osm_node_ids,
+                                  extractor::PackedOSMIDs &osm_node_ids,
                                   std::vector<EdgeBasedNode> &node_based_edge_list,
                                   std::vector<bool> &node_is_startpoint,
                                   std::vector<EdgeWeight> &edge_based_node_weights,

src/storage/storage.cpp

@@ -19,6 +19,7 @@
 #include "extractor/profile_properties.hpp"
 #include "extractor/query_node.hpp"
 #include "extractor/travel_mode.hpp"
+#include "extractor/packed_osm_ids.hpp"
 
 #include "partition/cell_storage.hpp"
 #include "partition/edge_based_graph_reader.hpp"
@@ -326,7 +327,7 @@ void Storage::PopulateLayout(DataLayout &layout)
         // number of items:
         layout.SetBlockSize<std::uint64_t>(
             DataLayout::OSM_NODE_ID_LIST,
-            util::PackedVectorView<OSMNodeID>::elements_to_blocks(coordinate_list_size));
+            extractor::PackedOSMIDsView::elements_to_blocks(coordinate_list_size));
     }
 
     // load geometries sizes
@@ -694,7 +695,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
             layout.GetBlockPtr<std::uint64_t, true>(memory_ptr, DataLayout::OSM_NODE_ID_LIST);
         util::vector_view<util::Coordinate> coordinates(
             coordinates_ptr, layout.num_entries[DataLayout::COORDINATE_LIST]);
-        util::PackedVectorView<OSMNodeID> osm_node_ids;
+        extractor::PackedOSMIDsView osm_node_ids;
         osm_node_ids.reset(osmnodeid_ptr, layout.num_entries[DataLayout::OSM_NODE_ID_LIST]);
 
         extractor::files::readNodes(config.nodes_data_path, coordinates, osm_node_ids);

src/tools/components.cpp

@@ -33,7 +33,7 @@ using TarjanEdge = util::static_graph_details::SortableEdgeWithData<void>;
 
 std::size_t loadGraph(const std::string &path,
                       std::vector<util::Coordinate> &coordinate_list,
-                      util::PackedVector<OSMNodeID> &osm_node_ids,
+                      extractor::PackedOSMIDs &osm_node_ids,
                       std::vector<TarjanEdge> &graph_edge_list)
 {
     storage::io::FileReader file_reader(path, storage::io::FileReader::VerifyFingerprint);
@@ -142,7 +142,7 @@ int main(int argc, char *argv[])
 
     std::vector<tools::TarjanEdge> graph_edge_list;
     std::vector<util::Coordinate> coordinate_list;
-    util::PackedVector<OSMNodeID> osm_node_ids;
+    extractor::PackedOSMIDs osm_node_ids;
     auto number_of_nodes = tools::loadGraph(inpath, coordinate_list, osm_node_ids, graph_edge_list);
 
     tbb::parallel_sort(graph_edge_list.begin(), graph_edge_list.end());

src/updater/updater.cpp

@@ -134,7 +134,7 @@ updateSegmentData(const UpdaterConfig &config,
                   extractor::SegmentDataContainer &segment_data)
 {
     std::vector<util::Coordinate> coordinates;
-    util::PackedVector<OSMNodeID> osm_node_ids;
+    extractor::PackedOSMIDs osm_node_ids;
     extractor::files::readNodes(config.node_based_graph_path, coordinates, osm_node_ids);
 
     // vector to count used speeds for logging

unit_tests/util/packed_vector.cpp

@@ -12,7 +12,7 @@ using namespace osrm::util;
 // Verify that the packed vector behaves as expected
 BOOST_AUTO_TEST_CASE(insert_and_retrieve_packed_test)
 {
-    PackedVector<OSMNodeID> packed_ids;
+    PackedVector<OSMNodeID, 33> packed_ids;
     std::vector<OSMNodeID> original_ids;
 
     const constexpr std::size_t num_test_cases = 399;
@@ -33,7 +33,7 @@ BOOST_AUTO_TEST_CASE(insert_and_retrieve_packed_test)
 
 BOOST_AUTO_TEST_CASE(packed_vector_capacity_test)
 {
-    PackedVector<OSMNodeID> packed_vec;
+    PackedVector<OSMNodeID, 33> packed_vec;
     const std::size_t original_size = packed_vec.capacity();
     std::vector<OSMNodeID> dummy_vec;