Rewrite packed vector to also allow random access

This fixes issues #3952. The new approach pre-computes masks for fast
access. Since elements can potentially span multiple words we need masks
and offsets for each upper and lower word.

Due to a bug in the C++14 standart the mask computation is not
recognized as constexpr, but would work on C++17.

src/benchmarks/CMakeLists.txt

@@ -1,6 +1,7 @@
 file(GLOB RTreeBenchmarkSources static_rtree.cpp)
 file(GLOB MatchBenchmarkSources match.cpp)
 file(GLOB AliasBenchmarkSources alias.cpp)
+file(GLOB PackedVectorBenchmarkSources packed_vector.cpp)
 
 add_executable(rtree-bench
 	EXCLUDE_FROM_ALL
@@ -40,8 +41,21 @@ target_link_libraries(alias-bench
 	${TBB_LIBRARIES}
 	${MAYBE_SHAPEFILE})
 
+add_executable(packedvector-bench
+	EXCLUDE_FROM_ALL
+    ${PackedVectorBenchmarkSources}
+	$<TARGET_OBJECTS:UTIL>)
+
+target_link_libraries(packedvector-bench
+	${BOOST_BASE_LIBRARIES}
+	${CMAKE_THREAD_LIBS_INIT}
+	${TBB_LIBRARIES}
+    ${MAYBE_SHAPEFILE})
+
+
 add_custom_target(benchmarks
 	DEPENDS
 	rtree-bench
+	packedvector-bench
 	match-bench
     alias-bench)

src/benchmarks/packed_vector.cpp

@@ -0,0 +1,81 @@
+#include "util/packed_vector.hpp"
+#include "util/integer_range.hpp"
+#include "util/log.hpp"
+#include "util/timing_util.hpp"
+
+#include <algorithm>
+#include <iomanip>
+#include <numeric>
+#include <random>
+#include <string>
+#include <vector>
+
+using namespace osrm;
+
+struct Measurement
+{
+    double random_write_ms;
+    double random_read_ms;
+};
+
+#ifdef _WIN32
+#pragma optimize("", off)
+template <class T> void dont_optimize_away(T &&datum) { T local = datum; }
+#pragma optimize("", on)
+#else
+template <class T> void dont_optimize_away(T &&datum) { asm volatile("" : "+r"(datum)); }
+#endif
+
+template <std::size_t num_rounds, std::size_t num_entries, typename VectorT>
+auto measure_random_access()
+{
+    std::vector<std::size_t> indices(num_entries);
+    std::iota(indices.begin(), indices.end(), 0);
+    std::mt19937 g(1337);
+    std::shuffle(indices.begin(), indices.end(), g);
+
+    VectorT vector(num_entries);
+
+    TIMER_START(write);
+    for (auto round : util::irange<std::size_t>(0, num_rounds))
+    {
+        for (auto idx : util::irange<std::size_t>(0, num_entries))
+        {
+            vector[indices[idx]] = idx + round;
+        }
+    }
+    TIMER_STOP(write);
+
+    TIMER_START(read);
+    auto sum = 0;
+    for (auto round : util::irange<std::size_t>(0, num_rounds))
+    {
+        sum = round;
+        for (auto idx : util::irange<std::size_t>(0, num_entries))
+        {
+            sum += vector[indices[idx]];
+        }
+        dont_optimize_away(sum);
+    }
+    TIMER_STOP(read);
+
+    return Measurement{TIMER_MSEC(write), TIMER_MSEC(read)};
+}
+
+int main(int, char **)
+{
+    util::LogPolicy::GetInstance().Unmute();
+
+    auto result_plain = measure_random_access<10000, 1000000, std::vector<std::uint32_t>>();
+    auto result_packed =
+        measure_random_access<10000, 1000000, util::PackedVector<std::uint32_t, 22>>();
+
+    auto write_slowdown = result_packed.random_write_ms / result_plain.random_write_ms;
+    auto read_slowdown = result_packed.random_read_ms / result_plain.random_read_ms;
+    util::Log() << "random write: std::vector " << result_plain.random_write_ms
+                << " ms, util::packed_vector " << result_packed.random_write_ms << " ms. "
+                << write_slowdown;
+    util::Log() << "random read: std::vector " << result_plain.random_read_ms
+                << " ms, util::packed_vector " << result_packed.random_read_ms << " ms. "
+                << read_slowdown;
+}

src/storage/storage.cpp

@@ -329,11 +329,14 @@ void Storage::PopulateLayout(DataLayout &layout)
         io::FileReader node_file(config.nodes_data_path, io::FileReader::VerifyFingerprint);
         const auto coordinate_list_size = node_file.ReadElementCount64();
         layout.SetBlockSize<util::Coordinate>(DataLayout::COORDINATE_LIST, coordinate_list_size);
+        node_file.Skip<util::Coordinate>(coordinate_list_size);
+        // skip number of elements
+        node_file.Skip<std::uint64_t>(1);
+        const auto num_id_blocks = node_file.ReadElementCount64();
         // we'll read a list of OSM node IDs from the same data, so set the block size for the same
         // number of items:
-        layout.SetBlockSize<std::uint64_t>(
-            DataLayout::OSM_NODE_ID_LIST,
-            extractor::PackedOSMIDsView::elements_to_blocks(coordinate_list_size));
+        layout.SetBlockSize<extractor::PackedOSMIDsView::block_type>(DataLayout::OSM_NODE_ID_LIST,
+                                                                     num_id_blocks);
     }
 
     // load geometries sizes
@@ -703,11 +706,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
         const auto coordinates_ptr =
             layout.GetBlockPtr<util::Coordinate, true>(memory_ptr, DataLayout::COORDINATE_LIST);
         const auto osmnodeid_ptr =
-            layout.GetBlockPtr<std::uint64_t, true>(memory_ptr, DataLayout::OSM_NODE_ID_LIST);
+            layout.GetBlockPtr<extractor::PackedOSMIDsView::block_type, true>(
+                memory_ptr, DataLayout::OSM_NODE_ID_LIST);
         util::vector_view<util::Coordinate> coordinates(
             coordinates_ptr, layout.num_entries[DataLayout::COORDINATE_LIST]);
-        extractor::PackedOSMIDsView osm_node_ids;
-        osm_node_ids.reset(osmnodeid_ptr, layout.num_entries[DataLayout::OSM_NODE_ID_LIST]);
+        extractor::PackedOSMIDsView osm_node_ids(
+            util::vector_view<extractor::PackedOSMIDsView::block_type>(
+                osmnodeid_ptr, layout.num_entries[DataLayout::OSM_NODE_ID_LIST]),
+            layout.num_entries[DataLayout::OSM_NODE_ID_LIST] *
+                extractor::PackedOSMIDsView::BLOCK_ELEMENTS);
 
         extractor::files::readNodes(config.nodes_data_path, coordinates, osm_node_ids);
     }