Port .names file to tar

include/engine/datafacade/contiguous_internalmem_datafacade.hpp

@@ -15,6 +15,7 @@
 #include "extractor/edge_based_node.hpp"
 #include "extractor/intersection_bearings_container.hpp"
 #include "extractor/maneuver_override.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/node_data_container.hpp"
 #include "extractor/packed_osm_ids.hpp"
 #include "extractor/profile_properties.hpp"
@@ -40,7 +41,6 @@
 #include "util/guidance/entry_class.hpp"
 #include "util/guidance/turn_lanes.hpp"
 #include "util/log.hpp"
-#include "util/name_table.hpp"
 #include "util/packed_vector.hpp"
 #include "util/range_table.hpp"
 #include "util/rectangle.hpp"
@@ -215,7 +215,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     extractor::IntersectionBearingsView intersection_bearings_view;
 
-    util::NameTable m_name_table;
+    extractor::NameTableView m_name_table;
     // the look-up table for entry classes. An entry class lists the possibility of entry for all
     // available turns. Such a class id is stored with every edge.
     util::vector_view<util::guidance::EntryClass> m_entry_class_table;
@@ -353,11 +353,20 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     void InitializeNamePointers(storage::DataLayout &data_layout, char *memory_block)
     {
-        auto name_data_ptr =
+        const auto name_blocks_ptr =
-            data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::NAME_CHAR_DATA);
+            data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference>(
-        const auto name_data_size =
+                memory_block, storage::DataLayout::NAME_BLOCKS);
-            data_layout.GetBlockEntries(storage::DataLayout::NAME_CHAR_DATA);
+        const auto name_values_ptr =
-        m_name_table.reset(name_data_ptr, name_data_ptr + name_data_size);
+            data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType>(
+                memory_block, storage::DataLayout::NAME_VALUES);
+
+        util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
+            name_blocks_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS));
+        util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
+            name_values_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_VALUES));
+
+        extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
+        m_name_table = extractor::NameTableView{std::move(index_data_view)};
     }
 
     void InitializeTurnLaneDescriptionsPointers(storage::DataLayout &data_layout,
@@ -938,7 +947,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
         auto found_range = std::equal_range(
             m_maneuver_overrides.begin(), m_maneuver_overrides.end(), edge_based_node_id, Comp{});
 
-        std::for_each(found_range.first, found_range.second, [&](const auto & override) {
+        std::for_each(found_range.first, found_range.second, [&](const auto &override) {
             std::vector<NodeID> sequence(
                 m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_begin,
                 m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_end);

include/extractor/edge_based_graph_factory.hpp

@@ -9,6 +9,7 @@
 #include "extractor/edge_based_node_segment.hpp"
 #include "extractor/extraction_turn.hpp"
 #include "extractor/maneuver_override.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/nbg_to_ebg.hpp"
 #include "extractor/node_data_container.hpp"
 #include "extractor/query_node.hpp"
@@ -18,7 +19,6 @@
 
 #include "util/concurrent_id_map.hpp"
 #include "util/deallocating_vector.hpp"
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 #include "util/typedefs.hpp"
 
@@ -70,7 +70,7 @@ class EdgeBasedGraphFactory
                                    const std::unordered_set<NodeID> &barrier_nodes,
                                    const std::unordered_set<NodeID> &traffic_lights,
                                    const std::vector<util::Coordinate> &coordinates,
-                                   const util::NameTable &name_table,
+                                   const NameTable &name_table,
                                    const std::unordered_set<EdgeID> &segregated_edges,
                                    const LaneDescriptionMap &lane_description_map);
 
@@ -138,7 +138,7 @@ class EdgeBasedGraphFactory
     const std::unordered_set<NodeID> &m_traffic_lights;
     const CompressedEdgeContainer &m_compressed_edge_container;
 
-    const util::NameTable &name_table;
+    const NameTable &name_table;
     const std::unordered_set<EdgeID> &segregated_edges;
     const LaneDescriptionMap &lane_description_map;
 

include/extractor/extractor.hpp

@@ -77,7 +77,7 @@ class Extractor
         const std::vector<TurnRestriction> &turn_restrictions,
         const std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions,
         const std::unordered_set<EdgeID> &segregated_edges,
-        const util::NameTable &name_table,
+        const NameTable &name_table,
         const std::vector<UnresolvedManeuverOverride> &maneuver_overrides,
         const LaneDescriptionMap &turn_lane_map,
         // for calculating turn penalties
@@ -116,7 +116,7 @@ class Extractor
         const std::unordered_set<NodeID> &barrier_nodes,
         const std::vector<TurnRestriction> &turn_restrictions,
         const std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions,
-        const util::NameTable &name_table,
+        const NameTable &name_table,
         LaneDescriptionMap lane_description_map,
         ScriptingEnvironment &scripting_environment);
 };

include/extractor/files.hpp

@@ -411,7 +411,8 @@ void readRawNBGraph(const boost::filesystem::path &path,
         osm_node_ids.push_back(current_node.node_id);
         index++;
     };
-    reader.ReadStreaming<extractor::QueryNode>("/extractor/nodes", boost::make_function_output_iterator(decode));
+    reader.ReadStreaming<extractor::QueryNode>("/extractor/nodes",
+                                               boost::make_function_output_iterator(decode));
 
     reader.ReadStreaming<NodeID>("/extractor/barriers", barriers);
 
@@ -420,6 +421,24 @@ void readRawNBGraph(const boost::filesystem::path &path,
     storage::serialization::read(reader, "/extractor/edges", edge_list);
     storage::serialization::read(reader, "/extractor/annotations", annotations);
 }
+
+template <typename NameTableT>
+void readNames(const boost::filesystem::path &path, NameTableT &table)
+{
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    serialization::read(reader, "/common/names", table);
+}
+
+template <typename NameTableT>
+void writeNames(const boost::filesystem::path &path, const NameTableT &table)
+{
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
+
+    serialization::write(writer, "/common/names", table);
+}
 }
 }
 }

include/extractor/intersection/have_identical_names.hpp

@@ -1,9 +1,9 @@
 #ifndef OSRM_EXTRACTOR_INTERSECTION_HAVE_IDENTICAL_NAMES_HPP_
 #define OSRM_EXTRACTOR_INTERSECTION_HAVE_IDENTICAL_NAMES_HPP_
 
+#include "extractor/name_table.hpp"
 #include "extractor/suffix_table.hpp"
 #include "guidance/constants.hpp"
-#include "util/name_table.hpp"
 
 namespace osrm
 {
@@ -17,7 +17,7 @@ namespace intersection
 // rhs->lhs)
 bool HaveIdenticalNames(const NameID lhs,
                         const NameID rhs,
-                        const util::NameTable &name_table,
+                        const NameTable &name_table,
                         const SuffixTable &street_name_suffix_table);
 
 } // namespace intersection

include/extractor/intersection/mergable_road_detector.hpp

@@ -6,7 +6,10 @@
 #include "extractor/intersection/have_identical_names.hpp"
 #include "extractor/restriction_index.hpp"
 #include "extractor/turn_lane_types.hpp"
+#include "extractor/name_table.hpp"
+
 #include "guidance/intersection.hpp"
+
 #include "util/coordinate.hpp"
 #include "util/node_based_graph.hpp"
 #include "util/typedefs.hpp"
@@ -49,7 +52,7 @@ class MergableRoadDetector
                          const RestrictionMap &node_restriction_map,
                          const std::unordered_set<NodeID> &barrier_nodes,
                          const TurnLanesIndexedArray &turn_lanes_data,
-                         const util::NameTable &name_table,
+                         const extractor::NameTable &name_table,
                          const SuffixTable &street_name_suffix_table);
 
     // OSM ways tend to be modelled as separate ways for different directions. This is often due to
@@ -168,7 +171,7 @@ class MergableRoadDetector
     const TurnLanesIndexedArray &turn_lanes_data;
 
     // name detection
-    const util::NameTable &name_table;
+    const extractor::NameTable &name_table;
     const SuffixTable &street_name_suffix_table;
 
     const CoordinateExtractor coordinate_extractor;

include/extractor/name_table.hpp

@@ -0,0 +1,120 @@
+#ifndef OSRM_EXTRACTOR_NAME_TABLE_HPP
+#define OSRM_EXTRACTOR_NAME_TABLE_HPP
+
+#include "util/indexed_data.hpp"
+#include "util/string_view.hpp"
+#include "util/typedefs.hpp"
+
+#include <string>
+
+namespace osrm
+{
+namespace extractor
+{
+
+namespace detail
+{
+template <storage::Ownership Ownership> class NameTableImpl;
+}
+
+namespace serialization
+{
+template <storage::Ownership Ownership>
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::NameTableImpl<Ownership> &index_data);
+
+template <storage::Ownership Ownership>
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const detail::NameTableImpl<Ownership> &index_data);
+}
+
+namespace detail
+{
+// This class provides a limited view over all the string data we serialize out.
+// The following functions are a subset of what is available.
+// See the data facades for they provide full access to this serialized string data.
+// Way string data is stored in blocks based on `id` as follows:
+//
+// | name | destination | pronunciation | ref | exits
+//                      ^               ^
+//                      [range)
+//                       ^ id + 2
+//
+// `id + offset` gives us the range of chars.
+//
+// Offset 0 is name, 1 is destination, 2 is pronunciation, 3 is ref, 4 is exits
+// See datafacades and extractor callbacks for details.
+template <storage::Ownership Ownership> class NameTableImpl
+{
+  public:
+    using IndexedData =
+        util::detail::IndexedDataImpl<util::VariableGroupBlock<16, util::StringView>, Ownership>;
+    using ResultType = typename IndexedData::ResultType;
+    using ValueType = typename IndexedData::ValueType;
+
+    NameTableImpl() {}
+
+    NameTableImpl(IndexedData indexed_data_) : indexed_data{std::move(indexed_data_)} {}
+
+    util::StringView GetNameForID(const NameID id) const
+    {
+        if (id == INVALID_NAMEID)
+            return {};
+
+        return indexed_data.at(id + 0);
+    }
+
+    util::StringView GetDestinationsForID(const NameID id) const
+    {
+        if (id == INVALID_NAMEID)
+            return {};
+
+        return indexed_data.at(id + 1);
+    }
+
+    util::StringView GetExitsForID(const NameID id) const
+    {
+        if (id == INVALID_NAMEID)
+            return {};
+
+        return indexed_data.at(id + 4);
+    }
+
+    util::StringView GetRefForID(const NameID id) const
+    {
+        if (id == INVALID_NAMEID)
+            return {};
+
+        const constexpr auto OFFSET_REF = 3u;
+        return indexed_data.at(id + OFFSET_REF);
+    }
+
+    util::StringView GetPronunciationForID(const NameID id) const
+    {
+        if (id == INVALID_NAMEID)
+            return {};
+
+        const constexpr auto OFFSET_PRONUNCIATION = 2u;
+        return indexed_data.at(id + OFFSET_PRONUNCIATION);
+    }
+
+    friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
+                                               const std::string &name,
+                                               NameTableImpl &index_data);
+
+    friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
+                                                const std::string &name,
+                                                const NameTableImpl &index_data);
+  private:
+    IndexedData indexed_data;
+};
+}
+
+using NameTable = detail::NameTableImpl<storage::Ownership::Container>;
+using NameTableView = detail::NameTableImpl<storage::Ownership::View>;
+} // namespace extractor
+} // namespace osrm
+
+#endif // OSRM_EXTRACTOR_NAME_TABLE_HPP

include/extractor/serialization.hpp

@@ -1,10 +1,11 @@
 #ifndef OSRM_EXTRACTOR_IO_HPP
 #define OSRM_EXTRACTOR_IO_HPP
 
-#include "conditional_turn_penalty.hpp"
+#include "extractor/conditional_turn_penalty.hpp"
 #include "extractor/datasources.hpp"
 #include "extractor/intersection_bearings_container.hpp"
 #include "extractor/maneuver_override.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/nbg_to_ebg.hpp"
 #include "extractor/node_data_container.hpp"
 #include "extractor/profile_properties.hpp"
@@ -117,16 +118,18 @@ inline void read(storage::tar::FileReader &reader,
 {
     // read actual data
     storage::serialization::read(reader, name + "/nodes", node_data_container.nodes);
-    storage::serialization::read(reader, name + "/annotations", node_data_container.annotation_data);
+    storage::serialization::read(
+        reader, name + "/annotations", node_data_container.annotation_data);
 }
 
 template <storage::Ownership Ownership>
 inline void write(storage::tar::FileWriter &writer,
-                  const std::string& name,
+                  const std::string &name,
                   const detail::EdgeBasedNodeDataContainerImpl<Ownership> &node_data_container)
 {
     storage::serialization::write(writer, name + "/nodes", node_data_container.nodes);
-    storage::serialization::write(writer, name + "/annotations", node_data_container.annotation_data);
+    storage::serialization::write(
+        writer, name + "/annotations", node_data_container.annotation_data);
 }
 
 inline void read(storage::io::FileReader &reader, NodeRestriction &restriction)
@@ -307,7 +310,8 @@ inline void write(storage::io::BufferWriter &writer,
     }
 }
 
-inline void read(storage::io::BufferReader &reader, std::vector<ConditionalTurnPenalty> &conditional_penalties)
+inline void read(storage::io::BufferReader &reader,
+                 std::vector<ConditionalTurnPenalty> &conditional_penalties)
 {
     auto num_elements = reader.ReadElementCount64();
     conditional_penalties.resize(num_elements);
@@ -318,8 +322,8 @@ inline void read(storage::io::BufferReader &reader, std::vector<ConditionalTurnP
 }
 
 inline void write(storage::tar::FileWriter &writer,
-                 const std::string& name,
+                  const std::string &name,
-                 const std::vector<ConditionalTurnPenalty> &conditional_penalties)
+                  const std::vector<ConditionalTurnPenalty> &conditional_penalties)
 {
     storage::io::BufferWriter buffer_writer;
     write(buffer_writer, conditional_penalties);
@@ -328,7 +332,7 @@ inline void write(storage::tar::FileWriter &writer,
 }
 
 inline void read(storage::tar::FileReader &reader,
-                 const std::string& name,
+                 const std::string &name,
                  std::vector<ConditionalTurnPenalty> &conditional_penalties)
 {
     std::string buffer;
@@ -336,9 +340,25 @@ inline void read(storage::tar::FileReader &reader,
 
     storage::io::BufferReader buffer_reader{buffer};
     read(buffer_reader, conditional_penalties);
-
 }
 
+template <storage::Ownership Ownership>
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const detail::NameTableImpl<Ownership> &name_table)
+{
+    storage::io::BufferWriter buffer_writer;
+    util::serialization::write(writer, name, name_table.indexed_data);
+}
+
+template <storage::Ownership Ownership>
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                detail::NameTableImpl<Ownership> &name_table)
+{
+    std::string buffer;
+    util::serialization::read(reader, name, name_table.indexed_data);
+}
 }
 }
 }

include/guidance/driveway_handler.hpp

@@ -19,7 +19,7 @@ class DrivewayHandler final : public IntersectionHandler
                     const extractor::RestrictionMap &node_restriction_map,
                     const std::unordered_set<NodeID> &barrier_nodes,
                     const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                    const util::NameTable &name_table,
+                    const extractor::NameTable &name_table,
                     const extractor::SuffixTable &street_name_suffix_table);
 
     ~DrivewayHandler() override final = default;

include/guidance/guidance_processing.hpp

@@ -8,12 +8,12 @@
 #include "extractor/suffix_table.hpp"
 #include "extractor/turn_lane_types.hpp"
 #include "extractor/way_restriction_map.hpp"
+#include "extractor/name_table.hpp"
 
 #include "util/coordinate.hpp"
 #include "util/guidance/bearing_class.hpp"
 #include "util/guidance/entry_class.hpp"
 #include "util/guidance/turn_lanes.hpp"
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 
 #include <unordered_set>
@@ -33,7 +33,7 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
                    const std::unordered_set<NodeID> &barrier_nodes,
                    const extractor::RestrictionMap &node_restriction_map,
                    const extractor::WayRestrictionMap &way_restriction_map,
-                   const util::NameTable &name_table,
+                   const extractor::NameTable &name_table,
                    const extractor::SuffixTable &suffix_table,
                    const extractor::TurnLanesIndexedArray &turn_lanes_data,
                    extractor::LaneDescriptionMap &lane_description_map,

include/guidance/intersection_handler.hpp

@@ -4,13 +4,13 @@
 #include "extractor/intersection/intersection_analysis.hpp"
 #include "extractor/intersection/node_based_graph_walker.hpp"
 #include "extractor/suffix_table.hpp"
+#include "extractor/name_table.hpp"
 #include "guidance/constants.hpp"
 #include "guidance/intersection.hpp"
 
 #include "util/assert.hpp"
 #include "util/coordinate_calculation.hpp"
 #include "util/guidance/name_announcements.hpp"
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 
 #include <algorithm>
@@ -38,7 +38,7 @@ class IntersectionHandler
                         const extractor::RestrictionMap &node_restriction_map,
                         const std::unordered_set<NodeID> &barrier_nodes,
                         const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                        const util::NameTable &name_table,
+                        const extractor::NameTable &name_table,
                         const extractor::SuffixTable &street_name_suffix_table);
 
     virtual ~IntersectionHandler() = default;
@@ -59,7 +59,7 @@ class IntersectionHandler
     const extractor::RestrictionMap &node_restriction_map;
     const std::unordered_set<NodeID> &barrier_nodes;
     const extractor::TurnLanesIndexedArray &turn_lanes_data;
-    const util::NameTable &name_table;
+    const extractor::NameTable &name_table;
     const extractor::SuffixTable &street_name_suffix_table;
     const extractor::intersection::NodeBasedGraphWalker
         graph_walker; // for skipping traffic signal, distances etc.

include/guidance/is_through_street.hpp

@@ -7,6 +7,9 @@
 #include "extractor/node_data_container.hpp"
 #include "extractor/suffix_table.hpp"
 
+#include "util/node_based_graph.hpp"
+#include "util/bearing.hpp"
+
 #include "util/guidance/name_announcements.hpp"
 
 namespace osrm
@@ -19,7 +22,7 @@ inline bool isThroughStreet(const std::size_t index,
                             const IntersectionType &intersection,
                             const util::NodeBasedDynamicGraph &node_based_graph,
                             const extractor::EdgeBasedNodeDataContainer &node_data_container,
-                            const util::NameTable &name_table,
+                            const extractor::NameTable &name_table,
                             const extractor::SuffixTable &street_name_suffix_table)
 {
     using osrm::util::angularDeviation;

include/guidance/motorway_handler.hpp

@@ -1,12 +1,13 @@
 #ifndef OSRM_GUIDANCE_MOTORWAY_HANDLER_HPP_
 #define OSRM_GUIDANCE_MOTORWAY_HANDLER_HPP_
 
+#include "extractor/name_table.hpp"
+
 #include "guidance/intersection.hpp"
 #include "guidance/intersection_handler.hpp"
 #include "guidance/is_through_street.hpp"
 
 #include "util/attributes.hpp"
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 
 #include <vector>
@@ -27,7 +28,7 @@ class MotorwayHandler : public IntersectionHandler
                     const extractor::RestrictionMap &node_restriction_map,
                     const std::unordered_set<NodeID> &barrier_nodes,
                     const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                    const util::NameTable &name_table,
+                    const extractor::NameTable &name_table,
                     const extractor::SuffixTable &street_name_suffix_table);
 
     ~MotorwayHandler() override final = default;

include/guidance/roundabout_handler.hpp

@@ -4,12 +4,13 @@
 #include "extractor/compressed_edge_container.hpp"
 #include "extractor/intersection/coordinate_extractor.hpp"
 #include "extractor/query_node.hpp"
+#include "extractor/name_table.hpp"
+
 #include "guidance/intersection.hpp"
 #include "guidance/intersection_handler.hpp"
 #include "guidance/is_through_street.hpp"
 #include "guidance/roundabout_type.hpp"
 
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 #include "util/typedefs.hpp"
 
@@ -44,7 +45,7 @@ class RoundaboutHandler : public IntersectionHandler
                       const extractor::RestrictionMap &node_restriction_map,
                       const std::unordered_set<NodeID> &barrier_nodes,
                       const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                      const util::NameTable &name_table,
+                      const extractor::NameTable &name_table,
                       const extractor::SuffixTable &street_name_suffix_table);
 
     ~RoundaboutHandler() override final = default;

include/guidance/segregated_intersection_classification.hpp

@@ -1,14 +1,11 @@
+#include "extractor/name_table.hpp"
+
 #include "util/typedefs.hpp"
 
 #include <unordered_set>
 
 namespace osrm
 {
-namespace util
-{
-class NameTable;
-}
-
 namespace extractor
 {
 class NodeBasedGraphFactory;
@@ -22,6 +19,6 @@ namespace guidance
 // - staggered intersections (X-cross)
 // - square/circle intersections
 std::unordered_set<EdgeID> findSegregatedNodes(const extractor::NodeBasedGraphFactory &factory,
-                                               const util::NameTable &names);
+                                               const extractor::NameTable &names);
 }
 }

include/guidance/sliproad_handler.hpp

@@ -1,11 +1,12 @@
 #ifndef OSRM_GUIDANCE_SLIPROAD_HANDLER_HPP_
 #define OSRM_GUIDANCE_SLIPROAD_HANDLER_HPP_
 
+#include "extractor/name_table.hpp"
+
 #include "guidance/intersection.hpp"
 #include "guidance/intersection_handler.hpp"
 #include "guidance/is_through_street.hpp"
 
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 
 #include <vector>
@@ -28,7 +29,7 @@ class SliproadHandler final : public IntersectionHandler
                     const extractor::RestrictionMap &node_restriction_map,
                     const std::unordered_set<NodeID> &barrier_nodes,
                     const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                    const util::NameTable &name_table,
+                    const extractor::NameTable &name_table,
                     const extractor::SuffixTable &street_name_suffix_table);
 
     ~SliproadHandler() override final = default;

include/guidance/statistics_handler.hpp

@@ -32,7 +32,7 @@ class StatisticsHandler final : public IntersectionHandler
                       const extractor::RestrictionMap &node_restriction_map,
                       const std::unordered_set<NodeID> &barrier_nodes,
                       const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                      const util::NameTable &name_table,
+                      const extractor::NameTable &name_table,
                       const extractor::SuffixTable &street_name_suffix_table)
         : IntersectionHandler(node_based_graph,
                               node_data_container,

include/guidance/suppress_mode_handler.hpp

@@ -1,8 +1,11 @@
 #ifndef OSRM_GUIDANCE_SUPPRESS_MODE_HANDLER_HPP_
 #define OSRM_GUIDANCE_SUPPRESS_MODE_HANDLER_HPP_
 
+#include "extractor/name_table.hpp"
+
 #include "guidance/intersection.hpp"
 #include "guidance/intersection_handler.hpp"
+
 #include "util/node_based_graph.hpp"
 
 namespace osrm
@@ -23,7 +26,7 @@ class SuppressModeHandler final : public IntersectionHandler
                         const extractor::RestrictionMap &node_restriction_map,
                         const std::unordered_set<NodeID> &barrier_nodes,
                         const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                        const util::NameTable &name_table,
+                        const extractor::NameTable &name_table,
                         const extractor::SuffixTable &street_name_suffix_table);
 
     ~SuppressModeHandler() override final = default;

include/guidance/turn_analysis.hpp

@@ -5,6 +5,8 @@
 #include "extractor/intersection/intersection_view.hpp"
 #include "extractor/restriction_index.hpp"
 #include "extractor/suffix_table.hpp"
+#include "extractor/name_table.hpp"
+
 #include "guidance/driveway_handler.hpp"
 #include "guidance/intersection.hpp"
 #include "guidance/motorway_handler.hpp"
@@ -16,7 +18,6 @@
 #include "guidance/turn_handler.hpp"
 
 #include "util/attributes.hpp"
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 
 #include <cstdint>
@@ -43,7 +44,7 @@ class TurnAnalysis
                  const extractor::RestrictionMap &restriction_map,
                  const std::unordered_set<NodeID> &barrier_nodes,
                  const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                 const util::NameTable &name_table,
+                 const extractor::NameTable &name_table,
                  const extractor::SuffixTable &street_name_suffix_table);
 
     /* Full Analysis Process for a single node/edge combination. Use with caution, as the process is

include/guidance/turn_handler.hpp

@@ -2,12 +2,13 @@
 #define OSRM_GUIDANCE_TURN_HANDLER_HPP_
 
 #include "extractor/query_node.hpp"
+#include "extractor/name_table.hpp"
+
 #include "guidance/intersection.hpp"
 #include "guidance/intersection_handler.hpp"
 #include "guidance/is_through_street.hpp"
 
 #include "util/attributes.hpp"
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 
 #include <boost/optional.hpp>
@@ -32,7 +33,7 @@ class TurnHandler : public IntersectionHandler
                 const extractor::RestrictionMap &node_restriction_map,
                 const std::unordered_set<NodeID> &barrier_nodes,
                 const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                const util::NameTable &name_table,
+                const extractor::NameTable &name_table,
                 const extractor::SuffixTable &street_name_suffix_table);
 
     ~TurnHandler() override final = default;

include/guidance/turn_lane_handler.hpp

@@ -1,15 +1,16 @@
 #ifndef OSRM_GUIDANCE_TURN_LANE_HANDLER_HPP_
 #define OSRM_GUIDANCE_TURN_LANE_HANDLER_HPP_
 
+#include "extractor/name_table.hpp"
 #include "extractor/query_node.hpp"
 #include "extractor/turn_lane_types.hpp"
+
 #include "guidance/intersection.hpp"
 #include "guidance/turn_analysis.hpp"
 #include "guidance/turn_lane_data.hpp"
 
 #include "util/attributes.hpp"
 #include "util/guidance/turn_lanes.hpp"
-#include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 #include "util/typedefs.hpp"
 

include/storage/shared_datatype.hpp

@@ -20,7 +20,9 @@ namespace storage
 // Added at the start and end of each block as sanity check
 const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'};
 
-const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
+const constexpr char *block_id_to_name[] = {"IGNORE_BLOCK",
+                                            "NAME_BLOCKS",
+                                            "NAME_VALUES",
                                             "EDGE_BASED_NODE_DATA",
                                             "ANNOTATION_DATA",
                                             "CH_GRAPH_NODE_LIST",
@@ -92,15 +94,16 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
                                             "MLD_GRAPH_EDGE_LIST",
                                             "MLD_GRAPH_NODE_TO_OFFSET",
                                             "MANEUVER_OVERRIDES",
-                                            "MANEUVER_OVERRIDE_NODE_SEQUENCES",
+                                            "MANEUVER_OVERRIDE_NODE_SEQUENCES"};
-                                            "IGNORE_BLOCK"};
 
 class DataLayout
 {
   public:
     enum BlockID
     {
-        NAME_CHAR_DATA = 0,
+        IGNORE_BLOCK = 0,
+        NAME_BLOCKS,
+        NAME_VALUES,
         EDGE_BASED_NODE_DATA_LIST,
         ANNOTATION_DATA_LIST,
         CH_GRAPH_NODE_LIST,
@@ -173,7 +176,6 @@ class DataLayout
         MLD_GRAPH_NODE_TO_OFFSET,
         MANEUVER_OVERRIDES,
         MANEUVER_OVERRIDE_NODE_SEQUENCES,
-        IGNORE_BLOCK,
         NUM_BLOCKS
     };
 

include/util/guidance/name_announcements.hpp

@@ -3,9 +3,10 @@
 
 /* A set of tools required for guidance in both pre and post-processing */
 
+#include "extractor/name_table.hpp"
 #include "extractor/suffix_table.hpp"
+
 #include "util/attributes.hpp"
-#include "util/name_table.hpp"
 #include "util/typedefs.hpp"
 
 #include <algorithm>
@@ -216,7 +217,7 @@ inline bool requiresNameAnnounced(const std::string &from_name,
 
 inline bool requiresNameAnnounced(const NameID from_name_id,
                                   const NameID to_name_id,
-                                  const util::NameTable &name_table,
+                                  const extractor::NameTable &name_table,
                                   const extractor::SuffixTable &suffix_table)
 {
     if (from_name_id == to_name_id)

include/util/indexed_data.hpp

@@ -1,12 +1,14 @@
 #ifndef OSRM_INDEXED_DATA_HPP
 #define OSRM_INDEXED_DATA_HPP
 
-#include "storage/io.hpp"
+#include "storage/tar_fwd.hpp"
 
 #include "util/exception.hpp"
 #include "util/string_view.hpp"
+#include "util/vector_view.hpp"
 
 #include <boost/assert.hpp>
+#include <boost/function_output_iterator.hpp>
 
 #include <array>
 #include <iterator>
@@ -18,6 +20,23 @@ namespace osrm
 {
 namespace util
 {
+namespace detail
+{
+template <typename GroupBlockPolicy, storage::Ownership Ownership> struct IndexedDataImpl;
+}
+
+namespace serialization
+{
+template <typename BlockPolicy, storage::Ownership Ownership>
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::IndexedDataImpl<BlockPolicy, Ownership> &index_data);
+
+template <typename BlockPolicy, storage::Ownership Ownership>
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const detail::IndexedDataImpl<BlockPolicy, Ownership> &index_data);
+}
 
 template <int N, typename T = std::string> struct VariableGroupBlock
 {
@@ -85,11 +104,11 @@ template <int N, typename T = std::string> struct VariableGroupBlock
     /// of prefix length. sum(descriptor) equals to the block
     /// prefix length.
     /// Returns the block prefix length.
-    template <typename Offset, typename OffsetIterator>
+    template <typename Offset, typename OffsetIterator, typename OutIter>
-    Offset WriteBlockReference(storage::io::FileWriter &out,
+    OutIter WriteBlockReference(OffsetIterator first,
-                               Offset data_offset,
+                                OffsetIterator last,
-                               OffsetIterator first,
+                                Offset &data_offset,
-                               OffsetIterator last) const
+                                OutIter out) const
     {
         BOOST_ASSERT(data_offset <= std::numeric_limits<decltype(BlockReference::offset)>::max());
 
@@ -106,9 +125,9 @@ template <int N, typename T = std::string> struct VariableGroupBlock
             prefix_length += byte_length;
         }
 
-        out.WriteFrom(refernce);
+        data_offset += prefix_length;
-
+        *out++ = refernce;
-        return prefix_length;
+        return out;
     }
 
     /// Write a block prefix that is an array of variable encoded data lengths:
@@ -118,9 +137,8 @@ template <int N, typename T = std::string> struct VariableGroupBlock
     ///   65536..16777215 is 3 bytes.
     /// [first..last] is an inclusive range of block data.
     /// The length of the last item in the block is not stored.
-    template <typename OffsetIterator>
+    template <typename OffsetIterator, typename OutByteIter>
-    void
+    OutByteIter WriteBlockPrefix(OffsetIterator first, OffsetIterator last, OutByteIter out) const
-    WriteBlockPrefix(storage::io::FileWriter &out, OffsetIterator first, OffsetIterator last) const
     {
         for (OffsetIterator curr = first, next = std::next(first); curr != last; ++curr, ++next)
         {
@@ -131,8 +149,9 @@ template <int N, typename T = std::string> struct VariableGroupBlock
 
             // Here, we're only writing a few bytes from the 4-byte std::uint32_t,
             // so we need to cast to (char *)
-            out.WriteFrom((const char *)&data_length, byte_length);
+            out = std::copy_n((const char *)&data_length, byte_length, out);
         }
+        return out;
     }
 
     /// Advances the range to an item stored in the referenced block.
@@ -178,36 +197,39 @@ template <int N, typename T = std::string> struct FixedGroupBlock
 
     /// Write a block reference {offset}, where offset is a global block offset
     /// Returns the fixed block prefix length.
-    template <typename Offset, typename OffsetIterator>
+    template <typename Offset, typename OffsetIterator, typename OutIterator>
-    Offset WriteBlockReference(storage::io::FileWriter &out,
+    OutIterator
-                               Offset data_offset,
+    WriteBlockReference(OffsetIterator, OffsetIterator, Offset &data_offset, OutIterator out) const
-                               OffsetIterator,
-                               OffsetIterator) const
     {
         BOOST_ASSERT(data_offset <= std::numeric_limits<decltype(BlockReference::offset)>::max());
 
         BlockReference refernce{static_cast<decltype(BlockReference::offset)>(data_offset)};
-        out.WriteFrom(refernce);
+        data_offset += BLOCK_SIZE;
+        *out++ = refernce;
 
-        return BLOCK_SIZE;
+        return out;
     }
 
     /// Write a fixed length block prefix.
-    template <typename OffsetIterator>
+    template <typename OffsetIterator, typename OutByteIter>
-    void
+    OutByteIter WriteBlockPrefix(OffsetIterator first, OffsetIterator last, OutByteIter out) const
-    WriteBlockPrefix(storage::io::FileWriter &out, OffsetIterator first, OffsetIterator last) const
     {
-        std::uint32_t index = 0;
+        constexpr std::size_t MAX_LENGTH = std::numeric_limits<std::make_unsigned_t<ValueType>>::max();
-        std::array<ValueType, BLOCK_SIZE> block_prefix;
+
+        auto index = 0;
+        std::array<ValueType, BLOCK_SIZE> prefix;
+
         for (OffsetIterator curr = first, next = std::next(first); curr != last; ++curr, ++next)
         {
             const std::uint32_t data_length = *next - *curr;
-            if (data_length >= 0x100)
+            if (data_length > MAX_LENGTH)
-                throw util::exception(boost::format("too large data length %1%") % data_length);
+                throw util::exception(boost::format("too large data length %1% > %2%") % data_length % MAX_LENGTH);
 
-            block_prefix[index++] = static_cast<ValueType>(data_length);
+            prefix[index++] = data_length;
         }
-        out.WriteFrom(block_prefix.data(), block_prefix.size());
+
+        out = std::copy_n((const char *)prefix.data(), sizeof(ValueType)*BLOCK_SIZE, out);
+        return out;
     }
 
     /// Advances the range to an item stored in the referenced block.
@@ -233,28 +255,31 @@ template <int N, typename T = std::string> struct FixedGroupBlock
     }
 };
 
-template <typename GroupBlock> struct IndexedData
+namespace detail
 {
-    static constexpr std::uint32_t BLOCK_SIZE = GroupBlock::BLOCK_SIZE;
+template <typename GroupBlockPolicy, storage::Ownership Ownership> struct IndexedDataImpl
+{
+    static constexpr std::uint32_t BLOCK_SIZE = GroupBlockPolicy::BLOCK_SIZE;
 
     using BlocksNumberType = std::uint32_t;
     using DataSizeType = std::uint64_t;
 
-    using BlockReference = typename GroupBlock::BlockReference;
+    using BlockReference = typename GroupBlockPolicy::BlockReference;
-    using ResultType = typename GroupBlock::ResultType;
+    using ResultType = typename GroupBlockPolicy::ResultType;
-    using ValueType = typename GroupBlock::ValueType;
+    using ValueType = typename GroupBlockPolicy::ValueType;
 
     static_assert(sizeof(ValueType) == 1, "data basic type must char");
 
-    IndexedData() : blocks_number{0}, block_references{nullptr}, begin{nullptr}, end{nullptr} {}
+    IndexedDataImpl() = default;
+    IndexedDataImpl(util::vector_view<BlockReference> blocks_, util::vector_view<ValueType> values_)
+        : blocks(std::move(blocks_)), values(std::move(values_))
+    {
+    }
 
-    bool empty() const { return blocks_number == 0; }
+    bool empty() const { return blocks.empty(); }
 
     template <typename OffsetIterator, typename DataIterator>
-    void write(storage::io::FileWriter &out,
+    IndexedDataImpl(OffsetIterator first, OffsetIterator last, DataIterator data)
-               OffsetIterator first,
-               OffsetIterator last,
-               DataIterator data) const
     {
         static_assert(sizeof(typename DataIterator::value_type) == 1, "data basic type must char");
 
@@ -268,69 +293,37 @@ template <typename GroupBlock> struct IndexedData
         const BlocksNumberType number_of_blocks =
             number_of_elements == 0 ? 0
                                     : 1 + (std::distance(first, sentinel) - 1) / (BLOCK_SIZE + 1);
-        out.WriteFrom(number_of_blocks);
+        blocks.resize(number_of_blocks);
 
         // Write block references and compute the total data size that includes prefix and data
-        const GroupBlock block;
+        const GroupBlockPolicy block;
+
+        auto block_iter = blocks.begin();
         DataSizeType data_size = 0;
         for (OffsetIterator curr = first, next = first; next != sentinel; curr = next)
         {
             std::advance(next, std::min<diff_type>(BLOCK_SIZE, std::distance(next, sentinel)));
-            data_size += block.WriteBlockReference(out, data_size, curr, next);
+            block_iter = block.WriteBlockReference(curr, next, data_size, block_iter);
             std::advance(next, std::min<diff_type>(1, std::distance(next, sentinel)));
             data_size += *next - *curr;
         }
 
-        // Write the total data size
+        values.resize(data_size);
-        out.WriteFrom(data_size);
+        auto values_byte_iter = reinterpret_cast<char *>(values.data());
-
         // Write data blocks that are (prefix, data)
         for (OffsetIterator curr = first, next = first; next != sentinel; curr = next)
         {
             std::advance(next, std::min<diff_type>(BLOCK_SIZE, std::distance(next, sentinel)));
-            block.WriteBlockPrefix(out, curr, next);
+            values_byte_iter = block.WriteBlockPrefix(curr, next, values_byte_iter);
             std::advance(next, std::min<diff_type>(1, std::distance(next, sentinel)));
-            std::for_each(
+
-                data + *curr, data + *next, [&out](const auto &element) { out.WriteFrom(element); });
+            auto to_bytes = [&](const auto &data) {
+                values_byte_iter = std::copy_n(&data, sizeof(ValueType), values_byte_iter);
+            };
+            std::copy(data + *curr, data + *next, boost::make_function_output_iterator(to_bytes));
         }
     }
 
-    /// Set internal pointers from the buffer [first, last).
-    /// Data buffer pointed by ptr must exists during IndexedData life-time.
-    /// No ownership is transferred.
-    void reset(const ValueType *first, const ValueType *last)
-    {
-        // Read blocks number
-        if (first + sizeof(BlocksNumberType) > last)
-            throw util::exception("incorrect memory block");
-
-        blocks_number = *reinterpret_cast<const BlocksNumberType *>(first);
-        first += sizeof(BlocksNumberType);
-
-        // Get block references pointer
-        if (first + sizeof(BlockReference) * blocks_number > last)
-            throw util::exception("incorrect memory block");
-
-        block_references = reinterpret_cast<const BlockReference *>(first);
-        first += sizeof(BlockReference) * blocks_number;
-
-        // Read total data size
-        if (first + sizeof(DataSizeType) > last)
-            throw util::exception("incorrect memory block");
-
-        auto data_size = *reinterpret_cast<const DataSizeType *>(first);
-        first += sizeof(DataSizeType);
-
-        // Get data blocks begin and end iterators
-        begin = reinterpret_cast<const ValueType *>(first);
-        first += sizeof(ValueType) * data_size;
-
-        if (first > last)
-            throw util::exception("incorrect memory block");
-
-        end = reinterpret_cast<const ValueType *>(first);
-    }
-
     // Return value at the given index
     ResultType at(std::uint32_t index) const
     {
@@ -338,20 +331,29 @@ template <typename GroupBlock> struct IndexedData
         const BlocksNumberType block_idx = index / (BLOCK_SIZE + 1);
         const std::uint32_t internal_idx = index % (BLOCK_SIZE + 1);
 
-        if (block_idx >= blocks_number)
+        if (block_idx >= blocks.size())
             return ResultType();
 
         // Get block first and last iterators
-        auto first = begin + block_references[block_idx].offset;
+        auto first = values.begin() + blocks[block_idx].offset;
-        auto last =
+        auto last = block_idx + 1 == blocks.size() ? values.end()
-            block_idx + 1 == blocks_number ? end : begin + block_references[block_idx + 1].offset;
+                                                   : values.begin() + blocks[block_idx + 1].offset;
 
-        const GroupBlock block;
+        const GroupBlockPolicy block;
-        block.ReadRefrencedBlock(block_references[block_idx], internal_idx, first, last);
+        block.ReadRefrencedBlock(blocks[block_idx], internal_idx, first, last);
 
-        return adapt(first, last);
+        return adapt(&*first, &*last);
     }
 
+    friend void serialization::read<GroupBlockPolicy, Ownership>(storage::tar::FileReader &reader,
+                                                                 const std::string &name,
+                                                                 IndexedDataImpl &index_data);
+
+    friend void
+    serialization::write<GroupBlockPolicy, Ownership>(storage::tar::FileWriter &writer,
+                                                      const std::string &name,
+                                                      const IndexedDataImpl &index_data);
+
   private:
     template <class T = ResultType>
     typename std::enable_if<!std::is_same<T, StringView>::value, T>::type
@@ -367,10 +369,16 @@ template <typename GroupBlock> struct IndexedData
         return ResultType(first, std::distance(first, last));
     }
 
-    BlocksNumberType blocks_number;
+    template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
-    const BlockReference *block_references;
+    Vector<BlockReference> blocks;
-    const ValueType *begin, *end;
+    Vector<ValueType> values;
 };
 }
+
+template <typename GroupBlockPolicy>
+using IndexedData = detail::IndexedDataImpl<GroupBlockPolicy, storage::Ownership::Container>;
+template <typename GroupBlockPolicy>
+using IndexedDataView = detail::IndexedDataImpl<GroupBlockPolicy, storage::Ownership::View>;
+}
 }
 #endif // OSRM_INDEXED_DATA_HPP

include/util/name_table.hpp

@@ -1,51 +0,0 @@
-#ifndef OSRM_UTIL_NAME_TABLE_HPP
-#define OSRM_UTIL_NAME_TABLE_HPP
-
-#include "util/indexed_data.hpp"
-#include "util/string_view.hpp"
-#include "util/typedefs.hpp"
-
-#include <string>
-
-namespace osrm
-{
-namespace util
-{
-
-// While this could, theoretically, hold any names in the fitting format,
-// the NameTable allows access to a part of the Datafacade to allow
-// processing based on name indices.
-class NameTable
-{
-  public:
-    using IndexedData = util::IndexedData<util::VariableGroupBlock<16, util::StringView>>;
-    using ResultType = IndexedData::ResultType;
-    using ValueType = IndexedData::ValueType;
-
-    NameTable() {}
-
-    // Read filename and store own data in m_buffer
-    NameTable(const std::string &filename);
-
-    // Keep pointers only in m_name_table and don't own data in m_buffer
-    void reset(ValueType *begin, ValueType *end);
-
-    // This class provides a limited view over all the string data we serialize out.
-    // The following functions are a subset of what is available.
-    // See the data facades for they provide full access to this serialized string data.
-    util::StringView GetNameForID(const NameID id) const;
-    util::StringView GetDestinationsForID(const NameID id) const;
-    util::StringView GetExitsForID(const NameID id) const;
-    util::StringView GetRefForID(const NameID id) const;
-    util::StringView GetPronunciationForID(const NameID id) const;
-
-  private:
-    using BufferType = std::unique_ptr<ValueType, std::function<void(void *)>>;
-
-    BufferType m_buffer;
-    IndexedData m_name_table;
-};
-} // namespace util
-} // namespace osrm
-
-#endif // OSRM_UTIL_NAME_TABLE_HPP

include/util/serialization.hpp

@@ -5,6 +5,7 @@
 #include "util/packed_vector.hpp"
 #include "util/range_table.hpp"
 #include "util/static_graph.hpp"
+#include "util/indexed_data.hpp"
 
 #include "storage/io.hpp"
 #include "storage/serialization.hpp"
@@ -168,6 +169,22 @@ inline void write(storage::tar::FileWriter &writer,
     writer.WriteStreaming<typename std::remove_reference_t<decltype(graph)>::Edge>(
         name + "/edge_list", graph.edge_list.begin(), graph.number_of_edges);
 }
+
+template <typename BlockPolicy, storage::Ownership Ownership>
+inline void
+read(storage::tar::FileReader &reader, const std::string &name, detail::IndexedDataImpl<BlockPolicy, Ownership> &index_data)
+{
+    storage::serialization::read(reader, name + "/blocks", index_data.blocks);
+    storage::serialization::read(reader, name + "/values", index_data.values);
+}
+
+template <typename BlockPolicy, storage::Ownership Ownership>
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name, const detail::IndexedDataImpl<BlockPolicy, Ownership> &index_data)
+{
+    storage::serialization::write(writer, name + "/blocks", index_data.blocks);
+    storage::serialization::write(writer, name + "/values", index_data.values);
+}
 }
 }
 }

src/extractor/edge_based_graph_factory.cpp

@@ -69,7 +69,7 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory(
     const std::unordered_set<NodeID> &barrier_nodes,
     const std::unordered_set<NodeID> &traffic_lights,
     const std::vector<util::Coordinate> &coordinates,
-    const util::NameTable &name_table,
+    const NameTable &name_table,
     const std::unordered_set<EdgeID> &segregated_edges,
     const extractor::LaneDescriptionMap &lane_description_map)
     : m_edge_based_node_container(node_data_container), m_connectivity_checksum(0),

src/extractor/extraction_containers.cpp

@@ -3,6 +3,8 @@
 #include "extractor/extraction_way.hpp"
 #include "extractor/restriction.hpp"
 #include "extractor/serialization.hpp"
+#include "extractor/name_table.hpp"
+#include "extractor/files.hpp"
 
 #include "util/coordinate_calculation.hpp"
 
@@ -10,7 +12,6 @@
 #include "util/exception_utils.hpp"
 #include "util/fingerprint.hpp"
 #include "util/log.hpp"
-#include "util/name_table.hpp"
 #include "util/timing_util.hpp"
 
 #include "storage/io.hpp"
@@ -163,10 +164,8 @@ void ExtractionContainers::WriteCharData(const std::string &file_name)
     util::UnbufferedLog log;
     log << "writing street name index ... ";
     TIMER_START(write_index);
-    storage::io::FileWriter file(file_name, storage::io::FileWriter::GenerateFingerprint);
 
-    const util::NameTable::IndexedData indexed_data;
+    files::writeNames(file_name, NameTable {NameTable::IndexedData (name_offsets.begin(), name_offsets.end(), name_char_data.begin())});
-    indexed_data.write(file, name_offsets.begin(), name_offsets.end(), name_char_data.begin());
 
     TIMER_STOP(write_index);
     log << "ok, after " << TIMER_SEC(write_index) << "s";

src/extractor/extractor.cpp

@@ -13,6 +13,7 @@
 #include "extractor/restriction_filter.hpp"
 #include "extractor/restriction_parser.hpp"
 #include "extractor/scripting_environment.hpp"
+#include "extractor/name_table.hpp"
 
 #include "guidance/files.hpp"
 #include "guidance/guidance_processing.hpp"
@@ -25,7 +26,6 @@
 #include "util/exception_utils.hpp"
 #include "util/integer_range.hpp"
 #include "util/log.hpp"
-#include "util/name_table.hpp"
 #include "util/range_table.hpp"
 #include "util/timing_util.hpp"
 
@@ -230,11 +230,13 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
                                                    conditional_turn_restrictions,
                                                    unresolved_maneuver_overrides);
 
+    NameTable name_table;
+    files::readNames(config.GetPath(".osrm.names"), name_table);
+
     util::Log() << "Find segregated edges in node-based graph ..." << std::flush;
     TIMER_START(segregated);
 
-    util::NameTable names(config.GetPath(".osrm.names").string());
+    auto segregated_edges = guidance::findSegregatedNodes(node_based_graph_factory, name_table);
-    auto segregated_edges = guidance::findSegregatedNodes(node_based_graph_factory, names);
 
     TIMER_STOP(segregated);
     util::Log() << "ok, after " << TIMER_SEC(segregated) << "s";
@@ -283,8 +285,6 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
 
     const auto number_of_node_based_nodes = node_based_graph.GetNumberOfNodes();
 
-    const util::NameTable name_table(config.GetPath(".osrm.names").string());
-
     const auto number_of_edge_based_nodes =
         BuildEdgeExpandedGraph(node_based_graph,
                                coordinates,
@@ -713,7 +713,7 @@ EdgeID Extractor::BuildEdgeExpandedGraph(
     const std::vector<TurnRestriction> &turn_restrictions,
     const std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions,
     const std::unordered_set<EdgeID> &segregated_edges,
-    const util::NameTable &name_table,
+    const NameTable &name_table,
     const std::vector<UnresolvedManeuverOverride> &maneuver_overrides,
     const LaneDescriptionMap &turn_lane_map,
     // for calculating turn penalties
@@ -884,7 +884,7 @@ void Extractor::ProcessGuidanceTurns(
     const std::unordered_set<NodeID> &barrier_nodes,
     const std::vector<TurnRestriction> &turn_restrictions,
     const std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions,
-    const util::NameTable &name_table,
+    const NameTable &name_table,
     LaneDescriptionMap lane_description_map,
     ScriptingEnvironment &scripting_environment)
 {

src/extractor/intersection/have_identical_names.cpp

@@ -1,3 +1,5 @@
+#include "extractor/intersection/have_identical_names.hpp"
+
 #include "util/guidance/name_announcements.hpp"
 
 namespace osrm
@@ -12,7 +14,7 @@ namespace intersection
 // rhs->lhs)
 bool HaveIdenticalNames(const NameID lhs,
                         const NameID rhs,
-                        const util::NameTable &name_table,
+                        const NameTable &name_table,
                         const extractor::SuffixTable &street_name_suffix_table)
 {
     const auto non_empty = (lhs != EMPTY_NAMEID) && (rhs != EMPTY_NAMEID);

src/extractor/intersection/mergable_road_detector.cpp

@@ -3,12 +3,12 @@
 #include "extractor/intersection/node_based_graph_walker.hpp"
 #include "extractor/query_node.hpp"
 #include "extractor/suffix_table.hpp"
+#include "extractor/name_table.hpp"
 #include "guidance/constants.hpp"
 
 #include "util/bearing.hpp"
 #include "util/coordinate_calculation.hpp"
 #include "util/guidance/name_announcements.hpp"
-#include "util/name_table.hpp"
 
 using osrm::util::angularDeviation;
 
@@ -27,7 +27,7 @@ namespace
 inline auto makeCheckRoadForName(const NameID name_id,
                                  const util::NodeBasedDynamicGraph &node_based_graph,
                                  const EdgeBasedNodeDataContainer &node_data_container,
-                                 const util::NameTable &name_table,
+                                 const NameTable &name_table,
                                  const SuffixTable &suffix_table)
 {
     return [name_id, &node_based_graph, &node_data_container, &name_table, &suffix_table](
@@ -59,7 +59,7 @@ MergableRoadDetector::MergableRoadDetector(
     const RestrictionMap &node_restriction_map,
     const std::unordered_set<NodeID> &barrier_nodes,
     const extractor::TurnLanesIndexedArray &turn_lanes_data,
-    const util::NameTable &name_table,
+    const NameTable &name_table,
     const SuffixTable &street_name_suffix_table)
     : node_based_graph(node_based_graph), node_data_container(node_data_container),
       node_coordinates(node_coordinates), compressed_geometries(compressed_geometries),

src/guidance/driveway_handler.cpp

@@ -17,7 +17,7 @@ DrivewayHandler::DrivewayHandler(const util::NodeBasedDynamicGraph &node_based_g
                                  const extractor::RestrictionMap &node_restriction_map,
                                  const std::unordered_set<NodeID> &barrier_nodes,
                                  const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                                 const util::NameTable &name_table,
+                                 const extractor::NameTable &name_table,
                                  const extractor::SuffixTable &street_name_suffix_table)
     : IntersectionHandler(node_based_graph,
                           node_data_container,

src/guidance/guidance_processing.cpp

@@ -24,7 +24,7 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
                    const std::unordered_set<NodeID> &barrier_nodes,
                    const extractor::RestrictionMap &node_restriction_map,
                    const extractor::WayRestrictionMap &way_restriction_map,
-                   const util::NameTable &name_table,
+                   const extractor::NameTable &name_table,
                    const extractor::SuffixTable &suffix_table,
                    const extractor::TurnLanesIndexedArray &turn_lanes_data,
                    extractor::LaneDescriptionMap &lane_description_map,

src/guidance/intersection_handler.cpp

@@ -52,7 +52,7 @@ IntersectionHandler::IntersectionHandler(
     const extractor::RestrictionMap &node_restriction_map,
     const std::unordered_set<NodeID> &barrier_nodes,
     const extractor::TurnLanesIndexedArray &turn_lanes_data,
-    const util::NameTable &name_table,
+    const extractor::NameTable &name_table,
     const extractor::SuffixTable &street_name_suffix_table)
     : node_based_graph(node_based_graph), node_data_container(node_data_container),
       node_coordinates(node_coordinates), compressed_geometries(compressed_geometries),

src/guidance/motorway_handler.cpp

@@ -49,7 +49,7 @@ MotorwayHandler::MotorwayHandler(const util::NodeBasedDynamicGraph &node_based_g
                                  const extractor::RestrictionMap &node_restriction_map,
                                  const std::unordered_set<NodeID> &barrier_nodes,
                                  const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                                 const util::NameTable &name_table,
+                                 const extractor::NameTable &name_table,
                                  const extractor::SuffixTable &street_name_suffix_table)
     : IntersectionHandler(node_based_graph,
                           node_data_container,

src/guidance/roundabout_handler.cpp

@@ -30,7 +30,7 @@ RoundaboutHandler::RoundaboutHandler(
     const extractor::RestrictionMap &node_restriction_map,
     const std::unordered_set<NodeID> &barrier_nodes,
     const extractor::TurnLanesIndexedArray &turn_lanes_data,
-    const util::NameTable &name_table,
+    const extractor::NameTable &name_table,
     const extractor::SuffixTable &street_name_suffix_table)
     : IntersectionHandler(node_based_graph,
                           node_data_container,

src/guidance/segregated_intersection_classification.cpp

@@ -4,7 +4,6 @@
 #include "guidance/turn_instruction.hpp"
 
 #include "util/coordinate_calculation.hpp"
-#include "util/name_table.hpp"
 #include <set>
 
 using osrm::guidance::getTurnDirection;
@@ -42,7 +41,7 @@ struct EdgeInfo
 };
 
 std::unordered_set<EdgeID> findSegregatedNodes(const extractor::NodeBasedGraphFactory &factory,
-                                               const util::NameTable &names)
+                                               const extractor::NameTable &names)
 {
     auto const &graph = factory.GetGraph();
     auto const &annotation = factory.GetAnnotationData();

src/guidance/sliproad_handler.cpp

@@ -27,7 +27,7 @@ SliproadHandler::SliproadHandler(const util::NodeBasedDynamicGraph &node_based_g
                                  const extractor::RestrictionMap &node_restriction_map,
                                  const std::unordered_set<NodeID> &barrier_nodes,
                                  const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                                 const util::NameTable &name_table,
+                                 const extractor::NameTable &name_table,
                                  const extractor::SuffixTable &street_name_suffix_table)
     : IntersectionHandler(node_based_graph,
                           node_data_container,

src/guidance/suppress_mode_handler.cpp

@@ -17,7 +17,7 @@ SuppressModeHandler::SuppressModeHandler(
     const extractor::RestrictionMap &node_restriction_map,
     const std::unordered_set<NodeID> &barrier_nodes,
     const extractor::TurnLanesIndexedArray &turn_lanes_data,
-    const util::NameTable &name_table,
+    const extractor::NameTable &name_table,
     const extractor::SuffixTable &street_name_suffix_table)
     : IntersectionHandler(node_based_graph,
                           node_data_container,

src/guidance/turn_analysis.cpp

@@ -26,7 +26,7 @@ TurnAnalysis::TurnAnalysis(const util::NodeBasedDynamicGraph &node_based_graph,
                            const extractor::RestrictionMap &restriction_map,
                            const std::unordered_set<NodeID> &barrier_nodes,
                            const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                           const util::NameTable &name_table,
+                           const extractor::NameTable &name_table,
                            const extractor::SuffixTable &street_name_suffix_table)
     : node_based_graph(node_based_graph), roundabout_handler(node_based_graph,
                                                              node_data_container,

src/guidance/turn_handler.cpp

@@ -115,7 +115,7 @@ TurnHandler::TurnHandler(const util::NodeBasedDynamicGraph &node_based_graph,
                          const extractor::RestrictionMap &node_restriction_map,
                          const std::unordered_set<NodeID> &barrier_nodes,
                          const extractor::TurnLanesIndexedArray &turn_lanes_data,
-                         const util::NameTable &name_table,
+                         const extractor::NameTable &name_table,
                          const extractor::SuffixTable &street_name_suffix_table)
     : IntersectionHandler(node_based_graph,
                           node_data_container,

src/osrm/osrm.cpp

@@ -35,7 +35,7 @@ OSRM::OSRM(engine::EngineConfig &config)
 
         auto mem = storage::makeSharedMemory(barrier.data().region);
         auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
-        if (layout->GetBlockSize(storage::DataLayout::NAME_CHAR_DATA) == 0)
+        if (layout->GetBlockSize(storage::DataLayout::NAME_VALUES) == 0)
             throw util::exception(
                 "No name data loaded, cannot continue.  Have you run osrm-datastore to load data?");
     }

src/storage/storage.cpp

@@ -18,6 +18,7 @@
 #include "extractor/edge_based_node.hpp"
 #include "extractor/files.hpp"
 #include "extractor/maneuver_override.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/packed_osm_ids.hpp"
 #include "extractor/profile_properties.hpp"
 #include "extractor/query_node.hpp"
@@ -243,13 +244,6 @@ void Storage::PopulateLayout(DataLayout &layout)
                         make_block<char>(absolute_file_index_path.string().length() + 1));
     }
 
-    {
-        util::Log() << "load names from: " << config.GetPath(".osrm.names");
-        // number of entries in name index
-        io::FileReader name_file(config.GetPath(".osrm.names"), io::FileReader::VerifyFingerprint);
-        layout.SetBlock(DataLayout::NAME_CHAR_DATA, make_block<char>(name_file.GetSize()));
-    }
-
     // load rsearch tree size
     {
         io::FileReader tree_node_file(config.GetPath(".osrm.ramIndex"),
@@ -345,6 +339,8 @@ void Storage::PopulateLayout(DataLayout &layout)
         {"/common/turn_data/lane_data_ids", DataLayout::LANE_DATA_ID},
         {"/common/turn_data/entry_class_ids", DataLayout::ENTRY_CLASSID},
         {"/common/turn_data/connectivity_checksum", DataLayout::IGNORE_BLOCK},
+        {"/common/names/blocks", DataLayout::NAME_BLOCKS},
+        {"/common/names/values", DataLayout::NAME_VALUES},
     };
     std::vector<NamedBlock> blocks;
 
@@ -368,6 +364,7 @@ void Storage::PopulateLayout(DataLayout &layout)
         {REQUIRED, config.GetPath(".osrm.turn_weight_penalties")},
         {REQUIRED, config.GetPath(".osrm.turn_duration_penalties")},
         {REQUIRED, config.GetPath(".osrm.edges")},
+        {REQUIRED, config.GetPath(".osrm.names")},
     };
 
     for (const auto &file : tar_files)
@@ -425,14 +422,21 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Name data
     {
-        io::FileReader name_file(config.GetPath(".osrm.names"), io::FileReader::VerifyFingerprint);
+        const auto name_blocks_ptr =
-        std::size_t name_file_size = name_file.GetSize();
+            layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference, true>(
+                memory_ptr, DataLayout::NAME_BLOCKS);
+        const auto name_values_ptr =
+            layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType, true>(
+                memory_ptr, DataLayout::NAME_VALUES);
 
-        BOOST_ASSERT(name_file_size == layout.GetBlockSize(DataLayout::NAME_CHAR_DATA));
+        util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
-        const auto name_char_ptr =
+            name_blocks_ptr, layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS));
-            layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::NAME_CHAR_DATA);
+        util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
+            name_values_ptr, layout.GetBlockEntries(storage::DataLayout::NAME_VALUES));
 
-        name_file.ReadInto<char>(name_char_ptr, name_file_size);
+        extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
+        extractor::NameTableView name_table{index_data_view};
+        extractor::files::readNames(config.GetPath(".osrm.names"), name_table);
     }
 
     // Turn lane data
@@ -624,7 +628,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
             turn_duration_penalties_ptr,
             layout.GetBlockEntries(storage::DataLayout::TURN_WEIGHT_PENALTIES));
         extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"),
-                                                  turn_duration_penalties);
+                                                turn_duration_penalties);
     }
 
     // load turn duration penalties

src/util/name_table.cpp

@@ -1,100 +0,0 @@
-#include "util/name_table.hpp"
-#include "storage/io.hpp"
-#include "util/log.hpp"
-
-namespace osrm
-{
-namespace util
-{
-
-NameTable::NameTable(const std::string &file_name)
-{
-    using FileReader = storage::io::FileReader;
-
-    FileReader name_stream_file_reader(file_name, FileReader::VerifyFingerprint);
-    const auto file_size = name_stream_file_reader.GetSize();
-
-    m_buffer = BufferType(static_cast<ValueType *>(::operator new(file_size)),
-                          [](void *ptr) { ::operator delete(ptr); });
-    name_stream_file_reader.ReadInto<char>(m_buffer.get(), file_size);
-    m_name_table.reset(m_buffer.get(), m_buffer.get() + file_size);
-
-    if (m_name_table.empty())
-    {
-        util::Log() << "list of street names is empty in construction of name table from: \""
-                    << file_name << "\"";
-    }
-}
-
-void NameTable::reset(ValueType *begin, ValueType *end)
-{
-    m_buffer.reset();
-    m_name_table.reset(begin, end);
-}
-
-StringView NameTable::GetNameForID(const NameID id) const
-{
-    if (id == INVALID_NAMEID)
-        return {};
-
-    return m_name_table.at(id + 0);
-}
-
-StringView NameTable::GetDestinationsForID(const NameID id) const
-{
-    if (id == INVALID_NAMEID)
-        return {};
-
-    return m_name_table.at(id + 1);
-}
-
-StringView NameTable::GetExitsForID(const NameID id) const
-{
-    if (id == INVALID_NAMEID)
-        return {};
-
-    return m_name_table.at(id + 4);
-}
-
-StringView NameTable::GetRefForID(const NameID id) const
-{
-    if (id == INVALID_NAMEID)
-        return {};
-
-    // Way string data is stored in blocks based on `id` as follows:
-    //
-    // | name | destination | pronunciation | ref | exits
-    //                                      ^     ^
-    //                                      [range)
-    //                                       ^ id + 3
-    //
-    // `id + offset` gives us the range of chars.
-    //
-    // Offset 0 is name, 1 is destination, 2 is pronunciation, 3 is ref, 4 is exits
-    // See datafacades and extractor callbacks for details.
-    const constexpr auto OFFSET_REF = 3u;
-    return m_name_table.at(id + OFFSET_REF);
-}
-
-StringView NameTable::GetPronunciationForID(const NameID id) const
-{
-    if (id == INVALID_NAMEID)
-        return {};
-
-    // Way string data is stored in blocks based on `id` as follows:
-    //
-    // | name | destination | pronunciation | ref | exits
-    //                      ^               ^
-    //                      [range)
-    //                       ^ id + 2
-    //
-    // `id + offset` gives us the range of chars.
-    //
-    // Offset 0 is name, 1 is destination, 2 is pronunciation, 3 is ref, 4 is exits
-    // See datafacades and extractor callbacks for details.
-    const constexpr auto OFFSET_PRONUNCIATION = 2u;
-    return m_name_table.at(id + OFFSET_PRONUNCIATION);
-}
-
-} // namespace util
-} // namespace osrm

unit_tests/extractor/name_table.cpp

@@ -1,4 +1,4 @@
-#include "util/name_table.hpp"
+#include "extractor/name_table.hpp"
 #include "common/temporary_file.hpp"
 #include "util/exception.hpp"
 
@@ -12,14 +12,12 @@
 #include <typeinfo>
 #include <vector>
 
-//#include <valgrind/callgrind.h>
-
 BOOST_AUTO_TEST_SUITE(name_table)
 
 using namespace osrm;
-using namespace osrm::util;
+using namespace osrm::extractor;
 
-std::string PrapareNameTableData(std::vector<std::string> &data, bool fill_all)
+NameTable::IndexedData PrapareNameTableData(std::vector<std::string> &data, bool fill_all)
 {
     NameTable::IndexedData indexed_data;
     std::vector<unsigned char> name_char_data;
@@ -60,19 +58,7 @@ std::string PrapareNameTableData(std::vector<std::string> &data, bool fill_all)
     }
     name_offsets.push_back(name_char_data.size());
 
-    TemporaryFile file;
+    return NameTable::IndexedData(name_offsets.begin(), name_offsets.end(), name_char_data.begin());
-    {
-        storage::io::FileWriter writer(file.path, storage::io::FileWriter::HasNoFingerprint);
-        indexed_data.write(
-            writer, name_offsets.begin(), name_offsets.end(), name_char_data.begin());
-    }
-
-    storage::io::FileReader reader(file.path, storage::io::FileReader::HasNoFingerprint);
-    auto length = reader.GetSize();
-    std::string str(length, '\0');
-    reader.ReadInto(const_cast<char *>(str.data()), length);
-
-    return str;
 }
 
 BOOST_AUTO_TEST_CASE(check_name_table_fill)
@@ -83,9 +69,7 @@ BOOST_AUTO_TEST_CASE(check_name_table_fill)
         "X",    "Y", "Z",          "",    "",     "",  "",   "",    "",     "",  "0",  ""};
 
     auto data = PrapareNameTableData(expected_names, true);
-
+    NameTable name_table {data};
-    NameTable name_table;
-    name_table.reset(&data[0], &data[data.size()]);
 
     for (std::size_t index = 0; index < expected_names.size(); ++index)
     {
@@ -106,9 +90,7 @@ BOOST_AUTO_TEST_CASE(check_name_table_nofill)
         "X",    "Y", "Z",          "",    "",     "",  "",   "",    "",     "",  "0",  ""};
 
     auto data = PrapareNameTableData(expected_names, false);
-
+    NameTable name_table {data};
-    NameTable name_table;
-    name_table.reset(&data[0], &data[data.size()]);
 
     // CALLGRIND_START_INSTRUMENTATION;
     for (std::size_t index = 0; index < expected_names.size(); ++index)

unit_tests/util/indexed_data.cpp

@@ -35,29 +35,7 @@ BOOST_AUTO_TEST_CASE(check_variable_group_block_bitops)
 template <typename IndexedData, typename Offsets, typename Data>
 void test_rw(const Offsets &offsets, const Data &data)
 {
-    TemporaryFile file;
+    IndexedData indexed_data(offsets.begin(), offsets.end(), data.begin());
-
-    IndexedData indexed_data;
-
-    {
-        storage::io::FileWriter writer(file.path, storage::io::FileWriter::HasNoFingerprint);
-        indexed_data.write(writer, offsets.begin(), offsets.end(), data.begin());
-    }
-
-    storage::io::FileReader reader(file.path, storage::io::FileReader::HasNoFingerprint);
-    auto length = reader.GetSize();
-    std::string str(length, '\0');
-    reader.ReadInto(const_cast<char *>(str.data()), length);
-
-#if 0
-    std::cout << "\n" << typeid(IndexedData).name() << "\nsaved size = " << str.size() << "\n";
-    for (auto c : str)
-        std::cout << std::hex << std::setw(2) << std::setfill('0')
-                  << (int)((unsigned char)c) << " ";
-    std::cout << std::dec << "\n";
-#endif
-
-    indexed_data.reset(str.c_str(), str.c_str() + str.size());
 
     for (std::size_t index = 0; index < offsets.size() - 1; ++index)
     {
@@ -151,56 +129,16 @@ BOOST_AUTO_TEST_CASE(check_max_size)
     name_offsets = {0, 256};
     BOOST_CHECK_THROW(test_fixed(), osrm::util::exception);
     name_offsets = {0, 255};
-    test_fixed();
+    BOOST_CHECK_NO_THROW(test_fixed());
-}
-
-BOOST_AUTO_TEST_CASE(check_corrupted_memory)
-{
-    std::vector<unsigned char> buf;
-
-    auto test_variable = [&buf]() {
-        IndexedData<VariableGroupBlock<16, std::vector<unsigned char>>> indexed_data;
-        indexed_data.reset(&buf[0], &buf[buf.size()]);
-        const auto result = indexed_data.at(0);
-        return std::string(reinterpret_cast<const char *>(&result[0]), result.size());
-    };
-
-    // Use LE internal representation
-    buf = {0, 42};
-    BOOST_CHECK_THROW(test_variable(), osrm::util::exception);
-
-    buf = {1, 0, 0, 0, 0};
-    BOOST_CHECK_THROW(test_variable(), osrm::util::exception);
-
-    buf = {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42};
-    BOOST_CHECK_THROW(test_variable(), osrm::util::exception);
-
-    buf = {1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 42};
-    BOOST_CHECK_THROW(test_variable(), osrm::util::exception);
-
-    buf = {1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0xF0, 0x9F, 0x90, 0xBC};
-    BOOST_CHECK_EQUAL(test_variable(), "🐼");
 }
 
 BOOST_AUTO_TEST_CASE(check_string_view)
 {
-    TemporaryFile file;
-
     std::string name_data = "hellostringview";
     std::vector<std::uint32_t> name_offsets = {0, 5, 11, 15};
 
-    IndexedData<VariableGroupBlock<16, StringView>> indexed_data;
+    IndexedData<VariableGroupBlock<16, StringView>> indexed_data(
-    {
+        name_offsets.begin(), name_offsets.end(), name_data.begin());
-        storage::io::FileWriter writer(file.path, storage::io::FileWriter::HasNoFingerprint);
-        indexed_data.write(writer, name_offsets.begin(), name_offsets.end(), name_data.begin());
-    }
-
-    storage::io::FileReader reader(file.path, storage::io::FileReader::HasNoFingerprint);
-    auto length = reader.GetSize();
-    std::string str(length, '\0');
-    reader.ReadInto(const_cast<char *>(str.data()), length);
-
-    indexed_data.reset(str.c_str(), str.c_str() + str.size());
 
     BOOST_CHECK_EQUAL(indexed_data.at(0), "hello");
     BOOST_CHECK_EQUAL(indexed_data.at(1), "string");

unit_tests/util/serialization.cpp

@@ -71,6 +71,46 @@ BOOST_AUTO_TEST_CASE(tar_serialize_packed_vector)
             TestPackedVector result;
             storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
             util::serialization::read(reader, "my_packed_vector", result);
+
+            CHECK_EQUAL_COLLECTIONS(result, v);
+        }
+    }
+}
+
+BOOST_AUTO_TEST_CASE(tar_serialize_variable_indexed_data)
+{
+    TemporaryFile tmp;
+    {
+        using TestIndexedData = IndexedData<VariableGroupBlock<16, std::string>>;
+
+        std::vector<std::vector<unsigned>> offset_data = {
+            {5, 8, 8},
+            {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17},
+            {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
+        };
+        std::vector<std::string> char_data = {
+            "HalloFoo",
+            "ABCDEFGHIJKLMNOPQR",
+            "ABCDEFGHIJKLMNOP",
+        };
+
+        for (const auto i : util::irange<std::size_t>(0, offset_data.size()))
+        {
+            TestIndexedData indexed {offset_data[i].begin(), offset_data[i].end(), char_data[i].begin()};
+            {
+                storage::tar::FileWriter writer(tmp.path,
+                                                storage::tar::FileWriter::GenerateFingerprint);
+                util::serialization::write(writer, "my_indexed_data", indexed);
+            }
+
+            TestIndexedData result;
+            storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
+            util::serialization::read(reader, "my_indexed_data", result);
+
+            for(auto j : util::irange<std::size_t>(0, offset_data[i].size() - 1))
+            {
+                BOOST_CHECK_EQUAL(indexed.at(j), result.at(j));
+            }
         }
     }
 }