refactor loading code of .datasource_indexes and .datasource_names files and move to io.hpp

include/engine/datafacade/internal_datafacade.hpp

@@ -292,14 +292,12 @@ class InternalDataFacade final : public BaseDataFacade
         }
         BOOST_ASSERT(datasources_stream);
 
-        std::uint64_t number_of_datasources = 0;
+        auto number_of_datasources = storage::io::readDatasourceIndexesSize(datasources_stream);
-        datasources_stream.read(reinterpret_cast<char *>(&number_of_datasources),
-                                sizeof(number_of_datasources));
         if (number_of_datasources > 0)
         {
             m_datasource_list.resize(number_of_datasources);
-            datasources_stream.read(reinterpret_cast<char *>(&(m_datasource_list[0])),
+            storage::io::readDatasourceIndexes(
-                                    number_of_datasources * sizeof(uint8_t));
+                datasources_stream, &m_datasource_list.front(), number_of_datasources);
         }
 
         boost::filesystem::ifstream datasourcenames_stream(datasource_names_file, std::ios::binary);
@@ -309,10 +307,16 @@ class InternalDataFacade final : public BaseDataFacade
                                   " for reading!");
         }
         BOOST_ASSERT(datasourcenames_stream);
-        std::string name;
+
-        while (std::getline(datasourcenames_stream, name))
+        auto datasource_names_data = storage::io::readDatasourceNamesData(datasourcenames_stream);
+        m_datasource_names.resize(datasource_names_data.lengths.size());
+        for (std::uint32_t i = 0; i < datasource_names_data.lengths.size(); ++i)
         {
-            m_datasource_names.push_back(std::move(name));
+            auto name_begin =
+                datasource_names_data.names.begin() + datasource_names_data.offsets[i];
+            auto name_end = datasource_names_data.names.begin() + datasource_names_data.offsets[i] +
+                            datasource_names_data.lengths[i];
+            m_datasource_names[i] = std::move(std::string(name_begin, name_end));
         }
     }
 

include/storage/io.hpp

@@ -98,6 +98,7 @@ inline std::uint32_t readEdgesSize(boost::filesystem::ifstream &edges_input_stre
 
 // Reads edge data from .edge files which includes its
 // geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID
+// Needs to be called after readEdgesSize() to get the correct offset in the stream
 template <typename GeometryIDT,
           typename NameIDT,
           typename TurnInstructionT,
@@ -141,6 +142,7 @@ inline std::uint32_t readNodesSize(boost::filesystem::ifstream &nodes_input_stre
 }
 
 // Reads coordinates and OSM node IDs from .nodes files
+// Needs to be called after readNodesSize() to get the correct offset in the stream
 template <typename CoordinateT, typename OSMNodeIDVectorT>
 void readNodesData(boost::filesystem::ifstream &nodes_input_stream,
                    CoordinateT coordinate_list[],
@@ -156,6 +158,55 @@ void readNodesData(boost::filesystem::ifstream &nodes_input_stream,
         BOOST_ASSERT(coordinate_list[i].IsValid());
     }
 }
+
+// Returns the number of indexes in a .datasource_indexes file
+// TODO: The original code used uint_64t to store the number of indexes. Can someone confirm that
+// this makes sense?
+inline std::uint64_t
+readDatasourceIndexesSize(boost::filesystem::ifstream &datasource_indexes_input_stream)
+{
+    std::uint64_t number_of_datasource_indexes;
+    datasource_indexes_input_stream.read(reinterpret_cast<char *>(&number_of_datasource_indexes),
+                                         sizeof(std::uint64_t));
+    return number_of_datasource_indexes;
+}
+
+// Reads datasource_indexes
+// Needs to be called after readDatasourceIndexesSize() to get the correct offset in the stream
+inline void readDatasourceIndexes(boost::filesystem::ifstream &datasource_indexes_input_stream,
+                                  uint8_t datasource_buffer[],
+                                  std::uint32_t number_of_datasource_indexes)
+{
+    datasource_indexes_input_stream.read(reinterpret_cast<char *>(datasource_buffer),
+                                         number_of_datasource_indexes * sizeof(std::uint8_t));
+}
+
+// Reads datasource names out of .datasource_names files and metadata such as
+// the length and offset of each name
+struct DatasourceNamesData
+{
+    std::vector<char> names;
+    std::vector<std::uint32_t> offsets;
+    std::vector<std::uint32_t> lengths;
+};
+inline DatasourceNamesData
+readDatasourceNamesData(boost::filesystem::ifstream &datasource_names_input_stream)
+{
+    DatasourceNamesData datasource_names_data;
+    if (datasource_names_input_stream)
+    {
+        std::string name;
+        while (std::getline(datasource_names_input_stream, name))
+        {
+            datasource_names_data.offsets.push_back(datasource_names_data.names.size());
+            datasource_names_data.lengths.push_back(name.size());
+            std::copy(name.c_str(),
+                      name.c_str() + name.size(),
+                      std::back_inserter(datasource_names_data.names));
+        }
+    }
+    return datasource_names_data;
+}
 }
 }
 }

src/storage/storage.cpp

@@ -1,3 +1,4 @@
+#include "storage/storage.hpp"
 #include "contractor/query_edge.hpp"
 #include "extractor/compressed_edge_container.hpp"
 #include "extractor/guidance/turn_instruction.hpp"
@@ -9,7 +10,6 @@
 #include "storage/shared_barriers.hpp"
 #include "storage/shared_datatype.hpp"
 #include "storage/shared_memory.hpp"
-#include "storage/storage.hpp"
 #include "engine/datafacade/datafacade_base.hpp"
 #include "util/coordinate.hpp"
 #include "util/exception.hpp"
@@ -346,15 +346,15 @@ Storage::ReturnCode Storage::Run(int max_wait)
     std::uint64_t number_of_compressed_datasources = 0;
     if (geometry_datasource_input_stream)
     {
-        geometry_datasource_input_stream.read(
+        number_of_compressed_datasources =
-            reinterpret_cast<char *>(&number_of_compressed_datasources),
+            io::readDatasourceIndexesSize(geometry_datasource_input_stream);
-            sizeof(number_of_compressed_datasources));
     }
     shared_layout_ptr->SetBlockSize<uint8_t>(SharedDataLayout::DATASOURCES_LIST,
                                              number_of_compressed_datasources);
 
     // Load datasource name sizes.  This file is optional, and it's non-fatal if it doesn't
     // exist
+
     boost::filesystem::ifstream datasource_names_input_stream(config.datasource_names_path,
                                                               std::ios::binary);
     if (!datasource_names_input_stream)
@@ -362,30 +362,22 @@ Storage::ReturnCode Storage::Run(int max_wait)
         throw util::exception("Could not open " + config.datasource_names_path.string() +
                               " for reading.");
     }
-    std::vector<char> m_datasource_name_data;
+    io::DatasourceNamesData datasource_names_data;
-    std::vector<std::size_t> m_datasource_name_offsets;
-    std::vector<std::size_t> m_datasource_name_lengths;
     if (datasource_names_input_stream)
     {
-        std::string name;
+        datasource_names_data = io::readDatasourceNamesData(datasource_names_input_stream);
-        while (std::getline(datasource_names_input_stream, name))
-        {
-            m_datasource_name_offsets.push_back(m_datasource_name_data.size());
-            std::copy(name.c_str(),
-                      name.c_str() + name.size(),
-                      std::back_inserter(m_datasource_name_data));
-            m_datasource_name_lengths.push_back(name.size());
-        }
     }
+
     shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::DATASOURCE_NAME_DATA,
-                                          m_datasource_name_data.size());
+                                          datasource_names_data.names.size());
-    shared_layout_ptr->SetBlockSize<std::size_t>(SharedDataLayout::DATASOURCE_NAME_OFFSETS,
+    shared_layout_ptr->SetBlockSize<std::uint32_t>(SharedDataLayout::DATASOURCE_NAME_OFFSETS,
-                                                 m_datasource_name_offsets.size());
+                                                   datasource_names_data.offsets.size());
-    shared_layout_ptr->SetBlockSize<std::size_t>(SharedDataLayout::DATASOURCE_NAME_LENGTHS,
+    shared_layout_ptr->SetBlockSize<std::uint32_t>(SharedDataLayout::DATASOURCE_NAME_LENGTHS,
-                                                 m_datasource_name_lengths.size());
+                                                   datasource_names_data.lengths.size());
 
     boost::filesystem::ifstream intersection_stream(config.intersection_class_path,
                                                     std::ios::binary);
+
     if (!static_cast<bool>(intersection_stream))
         throw util::exception("Could not open " + config.intersection_class_path.string() +
                               " for reading.");
@@ -643,35 +635,37 @@ Storage::ReturnCode Storage::Run(int max_wait)
         shared_memory_ptr, SharedDataLayout::DATASOURCES_LIST);
     if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCES_LIST) > 0)
     {
-        geometry_datasource_input_stream.read(
+        io::readDatasourceIndexes(geometry_datasource_input_stream,
-            reinterpret_cast<char *>(datasources_list_ptr),
+                                  datasources_list_ptr,
-            shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCES_LIST));
+                                  number_of_compressed_datasources);
     }
 
     // load datasource name information (if it exists)
+
     char *datasource_name_data_ptr = shared_layout_ptr->GetBlockPtr<char, true>(
         shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_DATA);
     if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_DATA) > 0)
     {
-        std::copy(
+        std::copy(datasource_names_data.names.begin(),
-            m_datasource_name_data.begin(), m_datasource_name_data.end(), datasource_name_data_ptr);
+                  datasource_names_data.names.end(),
+                  datasource_name_data_ptr);
     }
 
-    auto datasource_name_offsets_ptr = shared_layout_ptr->GetBlockPtr<std::size_t, true>(
+    auto datasource_name_offsets_ptr = shared_layout_ptr->GetBlockPtr<std::uint32_t, true>(
         shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_OFFSETS);
     if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_OFFSETS) > 0)
     {
-        std::copy(m_datasource_name_offsets.begin(),
+        std::copy(datasource_names_data.offsets.begin(),
-                  m_datasource_name_offsets.end(),
+                  datasource_names_data.offsets.end(),
                   datasource_name_offsets_ptr);
     }
 
-    auto datasource_name_lengths_ptr = shared_layout_ptr->GetBlockPtr<std::size_t, true>(
+    auto datasource_name_lengths_ptr = shared_layout_ptr->GetBlockPtr<std::uint32_t, true>(
         shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_LENGTHS);
     if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_LENGTHS) > 0)
     {
-        std::copy(m_datasource_name_lengths.begin(),
+        std::copy(datasource_names_data.lengths.begin(),
-                  m_datasource_name_lengths.end(),
+                  datasource_names_data.lengths.end(),
                   datasource_name_lengths_ptr);
     }
 
@@ -803,7 +797,6 @@ Storage::ReturnCode Storage::Run(int max_wait)
         static_cast<SharedDataTimestamp *>(data_type_memory->Ptr());
 
     {
-
         boost::interprocess::scoped_lock<boost::interprocess::named_upgradable_mutex>
             current_regions_exclusive_lock;