Include datasources in .geometries file and refactor .datasource_names

CHANGELOG.md

@@ -2,6 +2,8 @@
   - Changes from 5.6
     - Internals
       - Shared memory notification via conditional variables on Linux or semaphore queue on OS X and Windows with a limit of 128 OSRM Engine instances
+    - Files
+      - .osrm.datasource_index file was removed. Data is now part of .osrm.geometries.
 
 # 5.6.3
   - Changes from 5.6.0

features/options/contract/datasources.feature

@@ -1,38 +0,0 @@
-@prepare @options @files
-Feature: osrm-contract command line options: datasources
-# expansions:
-# {processed_file} => path to .osrm file
-
-    Background:
-        Given the profile "testbot"
-        Given the extract extra arguments "--generate-edge-lookup"
-        And the node map
-            """
-            a b
-            """
-        And the ways
-            | nodes |
-            | ab    |
-        And the speed file
-        """
-        1,2,50
-        2,1,50
-        2,3,50
-        3,2,50
-        1,4,50
-        4,1,50
-        """
-        And the data has been extracted
-
-    Scenario: osrm-contract - Passing base file
-        When I run "osrm-contract --segment-speed-file {speeds_file} {processed_file}"
-        Then datasource names should contain "lua profile,27_osrmcontract_passing_base_file_speeds"
-        And it should exit successfully
-
-    Scenario: osrm-contract - Passing base file
-        Given the speed file
-        """
-        """
-        And the data has been extracted
-        When I run "osrm-contract --segment-speed-file {speeds_file} {processed_file}"
-        Then it should exit successfully

features/step_definitions/options.js

@@ -103,11 +103,6 @@ module.exports = function () {
         assert.equal(this.stderr.split('\n').length - 1, parseInt(lines));
     });
 
-    this.Then(/^datasource names should contain "(.+)"$/, (expectedData) => {
-        const actualData = fs.readFileSync(this.processedCacheFile + '.datasource_names', {encoding:'UTF-8'}).trim().split('\n').join(',');
-        assert.equal(actualData, expectedData);
-    });
-
     this.Given(/^the query options$/, (table, callback) => {
         table.raw().forEach(tuple => {
             this.queryParams[tuple[0]] = tuple[1];

include/engine/datafacade/contiguous_internalmem_datafacade.hpp

@@ -8,6 +8,7 @@
 #include "engine/algorithm.hpp"
 #include "engine/geospatial_query.hpp"
 
+#include "extractor/datasources.hpp"
 #include "extractor/guidance/turn_instruction.hpp"
 #include "extractor/guidance/turn_lane_types.hpp"
 #include "extractor/profile_properties.hpp"
@@ -205,6 +206,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     std::string m_timestamp;
     extractor::ProfileProperties *m_profile_properties;
+    extractor::Datasources *m_datasources;
 
     unsigned m_check_sum;
     util::ShM<util::Coordinate, true>::vector m_coordinate_list;
@@ -219,7 +221,6 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
     util::NameTable m_names_table;
     util::ShM<unsigned, true>::vector m_name_begin_indices;
     util::ShM<bool, true>::vector m_is_core_node;
-    util::ShM<DatasourceID, true>::vector m_datasource_list;
     util::ShM<std::uint32_t, true>::vector m_lane_description_offsets;
     util::ShM<extractor::guidance::TurnLaneType::Mask, true>::vector m_lane_description_masks;
     util::ShM<TurnPenalty, true>::vector m_turn_weight_penalties;
@@ -467,39 +468,21 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
             geometries_rev_duration_list_ptr,
             data_layout.num_entries[storage::DataLayout::GEOMETRIES_REV_DURATION_LIST]);
 
+        auto datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
+            memory_block, storage::DataLayout::DATASOURCES_LIST);
+        util::ShM<DatasourceID, true>::vector datasources_list(
+            datasources_list_ptr, data_layout.num_entries[storage::DataLayout::DATASOURCES_LIST]);
+
         segment_data = extractor::SegmentDataView{std::move(geometry_begin_indices),
                                                   std::move(geometry_node_list),
                                                   std::move(geometry_fwd_weight_list),
                                                   std::move(geometry_rev_weight_list),
                                                   std::move(geometry_fwd_duration_list),
-                                                  std::move(geometry_rev_duration_list)};
+                                                  std::move(geometry_rev_duration_list),
+                                                  std::move(datasources_list)};
 
-        auto datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
+        m_datasources = data_layout.GetBlockPtr<extractor::Datasources>(
-            memory_block, storage::DataLayout::DATASOURCES_LIST);
+            memory_block, storage::DataLayout::DATASOURCES_NAMES);
-        util::ShM<DatasourceID, true>::vector datasources_list(
-            datasources_list_ptr, data_layout.num_entries[storage::DataLayout::DATASOURCES_LIST]);
-        m_datasource_list = std::move(datasources_list);
-
-        auto datasource_name_data_ptr =
-            data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::DATASOURCE_NAME_DATA);
-        util::ShM<char, true>::vector datasource_name_data(
-            datasource_name_data_ptr,
-            data_layout.num_entries[storage::DataLayout::DATASOURCE_NAME_DATA]);
-        m_datasource_name_data = std::move(datasource_name_data);
-
-        auto datasource_name_offsets_ptr = data_layout.GetBlockPtr<std::size_t>(
-            memory_block, storage::DataLayout::DATASOURCE_NAME_OFFSETS);
-        util::ShM<std::size_t, true>::vector datasource_name_offsets(
-            datasource_name_offsets_ptr,
-            data_layout.num_entries[storage::DataLayout::DATASOURCE_NAME_OFFSETS]);
-        m_datasource_name_offsets = std::move(datasource_name_offsets);
-
-        auto datasource_name_lengths_ptr = data_layout.GetBlockPtr<std::size_t>(
-            memory_block, storage::DataLayout::DATASOURCE_NAME_LENGTHS);
-        util::ShM<std::size_t, true>::vector datasource_name_lengths(
-            datasource_name_lengths_ptr,
-            data_layout.num_entries[storage::DataLayout::DATASOURCE_NAME_LENGTHS]);
-        m_datasource_name_lengths = std::move(datasource_name_lengths);
     }
 
     void InitializeIntersectionClassPointers(storage::DataLayout &data_layout, char *memory_block)
@@ -611,6 +594,24 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
         return std::vector<EdgeWeight>{range.begin(), range.end()};
     }
 
+    // Returns the data source ids that were used to supply the edge
+    // weights.
+    virtual std::vector<DatasourceID>
+    GetUncompressedForwardDatasources(const EdgeID id) const override final
+    {
+        auto range = segment_data.GetForwardDatasources(id);
+        return std::vector<DatasourceID>{range.begin(), range.end()};
+    }
+
+    // Returns the data source ids that were used to supply the edge
+    // weights.
+    virtual std::vector<DatasourceID>
+    GetUncompressedReverseDatasources(const EdgeID id) const override final
+    {
+        auto range = segment_data.GetReverseDatasources(id);
+        return std::vector<DatasourceID>{range.begin(), range.end()};
+    }
+
     virtual GeometryID GetGeometryIndexForEdgeID(const EdgeID id) const override final
     {
         return m_via_geometry_list.at(id);
@@ -781,89 +782,9 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
         return m_name_table.GetDestinationsForID(id);
     }
 
-    // Returns the data source ids that were used to supply the edge
-    // weights.
-    virtual std::vector<DatasourceID>
-    GetUncompressedForwardDatasources(const EdgeID id) const override final
-    {
-        /*
-         * Data sources for geometries are stored in one place for
-         * both forward and reverse segments along the same bi-
-         * directional edge. The indices stores
-         * refences to where to find the beginning of the bi-
-         * directional edge in the list vector. For
-         * forward datasources of bi-directional edges, edges 2 to
-         * n of that edge need to be read.
-         */
-        const auto begin = segment_data.GetOffset(id, 0) + 1;
-        const auto end = segment_data.GetOffset(id + 1, 0);
-
-        std::vector<DatasourceID> result_datasources;
-        result_datasources.reserve(end - begin);
-
-        // If there was no datasource info, return an array of 0's.
-        if (m_datasource_list.empty())
-        {
-            result_datasources.resize(end - begin, 0);
-        }
-        else
-        {
-            std::copy(m_datasource_list.begin() + begin,
-                      m_datasource_list.begin() + end,
-                      std::back_inserter(result_datasources));
-        }
-
-        return result_datasources;
-    }
-
-    // Returns the data source ids that were used to supply the edge
-    // weights.
-    virtual std::vector<DatasourceID>
-    GetUncompressedReverseDatasources(const EdgeID id) const override final
-    {
-        /*
-         * Datasources for geometries are stored in one place for
-         * both forward and reverse segments along the same bi-
-         * directional edge. The indices stores
-         * refences to where to find the beginning of the bi-
-         * directional edge in the list vector. For
-         * reverse datasources of bi-directional edges, edges 1 to
-         * n-1 of that edge need to be read in reverse.
-         */
-        const auto begin = segment_data.GetOffset(id, 0);
-        const auto end = segment_data.GetOffset(id + 1, 0) - 1;
-
-        std::vector<DatasourceID> result_datasources;
-        result_datasources.reserve(end - begin);
-
-        // If there was no datasource info, return an array of 0's.
-        if (m_datasource_list.empty())
-        {
-            result_datasources.resize(end - begin, 0);
-        }
-        else
-        {
-            std::reverse_copy(m_datasource_list.begin() + begin,
-                              m_datasource_list.begin() + end,
-                              std::back_inserter(result_datasources));
-        }
-
-        return result_datasources;
-    }
-
     StringView GetDatasourceName(const DatasourceID id) const override final
     {
-        BOOST_ASSERT(m_datasource_name_offsets.size() >= 1);
+        return m_datasources->GetSourceName(id);
-        BOOST_ASSERT(m_datasource_name_offsets.size() > id);
-
-        auto first = m_datasource_name_data.begin() + m_datasource_name_offsets[id];
-        auto last = m_datasource_name_data.begin() + m_datasource_name_offsets[id] +
-                    m_datasource_name_lengths[id];
-        // These iterators are useless: they're InputIterators onto a contiguous block of memory.
-        // Deref to get to the first element, then Addressof to get the memory address of the it.
-        const std::size_t len = &*last - &*first;
-
-        return StringView{&*first, len};
     }
 
     std::string GetTimestamp() const override final { return m_timestamp; }

include/extractor/datasources.hpp

@@ -0,0 +1,48 @@
+#ifndef OSRM_EXTRACT_DATASOURCES_HPP
+#define OSRM_EXTRACT_DATASOURCES_HPP
+
+#include <cstdint>
+#include <util/string_view.hpp>
+#include <util/typedefs.hpp>
+
+namespace osrm
+{
+namespace extractor
+{
+
+class Datasources
+{
+    static constexpr const std::uint8_t MAX_NUM_SOURES = 255;
+    static constexpr const std::uint8_t MAX_LENGTH_NAME = 255;
+
+  public:
+    Datasources()
+    {
+        std::fill(lengths.begin(), lengths.end(), 0);
+        std::fill(sources.begin(), sources.end(), '\0');
+    }
+
+    util::StringView GetSourceName(DatasourceID id) const
+    {
+        auto begin = sources.data() + (MAX_LENGTH_NAME * id);
+
+        return util::StringView{begin, lengths[id]};
+    }
+
+    void SetSourceName(DatasourceID id, const std::string &name)
+    {
+        BOOST_ASSERT(name.size() < MAX_LENGTH_NAME);
+        lengths[id] = std::min<std::uint32_t>(name.size(), MAX_LENGTH_NAME);
+
+        auto out = sources.data() + (MAX_LENGTH_NAME * id);
+        std::copy(name.begin(), name.begin() + lengths[id], out);
+    }
+
+  private:
+    std::array<std::uint32_t, MAX_NUM_SOURES> lengths;
+    std::array<char, MAX_LENGTH_NAME * MAX_NUM_SOURES> sources;
+};
+}
+}
+
+#endif

include/extractor/io.hpp

@@ -1,10 +1,13 @@
 #ifndef OSRM_EXTRACTOR_IO_HPP
 #define OSRM_EXTRACTOR_IO_HPP
 
+#include "extractor/datasources.hpp"
 #include "extractor/segment_data_container.hpp"
 
 #include "storage/io.hpp"
 
+#include <boost/assert.hpp>
+
 namespace osrm
 {
 namespace extractor
@@ -12,6 +15,22 @@ namespace extractor
 namespace io
 {
 
+void read(const boost::filesystem::path &path, Datasources &sources)
+{
+    const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
+    storage::io::FileReader reader{path, fingerprint};
+
+    reader.ReadInto(sources);
+}
+
+void write(const boost::filesystem::path &path, Datasources &sources)
+{
+    const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
+    storage::io::FileWriter writer{path, fingerprint};
+
+    writer.WriteFrom(sources);
+}
+
 template <> void read(const boost::filesystem::path &path, SegmentDataContainer &segment_data)
 {
     const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
@@ -27,12 +46,14 @@ template <> void read(const boost::filesystem::path &path, SegmentDataContainer
     segment_data.rev_weights.resize(num_entries);
     segment_data.fwd_durations.resize(num_entries);
     segment_data.rev_durations.resize(num_entries);
+    segment_data.datasources.resize(num_entries);
 
     reader.ReadInto(segment_data.nodes);
     reader.ReadInto(segment_data.fwd_weights);
     reader.ReadInto(segment_data.rev_weights);
     reader.ReadInto(segment_data.fwd_durations);
     reader.ReadInto(segment_data.rev_durations);
+    reader.ReadInto(segment_data.datasources);
 }
 
 template <>
@@ -45,11 +66,17 @@ void write(const boost::filesystem::path &path, const SegmentDataContainer &segm
     writer.WriteFrom(segment_data.index);
 
     writer.WriteElementCount32(segment_data.nodes.size());
+    BOOST_ASSERT(segment_data.fwd_weights.size() == segment_data.nodes.size());
+    BOOST_ASSERT(segment_data.rev_weights.size() == segment_data.nodes.size());
+    BOOST_ASSERT(segment_data.fwd_durations.size() == segment_data.nodes.size());
+    BOOST_ASSERT(segment_data.rev_durations.size() == segment_data.nodes.size());
+    BOOST_ASSERT(segment_data.datasources.size() == segment_data.nodes.size());
     writer.WriteFrom(segment_data.nodes);
     writer.WriteFrom(segment_data.fwd_weights);
     writer.WriteFrom(segment_data.rev_weights);
     writer.WriteFrom(segment_data.fwd_durations);
     writer.WriteFrom(segment_data.rev_durations);
+    writer.WriteFrom(segment_data.datasources);
 }
 }
 }

include/extractor/segment_data_container.hpp

@@ -55,10 +55,11 @@ template <bool UseShareMemory> class SegmentDataContainerImpl
                              Vector<EdgeWeight> fwd_weights_,
                              Vector<EdgeWeight> rev_weights_,
                              Vector<EdgeWeight> fwd_durations_,
-                             Vector<EdgeWeight> rev_durations_)
+                             Vector<EdgeWeight> rev_durations_,
+                             Vector<DatasourceID> datasources_)
         : index(std::move(index_)), nodes(std::move(nodes_)), fwd_weights(std::move(fwd_weights_)),
           rev_weights(std::move(rev_weights_)), fwd_durations(std::move(fwd_durations_)),
-          rev_durations(std::move(rev_durations_))
+          rev_durations(std::move(rev_durations_)), datasources(std::move(datasources_))
     {
     }
 
@@ -79,11 +80,13 @@ template <bool UseShareMemory> class SegmentDataContainerImpl
     {
         return rev_weights[index[id] + offset];
     }
-    // TODO we only need this for the datasource file since it breaks this
+    auto &ForwardDatasource(const DirectionalGeometryID id, const SegmentOffset offset)
-    // abstraction, but uses this index
-    auto GetOffset(const DirectionalGeometryID id, const SegmentOffset offset) const
     {
-        return index[id] + offset;
+        return datasources[index[id] + 1 + offset];
+    }
+    auto &ReverseDatasource(const DirectionalGeometryID id, const SegmentOffset offset)
+    {
+        return datasources[index[id] + offset];
     }
 
     auto GetForwardGeometry(const DirectionalGeometryID id) const
@@ -131,6 +134,22 @@ template <bool UseShareMemory> class SegmentDataContainerImpl
         return boost::adaptors::reverse(boost::make_iterator_range(begin, end));
     }
 
+    auto GetForwardDatasources(const DirectionalGeometryID id) const
+    {
+        const auto begin = datasources.cbegin() + index.at(id) + 1;
+        const auto end = datasources.cbegin() + index.at(id + 1);
+
+        return boost::make_iterator_range(begin, end);
+    }
+
+    auto GetReverseDatasources(const DirectionalGeometryID id) const
+    {
+        const auto begin = datasources.cbegin() + index.at(id);
+        const auto end = datasources.cbegin() + index.at(id + 1) - 1;
+
+        return boost::adaptors::reverse(boost::make_iterator_range(begin, end));
+    }
+
     auto GetNumberOfSegments() const { return fwd_weights.size(); }
 
     friend void
@@ -147,6 +166,7 @@ template <bool UseShareMemory> class SegmentDataContainerImpl
     Vector<EdgeWeight> rev_weights;
     Vector<EdgeWeight> fwd_durations;
     Vector<EdgeWeight> rev_durations;
+    Vector<DatasourceID> datasources;
 };
 }
 

include/storage/serialization.hpp

@@ -70,16 +70,6 @@ inline void readHSGR(io::FileReader &input_file,
     input_file.ReadInto(edge_buffer, number_of_edges);
 }
 
-// Loads datasource_indexes from .datasource_indexes into memory
-// Needs to be called after readElementCount() to get the correct offset in the stream
-inline void readDatasourceIndexes(io::FileReader &datasource_indexes_file,
-                                  uint8_t *datasource_buffer,
-                                  const std::uint64_t number_of_datasource_indexes)
-{
-    BOOST_ASSERT(datasource_buffer);
-    datasource_indexes_file.ReadInto(datasource_buffer, number_of_datasource_indexes);
-}
-
 // Loads edge data from .edge files into memory which includes its
 // geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID
 // Needs to be called after readElementCount() to get the correct offset in the stream
@@ -134,29 +124,6 @@ void readNodes(io::FileReader &nodes_file,
         BOOST_ASSERT(coordinate_list[i].IsValid());
     }
 }
-
-// Reads datasource names out of .datasource_names files and metadata such as
-// the length and offset of each name
-struct DatasourceNamesData
-{
-    std::vector<char> names;
-    std::vector<std::size_t> offsets;
-    std::vector<std::size_t> lengths;
-};
-inline DatasourceNamesData readDatasourceNames(io::FileReader &datasource_names_file)
-{
-    DatasourceNamesData datasource_names_data;
-    std::vector<std::string> lines = datasource_names_file.ReadLines();
-    for (const auto &name : lines)
-    {
-        datasource_names_data.offsets.push_back(datasource_names_data.names.size());
-        datasource_names_data.lengths.push_back(name.size());
-        std::copy(name.c_str(),
-                  name.c_str() + name.size(),
-                  std::back_inserter(datasource_names_data.names));
-    }
-    return datasource_names_data;
-}
 }
 }
 }

include/storage/shared_datatype.hpp

@@ -40,9 +40,7 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
                                             "FILE_INDEX_PATH",
                                             "CH_CORE_MARKER",
                                             "DATASOURCES_LIST",
-                                            "DATASOURCE_NAME_DATA",
+                                            "DATASOURCES_NAMES",
-                                            "DATASOURCE_NAME_OFFSETS",
-                                            "DATASOURCE_NAME_LENGTHS",
                                             "PROPERTIES",
                                             "BEARING_CLASSID",
                                             "BEARING_OFFSETS",
@@ -94,9 +92,7 @@ struct DataLayout
         FILE_INDEX_PATH,
         CH_CORE_MARKER,
         DATASOURCES_LIST,
-        DATASOURCE_NAME_DATA,
+        DATASOURCES_NAMES,
-        DATASOURCE_NAME_OFFSETS,
-        DATASOURCE_NAME_LENGTHS,
         PROPERTIES,
         BEARING_CLASSID,
         BEARING_OFFSETS,

include/updater/updater_config.hpp

@@ -51,7 +51,6 @@ struct UpdaterConfig final
         geometry_path = osrm_input_path.string() + ".geometry";
         rtree_leaf_path = osrm_input_path.string() + ".fileIndex";
         datasource_names_path = osrm_input_path.string() + ".datasource_names";
-        datasource_indexes_path = osrm_input_path.string() + ".datasource_indexes";
         profile_properties_path = osrm_input_path.string() + ".properties";
     }
 
@@ -71,7 +70,6 @@ struct UpdaterConfig final
 
     std::vector<std::string> segment_speed_lookup_paths;
     std::vector<std::string> turn_penalty_lookup_paths;
-    std::string datasource_indexes_path;
     std::string datasource_names_path;
     std::string profile_properties_path;
 };

src/extractor/compressed_edge_container.cpp

@@ -221,6 +221,7 @@ void CompressedEdgeContainer::InitializeBothwayVector()
     segment_data->rev_weights.reserve(m_compressed_oneway_geometries.size());
     segment_data->fwd_durations.reserve(m_compressed_oneway_geometries.size());
     segment_data->rev_durations.reserve(m_compressed_oneway_geometries.size());
+    segment_data->datasources.reserve(m_compressed_oneway_geometries.size());
 }
 
 unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID r_edge_id)
@@ -238,11 +239,14 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
 
     const auto &first_node = reverse_bucket.back();
 
+    constexpr DatasourceID LUA_SOURCE = 0;
+
     segment_data->nodes.emplace_back(first_node.node_id);
     segment_data->fwd_weights.emplace_back(INVALID_EDGE_WEIGHT);
     segment_data->rev_weights.emplace_back(first_node.weight);
     segment_data->fwd_durations.emplace_back(INVALID_EDGE_WEIGHT);
     segment_data->rev_durations.emplace_back(first_node.duration);
+    segment_data->datasources.emplace_back(LUA_SOURCE);
 
     for (std::size_t i = 0; i < forward_bucket.size() - 1; ++i)
     {
@@ -256,6 +260,7 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
         segment_data->rev_weights.emplace_back(rev_node.weight);
         segment_data->fwd_durations.emplace_back(fwd_node.duration);
         segment_data->rev_durations.emplace_back(rev_node.duration);
+        segment_data->datasources.emplace_back(LUA_SOURCE);
     }
 
     const auto &last_node = forward_bucket.back();
@@ -265,6 +270,7 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
     segment_data->rev_weights.emplace_back(INVALID_EDGE_WEIGHT);
     segment_data->fwd_durations.emplace_back(last_node.duration);
     segment_data->rev_durations.emplace_back(INVALID_EDGE_WEIGHT);
+    segment_data->datasources.emplace_back(LUA_SOURCE);
 
     return zipped_geometry_id;
 }

src/storage/storage.cpp

@@ -3,6 +3,7 @@
 #include "extractor/compressed_edge_container.hpp"
 #include "extractor/edge_based_edge.hpp"
 #include "extractor/guidance/turn_instruction.hpp"
+#include "extractor/io.hpp"
 #include "extractor/original_edge_data.hpp"
 #include "extractor/profile_properties.hpp"
 #include "extractor/query_node.hpp"
@@ -339,40 +340,13 @@ void Storage::PopulateLayout(DataLayout &layout)
                                         number_of_compressed_geometries);
         layout.SetBlockSize<EdgeWeight>(DataLayout::GEOMETRIES_REV_DURATION_LIST,
                                         number_of_compressed_geometries);
+        layout.SetBlockSize<DatasourceID>(DataLayout::DATASOURCES_LIST,
+                                          number_of_compressed_geometries);
     }
 
-    // load datasource sizes. This file is optional, and it's non-fatal if it doesn't exist.
+    // Load datasource name sizes.
-    if (boost::filesystem::exists(config.datasource_indexes_path))
     {
-        io::FileReader reader(config.datasource_indexes_path, io::FileReader::HasNoFingerprint);
+        layout.SetBlockSize<extractor::Datasources>(DataLayout::DATASOURCES_NAMES, 1);
-        const auto number_of_datasources = reader.ReadElementCount64();
-        layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST, number_of_datasources);
-    }
-    else
-    {
-        layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST, 0);
-    }
-
-    // Load datasource name sizes.  This file is optional, and it's non-fatal if it doesn't exist
-    if (boost::filesystem::exists(config.datasource_names_path))
-    {
-        io::FileReader reader(config.datasource_names_path, io::FileReader::HasNoFingerprint);
-
-        const serialization::DatasourceNamesData datasource_names_data =
-            serialization::readDatasourceNames(reader);
-
-        layout.SetBlockSize<char>(DataLayout::DATASOURCE_NAME_DATA,
-                                  datasource_names_data.names.size());
-        layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_OFFSETS,
-                                         datasource_names_data.offsets.size());
-        layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_LENGTHS,
-                                         datasource_names_data.lengths.size());
-    }
-    else
-    {
-        layout.SetBlockSize<char>(DataLayout::DATASOURCE_NAME_DATA, 0);
-        layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_OFFSETS, 0);
-        layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_LENGTHS, 0);
     }
 
     {
@@ -689,81 +663,17 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
         BOOST_ASSERT(geometry_node_lists_count ==
                      layout.num_entries[DataLayout::GEOMETRIES_REV_DURATION_LIST]);
         geometry_input_file.ReadInto(geometries_rev_duration_list_ptr, geometry_node_lists_count);
+
+        const auto datasource_list_ptr =
+            layout.GetBlockPtr<DatasourceID, true>(memory_ptr, DataLayout::DATASOURCES_LIST);
+        BOOST_ASSERT(geometry_node_lists_count == layout.num_entries[DataLayout::DATASOURCES_LIST]);
+        geometry_input_file.ReadInto(datasource_list_ptr, geometry_node_lists_count);
     }
 
-    if (boost::filesystem::exists(config.datasource_indexes_path))
     {
-        io::FileReader geometry_datasource_file(config.datasource_indexes_path,
+        const auto datasources_names_ptr = layout.GetBlockPtr<extractor::Datasources, true>(
-                                                io::FileReader::HasNoFingerprint);
+            memory_ptr, DataLayout::DATASOURCES_NAMES);
-        const auto number_of_compressed_datasources = geometry_datasource_file.ReadElementCount64();
+        extractor::io::read(config.datasource_names_path, *datasources_names_ptr);
-
-        // load datasource information (if it exists)
-        const auto datasources_list_ptr =
-            layout.GetBlockPtr<uint8_t, true>(memory_ptr, DataLayout::DATASOURCES_LIST);
-        if (number_of_compressed_datasources > 0)
-        {
-            serialization::readDatasourceIndexes(
-                geometry_datasource_file, datasources_list_ptr, number_of_compressed_datasources);
-        }
-    }
-    else
-    {
-        layout.GetBlockPtr<uint8_t, true>(memory_ptr, DataLayout::DATASOURCES_LIST);
-    }
-
-    if (boost::filesystem::exists(config.datasource_names_path))
-    {
-        io::FileReader datasource_names_file(config.datasource_names_path,
-                                             io::FileReader::HasNoFingerprint);
-
-        const auto datasource_names_data =
-            serialization::readDatasourceNames(datasource_names_file);
-
-        // load datasource name information (if it exists)
-        const auto datasource_name_data_ptr =
-            layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::DATASOURCE_NAME_DATA);
-        if (layout.GetBlockSize(DataLayout::DATASOURCE_NAME_DATA) > 0)
-        {
-            BOOST_ASSERT(std::distance(datasource_names_data.names.begin(),
-                                       datasource_names_data.names.end()) *
-                             sizeof(decltype(datasource_names_data.names)::value_type) <=
-                         layout.GetBlockSize(DataLayout::DATASOURCE_NAME_DATA));
-            std::copy(datasource_names_data.names.begin(),
-                      datasource_names_data.names.end(),
-                      datasource_name_data_ptr);
-        }
-
-        const auto datasource_name_offsets_ptr =
-            layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_OFFSETS);
-        if (layout.GetBlockSize(DataLayout::DATASOURCE_NAME_OFFSETS) > 0)
-        {
-            BOOST_ASSERT(std::distance(datasource_names_data.offsets.begin(),
-                                       datasource_names_data.offsets.end()) *
-                             sizeof(decltype(datasource_names_data.offsets)::value_type) <=
-                         layout.GetBlockSize(DataLayout::DATASOURCE_NAME_OFFSETS));
-            std::copy(datasource_names_data.offsets.begin(),
-                      datasource_names_data.offsets.end(),
-                      datasource_name_offsets_ptr);
-        }
-
-        const auto datasource_name_lengths_ptr =
-            layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_LENGTHS);
-        if (layout.GetBlockSize(DataLayout::DATASOURCE_NAME_LENGTHS) > 0)
-        {
-            BOOST_ASSERT(std::distance(datasource_names_data.lengths.begin(),
-                                       datasource_names_data.lengths.end()) *
-                             sizeof(decltype(datasource_names_data.lengths)::value_type) <=
-                         layout.GetBlockSize(DataLayout::DATASOURCE_NAME_LENGTHS));
-            std::copy(datasource_names_data.lengths.begin(),
-                      datasource_names_data.lengths.end(),
-                      datasource_name_lengths_ptr);
-        }
-    }
-    else
-    {
-        layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::DATASOURCE_NAME_DATA);
-        layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_OFFSETS);
-        layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_LENGTHS);
     }
 
     // Loading list of coordinates

src/storage/storage_config.cpp

@@ -32,7 +32,6 @@ StorageConfig::StorageConfig(const boost::filesystem::path &base)
       turn_weight_penalties_path{base.string() + ".turn_weight_penalties"},
       turn_duration_penalties_path{base.string() + ".turn_duration_penalties"},
       datasource_names_path{base.string() + ".datasource_names"},
-      datasource_indexes_path{base.string() + ".datasource_indexes"},
       names_data_path{base.string() + ".names"}, properties_path{base.string() + ".properties"},
       intersection_class_path{base.string() + ".icd"}, turn_lane_data_path{base.string() + ".tld"},
       turn_lane_description_path{base.string() + ".tls"},
@@ -54,7 +53,8 @@ bool StorageConfig::IsValid() const
                         turn_duration_penalties_path,
                         names_data_path,
                         properties_path,
-                        intersection_class_path}))
+                        intersection_class_path,
+                        datasource_names_path}))
     {
         return false;
     }
@@ -62,7 +62,7 @@ bool StorageConfig::IsValid() const
     // TODO: add algorithm checks
 
     // CH files
-    CheckFileList({hsgr_data_path, core_data_path, datasource_names_path, datasource_indexes_path});
+    CheckFileList({hsgr_data_path, core_data_path});
 
     // MLD files
     CheckFileList({mld_partition_path, mld_storage_path, edge_based_graph_path});

src/updater/updater.cpp

@@ -262,10 +262,6 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
     auto segment_speed_lookup = csv::readSegmentValues(config.segment_speed_lookup_paths);
     auto turn_penalty_lookup = csv::readTurnValues(config.turn_penalty_lookup_paths);
 
-    // If we update the edge weights, this file will hold the datasource information for each
-    // segment; the other files will also be conditionally filled concurrently if we make an update
-    std::vector<uint8_t> geometry_datasource;
-
     std::vector<extractor::QueryNode> internal_to_external_node_map;
     extractor::SegmentDataContainer segment_data;
 
@@ -294,14 +290,6 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
         // Here, we have to update the compressed geometry weights
         // First, we need the external-to-internal node lookup table
 
-        // This is a list of the "data source id" for every segment in the compressed
-        // geometry container.  We assume that everything so far has come from the
-        // profile (data source 0).  Here, we replace the 0's with the index of the
-        // CSV file that supplied the value that gets used for that segment, then
-        // we write out this list so that it can be returned by the debugging
-        // vector tiles later on.
-        geometry_datasource.resize(segment_data.GetNumberOfSegments(), 0);
-
         // Now, we iterate over all the segments stored in the StaticRTree, updating
         // the packed geometry weights in the `.geometries` file (note: we do not
         // update the RTree itself, we just use the leaf nodes to iterate over all segments)
@@ -364,8 +352,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
                     segment_data.ForwardWeight(geometry_id, segment_offset) = new_segment_weight;
                     segment_data.ForwardDuration(geometry_id, segment_offset) =
                         new_segment_duration;
-                    geometry_datasource[segment_data.GetOffset(geometry_id, segment_offset) + 1] =
+                    segment_data.ForwardDatasource(geometry_id, segment_offset) = value->source;
-                        value->source;
                     fwd_source = value->source;
                 }
 
@@ -385,8 +372,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
                     segment_data.ReverseWeight(geometry_id, segment_offset) = new_segment_weight;
                     segment_data.ReverseDuration(geometry_id, segment_offset) =
                         new_segment_duration;
-                    geometry_datasource[segment_data.GetOffset(geometry_id, segment_offset)] =
+                    segment_data.ReverseDatasource(geometry_id, segment_offset) = value->source;
-                        value->source;
                     rev_source = value->source;
                 }
 
@@ -430,44 +416,23 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
         extractor::io::write(config.geometry_path, segment_data);
     };
 
-    const auto save_datasource_indexes = [&] {
-        std::ofstream datasource_stream(config.datasource_indexes_path, std::ios::binary);
-        if (!datasource_stream)
-        {
-            const std::string message{"Failed to open " + config.datasource_indexes_path +
-                                      " for writing"};
-            throw util::exception(message + SOURCE_REF);
-        }
-        std::uint64_t number_of_datasource_entries = geometry_datasource.size();
-        datasource_stream.write(reinterpret_cast<const char *>(&number_of_datasource_entries),
-                                sizeof(number_of_datasource_entries));
-        if (number_of_datasource_entries > 0)
-        {
-            datasource_stream.write(reinterpret_cast<char *>(&(geometry_datasource[0])),
-                                    number_of_datasource_entries * sizeof(uint8_t));
-        }
-    };
-
     const auto save_datastore_names = [&] {
-        std::ofstream datasource_stream(config.datasource_names_path, std::ios::binary);
+        extractor::Datasources sources;
-        if (!datasource_stream)
+        DatasourceID source = 0;
-        {
+        sources.SetSourceName(source++, "lua profile");
-            const std::string message{"Failed to open " + config.datasource_names_path +
-                                      " for writing"};
-            throw util::exception(message + SOURCE_REF);
-        }
-        datasource_stream << "lua profile" << std::endl;
 
         // Only write the filename, without path or extension.
         // This prevents information leakage, and keeps names short
         // for rendering in the debug tiles.
         for (auto const &name : config.segment_speed_lookup_paths)
         {
-            datasource_stream << boost::filesystem::path(name).stem().string() << std::endl;
+            sources.SetSourceName(source++, boost::filesystem::path(name).stem().string());
         }
+
+        extractor::io::write(config.datasource_names_path, sources);
     };
 
-    tbb::parallel_invoke(maybe_save_geometries, save_datasource_indexes, save_datastore_names);
+    tbb::parallel_invoke(maybe_save_geometries, save_datastore_names);
 
     std::vector<TurnPenalty> turn_weight_penalties;
     std::vector<TurnPenalty> turn_duration_penalties;