From a636e8cc131fa21e25d65e75935792b29021c866 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 9 Mar 2017 17:01:04 +0000 Subject: [PATCH] Include datasources in .geometries file and refactor .datasource_names --- CHANGELOG.md | 2 + features/options/contract/datasources.feature | 38 ----- features/step_definitions/options.js | 5 - .../contiguous_internalmem_datafacade.hpp | 139 ++++-------------- include/extractor/datasources.hpp | 48 ++++++ include/extractor/io.hpp | 27 ++++ include/extractor/segment_data_container.hpp | 32 +++- include/storage/serialization.hpp | 33 ----- include/storage/shared_datatype.hpp | 8 +- include/updater/updater_config.hpp | 2 - src/extractor/compressed_edge_container.cpp | 6 + src/storage/storage.cpp | 116 ++------------- src/storage/storage_config.cpp | 6 +- src/updater/updater.cpp | 53 ++----- 14 files changed, 166 insertions(+), 349 deletions(-) delete mode 100644 features/options/contract/datasources.feature create mode 100644 include/extractor/datasources.hpp diff --git a/CHANGELOG.md b/CHANGELOG.md index b6bd30dfc..d2b0d512a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ - Changes from 5.6 - Internals - Shared memory notification via conditional variables on Linux or semaphore queue on OS X and Windows with a limit of 128 OSRM Engine instances + - Files + - .osrm.datasource_index file was removed. Data is now part of .osrm.geometries. # 5.6.3 - Changes from 5.6.0 diff --git a/features/options/contract/datasources.feature b/features/options/contract/datasources.feature deleted file mode 100644 index 6da539c31..000000000 --- a/features/options/contract/datasources.feature +++ /dev/null @@ -1,38 +0,0 @@ -@prepare @options @files -Feature: osrm-contract command line options: datasources -# expansions: -# {processed_file} => path to .osrm file - - Background: - Given the profile "testbot" - Given the extract extra arguments "--generate-edge-lookup" - And the node map - """ - a b - """ - And the ways - | nodes | - | ab | - And the speed file - """ - 1,2,50 - 2,1,50 - 2,3,50 - 3,2,50 - 1,4,50 - 4,1,50 - """ - And the data has been extracted - - Scenario: osrm-contract - Passing base file - When I run "osrm-contract --segment-speed-file {speeds_file} {processed_file}" - Then datasource names should contain "lua profile,27_osrmcontract_passing_base_file_speeds" - And it should exit successfully - - Scenario: osrm-contract - Passing base file - Given the speed file - """ - """ - And the data has been extracted - When I run "osrm-contract --segment-speed-file {speeds_file} {processed_file}" - Then it should exit successfully diff --git a/features/step_definitions/options.js b/features/step_definitions/options.js index 7f23c1101..948f89208 100644 --- a/features/step_definitions/options.js +++ b/features/step_definitions/options.js @@ -103,11 +103,6 @@ module.exports = function () { assert.equal(this.stderr.split('\n').length - 1, parseInt(lines)); }); - this.Then(/^datasource names should contain "(.+)"$/, (expectedData) => { - const actualData = fs.readFileSync(this.processedCacheFile + '.datasource_names', {encoding:'UTF-8'}).trim().split('\n').join(','); - assert.equal(actualData, expectedData); - }); - this.Given(/^the query options$/, (table, callback) => { table.raw().forEach(tuple => { this.queryParams[tuple[0]] = tuple[1]; diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp index f24d106ce..6d88de727 100644 --- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp +++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp @@ -8,6 +8,7 @@ #include "engine/algorithm.hpp" #include "engine/geospatial_query.hpp" +#include "extractor/datasources.hpp" #include "extractor/guidance/turn_instruction.hpp" #include "extractor/guidance/turn_lane_types.hpp" #include "extractor/profile_properties.hpp" @@ -205,6 +206,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade std::string m_timestamp; extractor::ProfileProperties *m_profile_properties; + extractor::Datasources *m_datasources; unsigned m_check_sum; util::ShM::vector m_coordinate_list; @@ -219,7 +221,6 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade util::NameTable m_names_table; util::ShM::vector m_name_begin_indices; util::ShM::vector m_is_core_node; - util::ShM::vector m_datasource_list; util::ShM::vector m_lane_description_offsets; util::ShM::vector m_lane_description_masks; util::ShM::vector m_turn_weight_penalties; @@ -467,39 +468,21 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade geometries_rev_duration_list_ptr, data_layout.num_entries[storage::DataLayout::GEOMETRIES_REV_DURATION_LIST]); + auto datasources_list_ptr = data_layout.GetBlockPtr( + memory_block, storage::DataLayout::DATASOURCES_LIST); + util::ShM::vector datasources_list( + datasources_list_ptr, data_layout.num_entries[storage::DataLayout::DATASOURCES_LIST]); + segment_data = extractor::SegmentDataView{std::move(geometry_begin_indices), std::move(geometry_node_list), std::move(geometry_fwd_weight_list), std::move(geometry_rev_weight_list), std::move(geometry_fwd_duration_list), - std::move(geometry_rev_duration_list)}; + std::move(geometry_rev_duration_list), + std::move(datasources_list)}; - auto datasources_list_ptr = data_layout.GetBlockPtr( - memory_block, storage::DataLayout::DATASOURCES_LIST); - util::ShM::vector datasources_list( - datasources_list_ptr, data_layout.num_entries[storage::DataLayout::DATASOURCES_LIST]); - m_datasource_list = std::move(datasources_list); - - auto datasource_name_data_ptr = - data_layout.GetBlockPtr(memory_block, storage::DataLayout::DATASOURCE_NAME_DATA); - util::ShM::vector datasource_name_data( - datasource_name_data_ptr, - data_layout.num_entries[storage::DataLayout::DATASOURCE_NAME_DATA]); - m_datasource_name_data = std::move(datasource_name_data); - - auto datasource_name_offsets_ptr = data_layout.GetBlockPtr( - memory_block, storage::DataLayout::DATASOURCE_NAME_OFFSETS); - util::ShM::vector datasource_name_offsets( - datasource_name_offsets_ptr, - data_layout.num_entries[storage::DataLayout::DATASOURCE_NAME_OFFSETS]); - m_datasource_name_offsets = std::move(datasource_name_offsets); - - auto datasource_name_lengths_ptr = data_layout.GetBlockPtr( - memory_block, storage::DataLayout::DATASOURCE_NAME_LENGTHS); - util::ShM::vector datasource_name_lengths( - datasource_name_lengths_ptr, - data_layout.num_entries[storage::DataLayout::DATASOURCE_NAME_LENGTHS]); - m_datasource_name_lengths = std::move(datasource_name_lengths); + m_datasources = data_layout.GetBlockPtr( + memory_block, storage::DataLayout::DATASOURCES_NAMES); } void InitializeIntersectionClassPointers(storage::DataLayout &data_layout, char *memory_block) @@ -611,6 +594,24 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade return std::vector{range.begin(), range.end()}; } + // Returns the data source ids that were used to supply the edge + // weights. + virtual std::vector + GetUncompressedForwardDatasources(const EdgeID id) const override final + { + auto range = segment_data.GetForwardDatasources(id); + return std::vector{range.begin(), range.end()}; + } + + // Returns the data source ids that were used to supply the edge + // weights. + virtual std::vector + GetUncompressedReverseDatasources(const EdgeID id) const override final + { + auto range = segment_data.GetReverseDatasources(id); + return std::vector{range.begin(), range.end()}; + } + virtual GeometryID GetGeometryIndexForEdgeID(const EdgeID id) const override final { return m_via_geometry_list.at(id); @@ -781,89 +782,9 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade return m_name_table.GetDestinationsForID(id); } - // Returns the data source ids that were used to supply the edge - // weights. - virtual std::vector - GetUncompressedForwardDatasources(const EdgeID id) const override final - { - /* - * Data sources for geometries are stored in one place for - * both forward and reverse segments along the same bi- - * directional edge. The indices stores - * refences to where to find the beginning of the bi- - * directional edge in the list vector. For - * forward datasources of bi-directional edges, edges 2 to - * n of that edge need to be read. - */ - const auto begin = segment_data.GetOffset(id, 0) + 1; - const auto end = segment_data.GetOffset(id + 1, 0); - - std::vector result_datasources; - result_datasources.reserve(end - begin); - - // If there was no datasource info, return an array of 0's. - if (m_datasource_list.empty()) - { - result_datasources.resize(end - begin, 0); - } - else - { - std::copy(m_datasource_list.begin() + begin, - m_datasource_list.begin() + end, - std::back_inserter(result_datasources)); - } - - return result_datasources; - } - - // Returns the data source ids that were used to supply the edge - // weights. - virtual std::vector - GetUncompressedReverseDatasources(const EdgeID id) const override final - { - /* - * Datasources for geometries are stored in one place for - * both forward and reverse segments along the same bi- - * directional edge. The indices stores - * refences to where to find the beginning of the bi- - * directional edge in the list vector. For - * reverse datasources of bi-directional edges, edges 1 to - * n-1 of that edge need to be read in reverse. - */ - const auto begin = segment_data.GetOffset(id, 0); - const auto end = segment_data.GetOffset(id + 1, 0) - 1; - - std::vector result_datasources; - result_datasources.reserve(end - begin); - - // If there was no datasource info, return an array of 0's. - if (m_datasource_list.empty()) - { - result_datasources.resize(end - begin, 0); - } - else - { - std::reverse_copy(m_datasource_list.begin() + begin, - m_datasource_list.begin() + end, - std::back_inserter(result_datasources)); - } - - return result_datasources; - } - StringView GetDatasourceName(const DatasourceID id) const override final { - BOOST_ASSERT(m_datasource_name_offsets.size() >= 1); - BOOST_ASSERT(m_datasource_name_offsets.size() > id); - - auto first = m_datasource_name_data.begin() + m_datasource_name_offsets[id]; - auto last = m_datasource_name_data.begin() + m_datasource_name_offsets[id] + - m_datasource_name_lengths[id]; - // These iterators are useless: they're InputIterators onto a contiguous block of memory. - // Deref to get to the first element, then Addressof to get the memory address of the it. - const std::size_t len = &*last - &*first; - - return StringView{&*first, len}; + return m_datasources->GetSourceName(id); } std::string GetTimestamp() const override final { return m_timestamp; } diff --git a/include/extractor/datasources.hpp b/include/extractor/datasources.hpp new file mode 100644 index 000000000..03af4deea --- /dev/null +++ b/include/extractor/datasources.hpp @@ -0,0 +1,48 @@ +#ifndef OSRM_EXTRACT_DATASOURCES_HPP +#define OSRM_EXTRACT_DATASOURCES_HPP + +#include +#include +#include + +namespace osrm +{ +namespace extractor +{ + +class Datasources +{ + static constexpr const std::uint8_t MAX_NUM_SOURES = 255; + static constexpr const std::uint8_t MAX_LENGTH_NAME = 255; + + public: + Datasources() + { + std::fill(lengths.begin(), lengths.end(), 0); + std::fill(sources.begin(), sources.end(), '\0'); + } + + util::StringView GetSourceName(DatasourceID id) const + { + auto begin = sources.data() + (MAX_LENGTH_NAME * id); + + return util::StringView{begin, lengths[id]}; + } + + void SetSourceName(DatasourceID id, const std::string &name) + { + BOOST_ASSERT(name.size() < MAX_LENGTH_NAME); + lengths[id] = std::min(name.size(), MAX_LENGTH_NAME); + + auto out = sources.data() + (MAX_LENGTH_NAME * id); + std::copy(name.begin(), name.begin() + lengths[id], out); + } + + private: + std::array lengths; + std::array sources; +}; +} +} + +#endif diff --git a/include/extractor/io.hpp b/include/extractor/io.hpp index a705016ae..12e930749 100644 --- a/include/extractor/io.hpp +++ b/include/extractor/io.hpp @@ -1,10 +1,13 @@ #ifndef OSRM_EXTRACTOR_IO_HPP #define OSRM_EXTRACTOR_IO_HPP +#include "extractor/datasources.hpp" #include "extractor/segment_data_container.hpp" #include "storage/io.hpp" +#include + namespace osrm { namespace extractor @@ -12,6 +15,22 @@ namespace extractor namespace io { +void read(const boost::filesystem::path &path, Datasources &sources) +{ + const auto fingerprint = storage::io::FileReader::HasNoFingerprint; + storage::io::FileReader reader{path, fingerprint}; + + reader.ReadInto(sources); +} + +void write(const boost::filesystem::path &path, Datasources &sources) +{ + const auto fingerprint = storage::io::FileWriter::HasNoFingerprint; + storage::io::FileWriter writer{path, fingerprint}; + + writer.WriteFrom(sources); +} + template <> void read(const boost::filesystem::path &path, SegmentDataContainer &segment_data) { const auto fingerprint = storage::io::FileReader::HasNoFingerprint; @@ -27,12 +46,14 @@ template <> void read(const boost::filesystem::path &path, SegmentDataContainer segment_data.rev_weights.resize(num_entries); segment_data.fwd_durations.resize(num_entries); segment_data.rev_durations.resize(num_entries); + segment_data.datasources.resize(num_entries); reader.ReadInto(segment_data.nodes); reader.ReadInto(segment_data.fwd_weights); reader.ReadInto(segment_data.rev_weights); reader.ReadInto(segment_data.fwd_durations); reader.ReadInto(segment_data.rev_durations); + reader.ReadInto(segment_data.datasources); } template <> @@ -45,11 +66,17 @@ void write(const boost::filesystem::path &path, const SegmentDataContainer &segm writer.WriteFrom(segment_data.index); writer.WriteElementCount32(segment_data.nodes.size()); + BOOST_ASSERT(segment_data.fwd_weights.size() == segment_data.nodes.size()); + BOOST_ASSERT(segment_data.rev_weights.size() == segment_data.nodes.size()); + BOOST_ASSERT(segment_data.fwd_durations.size() == segment_data.nodes.size()); + BOOST_ASSERT(segment_data.rev_durations.size() == segment_data.nodes.size()); + BOOST_ASSERT(segment_data.datasources.size() == segment_data.nodes.size()); writer.WriteFrom(segment_data.nodes); writer.WriteFrom(segment_data.fwd_weights); writer.WriteFrom(segment_data.rev_weights); writer.WriteFrom(segment_data.fwd_durations); writer.WriteFrom(segment_data.rev_durations); + writer.WriteFrom(segment_data.datasources); } } } diff --git a/include/extractor/segment_data_container.hpp b/include/extractor/segment_data_container.hpp index 67bd60eff..55174e393 100644 --- a/include/extractor/segment_data_container.hpp +++ b/include/extractor/segment_data_container.hpp @@ -55,10 +55,11 @@ template class SegmentDataContainerImpl Vector fwd_weights_, Vector rev_weights_, Vector fwd_durations_, - Vector rev_durations_) + Vector rev_durations_, + Vector datasources_) : index(std::move(index_)), nodes(std::move(nodes_)), fwd_weights(std::move(fwd_weights_)), rev_weights(std::move(rev_weights_)), fwd_durations(std::move(fwd_durations_)), - rev_durations(std::move(rev_durations_)) + rev_durations(std::move(rev_durations_)), datasources(std::move(datasources_)) { } @@ -79,11 +80,13 @@ template class SegmentDataContainerImpl { return rev_weights[index[id] + offset]; } - // TODO we only need this for the datasource file since it breaks this - // abstraction, but uses this index - auto GetOffset(const DirectionalGeometryID id, const SegmentOffset offset) const + auto &ForwardDatasource(const DirectionalGeometryID id, const SegmentOffset offset) { - return index[id] + offset; + return datasources[index[id] + 1 + offset]; + } + auto &ReverseDatasource(const DirectionalGeometryID id, const SegmentOffset offset) + { + return datasources[index[id] + offset]; } auto GetForwardGeometry(const DirectionalGeometryID id) const @@ -131,6 +134,22 @@ template class SegmentDataContainerImpl return boost::adaptors::reverse(boost::make_iterator_range(begin, end)); } + auto GetForwardDatasources(const DirectionalGeometryID id) const + { + const auto begin = datasources.cbegin() + index.at(id) + 1; + const auto end = datasources.cbegin() + index.at(id + 1); + + return boost::make_iterator_range(begin, end); + } + + auto GetReverseDatasources(const DirectionalGeometryID id) const + { + const auto begin = datasources.cbegin() + index.at(id); + const auto end = datasources.cbegin() + index.at(id + 1) - 1; + + return boost::adaptors::reverse(boost::make_iterator_range(begin, end)); + } + auto GetNumberOfSegments() const { return fwd_weights.size(); } friend void @@ -147,6 +166,7 @@ template class SegmentDataContainerImpl Vector rev_weights; Vector fwd_durations; Vector rev_durations; + Vector datasources; }; } diff --git a/include/storage/serialization.hpp b/include/storage/serialization.hpp index 086708ea1..d89c03d6f 100644 --- a/include/storage/serialization.hpp +++ b/include/storage/serialization.hpp @@ -70,16 +70,6 @@ inline void readHSGR(io::FileReader &input_file, input_file.ReadInto(edge_buffer, number_of_edges); } -// Loads datasource_indexes from .datasource_indexes into memory -// Needs to be called after readElementCount() to get the correct offset in the stream -inline void readDatasourceIndexes(io::FileReader &datasource_indexes_file, - uint8_t *datasource_buffer, - const std::uint64_t number_of_datasource_indexes) -{ - BOOST_ASSERT(datasource_buffer); - datasource_indexes_file.ReadInto(datasource_buffer, number_of_datasource_indexes); -} - // Loads edge data from .edge files into memory which includes its // geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID // Needs to be called after readElementCount() to get the correct offset in the stream @@ -134,29 +124,6 @@ void readNodes(io::FileReader &nodes_file, BOOST_ASSERT(coordinate_list[i].IsValid()); } } - -// Reads datasource names out of .datasource_names files and metadata such as -// the length and offset of each name -struct DatasourceNamesData -{ - std::vector names; - std::vector offsets; - std::vector lengths; -}; -inline DatasourceNamesData readDatasourceNames(io::FileReader &datasource_names_file) -{ - DatasourceNamesData datasource_names_data; - std::vector lines = datasource_names_file.ReadLines(); - for (const auto &name : lines) - { - datasource_names_data.offsets.push_back(datasource_names_data.names.size()); - datasource_names_data.lengths.push_back(name.size()); - std::copy(name.c_str(), - name.c_str() + name.size(), - std::back_inserter(datasource_names_data.names)); - } - return datasource_names_data; -} } } } diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp index cb1f75c10..0846ee3b6 100644 --- a/include/storage/shared_datatype.hpp +++ b/include/storage/shared_datatype.hpp @@ -40,9 +40,7 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA", "FILE_INDEX_PATH", "CH_CORE_MARKER", "DATASOURCES_LIST", - "DATASOURCE_NAME_DATA", - "DATASOURCE_NAME_OFFSETS", - "DATASOURCE_NAME_LENGTHS", + "DATASOURCES_NAMES", "PROPERTIES", "BEARING_CLASSID", "BEARING_OFFSETS", @@ -94,9 +92,7 @@ struct DataLayout FILE_INDEX_PATH, CH_CORE_MARKER, DATASOURCES_LIST, - DATASOURCE_NAME_DATA, - DATASOURCE_NAME_OFFSETS, - DATASOURCE_NAME_LENGTHS, + DATASOURCES_NAMES, PROPERTIES, BEARING_CLASSID, BEARING_OFFSETS, diff --git a/include/updater/updater_config.hpp b/include/updater/updater_config.hpp index b603a2d8b..4f905abeb 100644 --- a/include/updater/updater_config.hpp +++ b/include/updater/updater_config.hpp @@ -51,7 +51,6 @@ struct UpdaterConfig final geometry_path = osrm_input_path.string() + ".geometry"; rtree_leaf_path = osrm_input_path.string() + ".fileIndex"; datasource_names_path = osrm_input_path.string() + ".datasource_names"; - datasource_indexes_path = osrm_input_path.string() + ".datasource_indexes"; profile_properties_path = osrm_input_path.string() + ".properties"; } @@ -71,7 +70,6 @@ struct UpdaterConfig final std::vector segment_speed_lookup_paths; std::vector turn_penalty_lookup_paths; - std::string datasource_indexes_path; std::string datasource_names_path; std::string profile_properties_path; }; diff --git a/src/extractor/compressed_edge_container.cpp b/src/extractor/compressed_edge_container.cpp index 9b1dd1672..0e2b3132b 100644 --- a/src/extractor/compressed_edge_container.cpp +++ b/src/extractor/compressed_edge_container.cpp @@ -221,6 +221,7 @@ void CompressedEdgeContainer::InitializeBothwayVector() segment_data->rev_weights.reserve(m_compressed_oneway_geometries.size()); segment_data->fwd_durations.reserve(m_compressed_oneway_geometries.size()); segment_data->rev_durations.reserve(m_compressed_oneway_geometries.size()); + segment_data->datasources.reserve(m_compressed_oneway_geometries.size()); } unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID r_edge_id) @@ -238,11 +239,14 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID const auto &first_node = reverse_bucket.back(); + constexpr DatasourceID LUA_SOURCE = 0; + segment_data->nodes.emplace_back(first_node.node_id); segment_data->fwd_weights.emplace_back(INVALID_EDGE_WEIGHT); segment_data->rev_weights.emplace_back(first_node.weight); segment_data->fwd_durations.emplace_back(INVALID_EDGE_WEIGHT); segment_data->rev_durations.emplace_back(first_node.duration); + segment_data->datasources.emplace_back(LUA_SOURCE); for (std::size_t i = 0; i < forward_bucket.size() - 1; ++i) { @@ -256,6 +260,7 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID segment_data->rev_weights.emplace_back(rev_node.weight); segment_data->fwd_durations.emplace_back(fwd_node.duration); segment_data->rev_durations.emplace_back(rev_node.duration); + segment_data->datasources.emplace_back(LUA_SOURCE); } const auto &last_node = forward_bucket.back(); @@ -265,6 +270,7 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID segment_data->rev_weights.emplace_back(INVALID_EDGE_WEIGHT); segment_data->fwd_durations.emplace_back(last_node.duration); segment_data->rev_durations.emplace_back(INVALID_EDGE_WEIGHT); + segment_data->datasources.emplace_back(LUA_SOURCE); return zipped_geometry_id; } diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 0b0980967..3d631194b 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -3,6 +3,7 @@ #include "extractor/compressed_edge_container.hpp" #include "extractor/edge_based_edge.hpp" #include "extractor/guidance/turn_instruction.hpp" +#include "extractor/io.hpp" #include "extractor/original_edge_data.hpp" #include "extractor/profile_properties.hpp" #include "extractor/query_node.hpp" @@ -339,40 +340,13 @@ void Storage::PopulateLayout(DataLayout &layout) number_of_compressed_geometries); layout.SetBlockSize(DataLayout::GEOMETRIES_REV_DURATION_LIST, number_of_compressed_geometries); + layout.SetBlockSize(DataLayout::DATASOURCES_LIST, + number_of_compressed_geometries); } - // load datasource sizes. This file is optional, and it's non-fatal if it doesn't exist. - if (boost::filesystem::exists(config.datasource_indexes_path)) + // Load datasource name sizes. { - io::FileReader reader(config.datasource_indexes_path, io::FileReader::HasNoFingerprint); - const auto number_of_datasources = reader.ReadElementCount64(); - layout.SetBlockSize(DataLayout::DATASOURCES_LIST, number_of_datasources); - } - else - { - layout.SetBlockSize(DataLayout::DATASOURCES_LIST, 0); - } - - // Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't exist - if (boost::filesystem::exists(config.datasource_names_path)) - { - io::FileReader reader(config.datasource_names_path, io::FileReader::HasNoFingerprint); - - const serialization::DatasourceNamesData datasource_names_data = - serialization::readDatasourceNames(reader); - - layout.SetBlockSize(DataLayout::DATASOURCE_NAME_DATA, - datasource_names_data.names.size()); - layout.SetBlockSize(DataLayout::DATASOURCE_NAME_OFFSETS, - datasource_names_data.offsets.size()); - layout.SetBlockSize(DataLayout::DATASOURCE_NAME_LENGTHS, - datasource_names_data.lengths.size()); - } - else - { - layout.SetBlockSize(DataLayout::DATASOURCE_NAME_DATA, 0); - layout.SetBlockSize(DataLayout::DATASOURCE_NAME_OFFSETS, 0); - layout.SetBlockSize(DataLayout::DATASOURCE_NAME_LENGTHS, 0); + layout.SetBlockSize(DataLayout::DATASOURCES_NAMES, 1); } { @@ -689,81 +663,17 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) BOOST_ASSERT(geometry_node_lists_count == layout.num_entries[DataLayout::GEOMETRIES_REV_DURATION_LIST]); geometry_input_file.ReadInto(geometries_rev_duration_list_ptr, geometry_node_lists_count); + + const auto datasource_list_ptr = + layout.GetBlockPtr(memory_ptr, DataLayout::DATASOURCES_LIST); + BOOST_ASSERT(geometry_node_lists_count == layout.num_entries[DataLayout::DATASOURCES_LIST]); + geometry_input_file.ReadInto(datasource_list_ptr, geometry_node_lists_count); } - if (boost::filesystem::exists(config.datasource_indexes_path)) { - io::FileReader geometry_datasource_file(config.datasource_indexes_path, - io::FileReader::HasNoFingerprint); - const auto number_of_compressed_datasources = geometry_datasource_file.ReadElementCount64(); - - // load datasource information (if it exists) - const auto datasources_list_ptr = - layout.GetBlockPtr(memory_ptr, DataLayout::DATASOURCES_LIST); - if (number_of_compressed_datasources > 0) - { - serialization::readDatasourceIndexes( - geometry_datasource_file, datasources_list_ptr, number_of_compressed_datasources); - } - } - else - { - layout.GetBlockPtr(memory_ptr, DataLayout::DATASOURCES_LIST); - } - - if (boost::filesystem::exists(config.datasource_names_path)) - { - io::FileReader datasource_names_file(config.datasource_names_path, - io::FileReader::HasNoFingerprint); - - const auto datasource_names_data = - serialization::readDatasourceNames(datasource_names_file); - - // load datasource name information (if it exists) - const auto datasource_name_data_ptr = - layout.GetBlockPtr(memory_ptr, DataLayout::DATASOURCE_NAME_DATA); - if (layout.GetBlockSize(DataLayout::DATASOURCE_NAME_DATA) > 0) - { - BOOST_ASSERT(std::distance(datasource_names_data.names.begin(), - datasource_names_data.names.end()) * - sizeof(decltype(datasource_names_data.names)::value_type) <= - layout.GetBlockSize(DataLayout::DATASOURCE_NAME_DATA)); - std::copy(datasource_names_data.names.begin(), - datasource_names_data.names.end(), - datasource_name_data_ptr); - } - - const auto datasource_name_offsets_ptr = - layout.GetBlockPtr(memory_ptr, DataLayout::DATASOURCE_NAME_OFFSETS); - if (layout.GetBlockSize(DataLayout::DATASOURCE_NAME_OFFSETS) > 0) - { - BOOST_ASSERT(std::distance(datasource_names_data.offsets.begin(), - datasource_names_data.offsets.end()) * - sizeof(decltype(datasource_names_data.offsets)::value_type) <= - layout.GetBlockSize(DataLayout::DATASOURCE_NAME_OFFSETS)); - std::copy(datasource_names_data.offsets.begin(), - datasource_names_data.offsets.end(), - datasource_name_offsets_ptr); - } - - const auto datasource_name_lengths_ptr = - layout.GetBlockPtr(memory_ptr, DataLayout::DATASOURCE_NAME_LENGTHS); - if (layout.GetBlockSize(DataLayout::DATASOURCE_NAME_LENGTHS) > 0) - { - BOOST_ASSERT(std::distance(datasource_names_data.lengths.begin(), - datasource_names_data.lengths.end()) * - sizeof(decltype(datasource_names_data.lengths)::value_type) <= - layout.GetBlockSize(DataLayout::DATASOURCE_NAME_LENGTHS)); - std::copy(datasource_names_data.lengths.begin(), - datasource_names_data.lengths.end(), - datasource_name_lengths_ptr); - } - } - else - { - layout.GetBlockPtr(memory_ptr, DataLayout::DATASOURCE_NAME_DATA); - layout.GetBlockPtr(memory_ptr, DataLayout::DATASOURCE_NAME_OFFSETS); - layout.GetBlockPtr(memory_ptr, DataLayout::DATASOURCE_NAME_LENGTHS); + const auto datasources_names_ptr = layout.GetBlockPtr( + memory_ptr, DataLayout::DATASOURCES_NAMES); + extractor::io::read(config.datasource_names_path, *datasources_names_ptr); } // Loading list of coordinates diff --git a/src/storage/storage_config.cpp b/src/storage/storage_config.cpp index 0b1d4ea1e..800a7756a 100644 --- a/src/storage/storage_config.cpp +++ b/src/storage/storage_config.cpp @@ -32,7 +32,6 @@ StorageConfig::StorageConfig(const boost::filesystem::path &base) turn_weight_penalties_path{base.string() + ".turn_weight_penalties"}, turn_duration_penalties_path{base.string() + ".turn_duration_penalties"}, datasource_names_path{base.string() + ".datasource_names"}, - datasource_indexes_path{base.string() + ".datasource_indexes"}, names_data_path{base.string() + ".names"}, properties_path{base.string() + ".properties"}, intersection_class_path{base.string() + ".icd"}, turn_lane_data_path{base.string() + ".tld"}, turn_lane_description_path{base.string() + ".tls"}, @@ -54,7 +53,8 @@ bool StorageConfig::IsValid() const turn_duration_penalties_path, names_data_path, properties_path, - intersection_class_path})) + intersection_class_path, + datasource_names_path})) { return false; } @@ -62,7 +62,7 @@ bool StorageConfig::IsValid() const // TODO: add algorithm checks // CH files - CheckFileList({hsgr_data_path, core_data_path, datasource_names_path, datasource_indexes_path}); + CheckFileList({hsgr_data_path, core_data_path}); // MLD files CheckFileList({mld_partition_path, mld_storage_path, edge_based_graph_path}); diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index 1f44ee6fb..bfdf9f8f9 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -262,10 +262,6 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e auto segment_speed_lookup = csv::readSegmentValues(config.segment_speed_lookup_paths); auto turn_penalty_lookup = csv::readTurnValues(config.turn_penalty_lookup_paths); - // If we update the edge weights, this file will hold the datasource information for each - // segment; the other files will also be conditionally filled concurrently if we make an update - std::vector geometry_datasource; - std::vector internal_to_external_node_map; extractor::SegmentDataContainer segment_data; @@ -294,14 +290,6 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e // Here, we have to update the compressed geometry weights // First, we need the external-to-internal node lookup table - // This is a list of the "data source id" for every segment in the compressed - // geometry container. We assume that everything so far has come from the - // profile (data source 0). Here, we replace the 0's with the index of the - // CSV file that supplied the value that gets used for that segment, then - // we write out this list so that it can be returned by the debugging - // vector tiles later on. - geometry_datasource.resize(segment_data.GetNumberOfSegments(), 0); - // Now, we iterate over all the segments stored in the StaticRTree, updating // the packed geometry weights in the `.geometries` file (note: we do not // update the RTree itself, we just use the leaf nodes to iterate over all segments) @@ -364,8 +352,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e segment_data.ForwardWeight(geometry_id, segment_offset) = new_segment_weight; segment_data.ForwardDuration(geometry_id, segment_offset) = new_segment_duration; - geometry_datasource[segment_data.GetOffset(geometry_id, segment_offset) + 1] = - value->source; + segment_data.ForwardDatasource(geometry_id, segment_offset) = value->source; fwd_source = value->source; } @@ -385,8 +372,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e segment_data.ReverseWeight(geometry_id, segment_offset) = new_segment_weight; segment_data.ReverseDuration(geometry_id, segment_offset) = new_segment_duration; - geometry_datasource[segment_data.GetOffset(geometry_id, segment_offset)] = - value->source; + segment_data.ReverseDatasource(geometry_id, segment_offset) = value->source; rev_source = value->source; } @@ -430,44 +416,23 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e extractor::io::write(config.geometry_path, segment_data); }; - const auto save_datasource_indexes = [&] { - std::ofstream datasource_stream(config.datasource_indexes_path, std::ios::binary); - if (!datasource_stream) - { - const std::string message{"Failed to open " + config.datasource_indexes_path + - " for writing"}; - throw util::exception(message + SOURCE_REF); - } - std::uint64_t number_of_datasource_entries = geometry_datasource.size(); - datasource_stream.write(reinterpret_cast(&number_of_datasource_entries), - sizeof(number_of_datasource_entries)); - if (number_of_datasource_entries > 0) - { - datasource_stream.write(reinterpret_cast(&(geometry_datasource[0])), - number_of_datasource_entries * sizeof(uint8_t)); - } - }; - const auto save_datastore_names = [&] { - std::ofstream datasource_stream(config.datasource_names_path, std::ios::binary); - if (!datasource_stream) - { - const std::string message{"Failed to open " + config.datasource_names_path + - " for writing"}; - throw util::exception(message + SOURCE_REF); - } - datasource_stream << "lua profile" << std::endl; + extractor::Datasources sources; + DatasourceID source = 0; + sources.SetSourceName(source++, "lua profile"); // Only write the filename, without path or extension. // This prevents information leakage, and keeps names short // for rendering in the debug tiles. for (auto const &name : config.segment_speed_lookup_paths) { - datasource_stream << boost::filesystem::path(name).stem().string() << std::endl; + sources.SetSourceName(source++, boost::filesystem::path(name).stem().string()); } + + extractor::io::write(config.datasource_names_path, sources); }; - tbb::parallel_invoke(maybe_save_geometries, save_datasource_indexes, save_datastore_names); + tbb::parallel_invoke(maybe_save_geometries, save_datastore_names); std::vector turn_weight_penalties; std::vector turn_duration_penalties;