From b8260e44fae1ce7dd78621cbd283e1a6ae1eaefe Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 21 Mar 2018 11:10:02 +0000 Subject: [PATCH] Port .names file to tar --- .../contiguous_internalmem_datafacade.hpp | 25 ++- .../extractor/edge_based_graph_factory.hpp | 6 +- include/extractor/extractor.hpp | 4 +- include/extractor/files.hpp | 21 +- .../intersection/have_identical_names.hpp | 4 +- .../intersection/mergable_road_detector.hpp | 7 +- include/extractor/name_table.hpp | 120 +++++++++++ include/extractor/serialization.hpp | 38 +++- include/guidance/driveway_handler.hpp | 2 +- include/guidance/guidance_processing.hpp | 4 +- include/guidance/intersection_handler.hpp | 6 +- include/guidance/is_through_street.hpp | 5 +- include/guidance/motorway_handler.hpp | 5 +- include/guidance/roundabout_handler.hpp | 5 +- ...segregated_intersection_classification.hpp | 9 +- include/guidance/sliproad_handler.hpp | 5 +- include/guidance/statistics_handler.hpp | 2 +- include/guidance/suppress_mode_handler.hpp | 5 +- include/guidance/turn_analysis.hpp | 5 +- include/guidance/turn_handler.hpp | 5 +- include/guidance/turn_lane_handler.hpp | 3 +- include/storage/shared_datatype.hpp | 12 +- include/util/guidance/name_announcements.hpp | 5 +- include/util/indexed_data.hpp | 198 +++++++++--------- include/util/name_table.hpp | 51 ----- include/util/serialization.hpp | 17 ++ src/extractor/edge_based_graph_factory.cpp | 2 +- src/extractor/extraction_containers.cpp | 7 +- src/extractor/extractor.cpp | 14 +- .../intersection/have_identical_names.cpp | 4 +- .../intersection/mergable_road_detector.cpp | 6 +- src/guidance/driveway_handler.cpp | 2 +- src/guidance/guidance_processing.cpp | 2 +- src/guidance/intersection_handler.cpp | 2 +- src/guidance/motorway_handler.cpp | 2 +- src/guidance/roundabout_handler.cpp | 2 +- ...segregated_intersection_classification.cpp | 3 +- src/guidance/sliproad_handler.cpp | 2 +- src/guidance/suppress_mode_handler.cpp | 2 +- src/guidance/turn_analysis.cpp | 2 +- src/guidance/turn_handler.cpp | 2 +- src/osrm/osrm.cpp | 2 +- src/storage/storage.cpp | 32 +-- src/util/name_table.cpp | 100 --------- unit_tests/{util => extractor}/name_table.cpp | 30 +-- unit_tests/util/indexed_data.cpp | 70 +------ unit_tests/util/serialization.cpp | 40 ++++ 47 files changed, 459 insertions(+), 438 deletions(-) create mode 100644 include/extractor/name_table.hpp delete mode 100644 include/util/name_table.hpp delete mode 100644 src/util/name_table.cpp rename unit_tests/{util => extractor}/name_table.cpp (83%) diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp index 079bf47fd..32ed08151 100644 --- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp +++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp @@ -15,6 +15,7 @@ #include "extractor/edge_based_node.hpp" #include "extractor/intersection_bearings_container.hpp" #include "extractor/maneuver_override.hpp" +#include "extractor/name_table.hpp" #include "extractor/node_data_container.hpp" #include "extractor/packed_osm_ids.hpp" #include "extractor/profile_properties.hpp" @@ -40,7 +41,6 @@ #include "util/guidance/entry_class.hpp" #include "util/guidance/turn_lanes.hpp" #include "util/log.hpp" -#include "util/name_table.hpp" #include "util/packed_vector.hpp" #include "util/range_table.hpp" #include "util/rectangle.hpp" @@ -215,7 +215,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade extractor::IntersectionBearingsView intersection_bearings_view; - util::NameTable m_name_table; + extractor::NameTableView m_name_table; // the look-up table for entry classes. An entry class lists the possibility of entry for all // available turns. Such a class id is stored with every edge. util::vector_view m_entry_class_table; @@ -353,11 +353,20 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade void InitializeNamePointers(storage::DataLayout &data_layout, char *memory_block) { - auto name_data_ptr = - data_layout.GetBlockPtr(memory_block, storage::DataLayout::NAME_CHAR_DATA); - const auto name_data_size = - data_layout.GetBlockEntries(storage::DataLayout::NAME_CHAR_DATA); - m_name_table.reset(name_data_ptr, name_data_ptr + name_data_size); + const auto name_blocks_ptr = + data_layout.GetBlockPtr( + memory_block, storage::DataLayout::NAME_BLOCKS); + const auto name_values_ptr = + data_layout.GetBlockPtr( + memory_block, storage::DataLayout::NAME_VALUES); + + util::vector_view blocks( + name_blocks_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS)); + util::vector_view values( + name_values_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_VALUES)); + + extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)}; + m_name_table = extractor::NameTableView{std::move(index_data_view)}; } void InitializeTurnLaneDescriptionsPointers(storage::DataLayout &data_layout, @@ -938,7 +947,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade auto found_range = std::equal_range( m_maneuver_overrides.begin(), m_maneuver_overrides.end(), edge_based_node_id, Comp{}); - std::for_each(found_range.first, found_range.second, [&](const auto & override) { + std::for_each(found_range.first, found_range.second, [&](const auto &override) { std::vector sequence( m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_begin, m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_end); diff --git a/include/extractor/edge_based_graph_factory.hpp b/include/extractor/edge_based_graph_factory.hpp index fce5bc206..61e2f1cbf 100644 --- a/include/extractor/edge_based_graph_factory.hpp +++ b/include/extractor/edge_based_graph_factory.hpp @@ -9,6 +9,7 @@ #include "extractor/edge_based_node_segment.hpp" #include "extractor/extraction_turn.hpp" #include "extractor/maneuver_override.hpp" +#include "extractor/name_table.hpp" #include "extractor/nbg_to_ebg.hpp" #include "extractor/node_data_container.hpp" #include "extractor/query_node.hpp" @@ -18,7 +19,6 @@ #include "util/concurrent_id_map.hpp" #include "util/deallocating_vector.hpp" -#include "util/name_table.hpp" #include "util/node_based_graph.hpp" #include "util/typedefs.hpp" @@ -70,7 +70,7 @@ class EdgeBasedGraphFactory const std::unordered_set &barrier_nodes, const std::unordered_set &traffic_lights, const std::vector &coordinates, - const util::NameTable &name_table, + const NameTable &name_table, const std::unordered_set &segregated_edges, const LaneDescriptionMap &lane_description_map); @@ -138,7 +138,7 @@ class EdgeBasedGraphFactory const std::unordered_set &m_traffic_lights; const CompressedEdgeContainer &m_compressed_edge_container; - const util::NameTable &name_table; + const NameTable &name_table; const std::unordered_set &segregated_edges; const LaneDescriptionMap &lane_description_map; diff --git a/include/extractor/extractor.hpp b/include/extractor/extractor.hpp index 588488e6b..fc1df6382 100644 --- a/include/extractor/extractor.hpp +++ b/include/extractor/extractor.hpp @@ -77,7 +77,7 @@ class Extractor const std::vector &turn_restrictions, const std::vector &conditional_turn_restrictions, const std::unordered_set &segregated_edges, - const util::NameTable &name_table, + const NameTable &name_table, const std::vector &maneuver_overrides, const LaneDescriptionMap &turn_lane_map, // for calculating turn penalties @@ -116,7 +116,7 @@ class Extractor const std::unordered_set &barrier_nodes, const std::vector &turn_restrictions, const std::vector &conditional_turn_restrictions, - const util::NameTable &name_table, + const NameTable &name_table, LaneDescriptionMap lane_description_map, ScriptingEnvironment &scripting_environment); }; diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 598215d32..da4d9d96e 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -411,7 +411,8 @@ void readRawNBGraph(const boost::filesystem::path &path, osm_node_ids.push_back(current_node.node_id); index++; }; - reader.ReadStreaming("/extractor/nodes", boost::make_function_output_iterator(decode)); + reader.ReadStreaming("/extractor/nodes", + boost::make_function_output_iterator(decode)); reader.ReadStreaming("/extractor/barriers", barriers); @@ -420,6 +421,24 @@ void readRawNBGraph(const boost::filesystem::path &path, storage::serialization::read(reader, "/extractor/edges", edge_list); storage::serialization::read(reader, "/extractor/annotations", annotations); } + +template +void readNames(const boost::filesystem::path &path, NameTableT &table) +{ + const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; + storage::tar::FileReader reader{path, fingerprint}; + + serialization::read(reader, "/common/names", table); +} + +template +void writeNames(const boost::filesystem::path &path, const NameTableT &table) +{ + const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; + storage::tar::FileWriter writer{path, fingerprint}; + + serialization::write(writer, "/common/names", table); +} } } } diff --git a/include/extractor/intersection/have_identical_names.hpp b/include/extractor/intersection/have_identical_names.hpp index de51777fa..72168d829 100644 --- a/include/extractor/intersection/have_identical_names.hpp +++ b/include/extractor/intersection/have_identical_names.hpp @@ -1,9 +1,9 @@ #ifndef OSRM_EXTRACTOR_INTERSECTION_HAVE_IDENTICAL_NAMES_HPP_ #define OSRM_EXTRACTOR_INTERSECTION_HAVE_IDENTICAL_NAMES_HPP_ +#include "extractor/name_table.hpp" #include "extractor/suffix_table.hpp" #include "guidance/constants.hpp" -#include "util/name_table.hpp" namespace osrm { @@ -17,7 +17,7 @@ namespace intersection // rhs->lhs) bool HaveIdenticalNames(const NameID lhs, const NameID rhs, - const util::NameTable &name_table, + const NameTable &name_table, const SuffixTable &street_name_suffix_table); } // namespace intersection diff --git a/include/extractor/intersection/mergable_road_detector.hpp b/include/extractor/intersection/mergable_road_detector.hpp index b6a90df15..4739385ba 100644 --- a/include/extractor/intersection/mergable_road_detector.hpp +++ b/include/extractor/intersection/mergable_road_detector.hpp @@ -6,7 +6,10 @@ #include "extractor/intersection/have_identical_names.hpp" #include "extractor/restriction_index.hpp" #include "extractor/turn_lane_types.hpp" +#include "extractor/name_table.hpp" + #include "guidance/intersection.hpp" + #include "util/coordinate.hpp" #include "util/node_based_graph.hpp" #include "util/typedefs.hpp" @@ -49,7 +52,7 @@ class MergableRoadDetector const RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const SuffixTable &street_name_suffix_table); // OSM ways tend to be modelled as separate ways for different directions. This is often due to @@ -168,7 +171,7 @@ class MergableRoadDetector const TurnLanesIndexedArray &turn_lanes_data; // name detection - const util::NameTable &name_table; + const extractor::NameTable &name_table; const SuffixTable &street_name_suffix_table; const CoordinateExtractor coordinate_extractor; diff --git a/include/extractor/name_table.hpp b/include/extractor/name_table.hpp new file mode 100644 index 000000000..a791a7cc0 --- /dev/null +++ b/include/extractor/name_table.hpp @@ -0,0 +1,120 @@ +#ifndef OSRM_EXTRACTOR_NAME_TABLE_HPP +#define OSRM_EXTRACTOR_NAME_TABLE_HPP + +#include "util/indexed_data.hpp" +#include "util/string_view.hpp" +#include "util/typedefs.hpp" + +#include + +namespace osrm +{ +namespace extractor +{ + +namespace detail +{ +template class NameTableImpl; +} + +namespace serialization +{ +template +inline void read(storage::tar::FileReader &reader, + const std::string &name, + detail::NameTableImpl &index_data); + +template +inline void write(storage::tar::FileWriter &writer, + const std::string &name, + const detail::NameTableImpl &index_data); +} + +namespace detail +{ +// This class provides a limited view over all the string data we serialize out. +// The following functions are a subset of what is available. +// See the data facades for they provide full access to this serialized string data. +// Way string data is stored in blocks based on `id` as follows: +// +// | name | destination | pronunciation | ref | exits +// ^ ^ +// [range) +// ^ id + 2 +// +// `id + offset` gives us the range of chars. +// +// Offset 0 is name, 1 is destination, 2 is pronunciation, 3 is ref, 4 is exits +// See datafacades and extractor callbacks for details. +template class NameTableImpl +{ + public: + using IndexedData = + util::detail::IndexedDataImpl, Ownership>; + using ResultType = typename IndexedData::ResultType; + using ValueType = typename IndexedData::ValueType; + + NameTableImpl() {} + + NameTableImpl(IndexedData indexed_data_) : indexed_data{std::move(indexed_data_)} {} + + util::StringView GetNameForID(const NameID id) const + { + if (id == INVALID_NAMEID) + return {}; + + return indexed_data.at(id + 0); + } + + util::StringView GetDestinationsForID(const NameID id) const + { + if (id == INVALID_NAMEID) + return {}; + + return indexed_data.at(id + 1); + } + + util::StringView GetExitsForID(const NameID id) const + { + if (id == INVALID_NAMEID) + return {}; + + return indexed_data.at(id + 4); + } + + util::StringView GetRefForID(const NameID id) const + { + if (id == INVALID_NAMEID) + return {}; + + const constexpr auto OFFSET_REF = 3u; + return indexed_data.at(id + OFFSET_REF); + } + + util::StringView GetPronunciationForID(const NameID id) const + { + if (id == INVALID_NAMEID) + return {}; + + const constexpr auto OFFSET_PRONUNCIATION = 2u; + return indexed_data.at(id + OFFSET_PRONUNCIATION); + } + + friend void serialization::read(storage::tar::FileReader &reader, + const std::string &name, + NameTableImpl &index_data); + + friend void serialization::write(storage::tar::FileWriter &writer, + const std::string &name, + const NameTableImpl &index_data); + private: + IndexedData indexed_data; +}; +} + +using NameTable = detail::NameTableImpl; +using NameTableView = detail::NameTableImpl; +} // namespace extractor +} // namespace osrm + +#endif // OSRM_EXTRACTOR_NAME_TABLE_HPP diff --git a/include/extractor/serialization.hpp b/include/extractor/serialization.hpp index 3cf3fbab5..e388dc8cb 100644 --- a/include/extractor/serialization.hpp +++ b/include/extractor/serialization.hpp @@ -1,10 +1,11 @@ #ifndef OSRM_EXTRACTOR_IO_HPP #define OSRM_EXTRACTOR_IO_HPP -#include "conditional_turn_penalty.hpp" +#include "extractor/conditional_turn_penalty.hpp" #include "extractor/datasources.hpp" #include "extractor/intersection_bearings_container.hpp" #include "extractor/maneuver_override.hpp" +#include "extractor/name_table.hpp" #include "extractor/nbg_to_ebg.hpp" #include "extractor/node_data_container.hpp" #include "extractor/profile_properties.hpp" @@ -117,16 +118,18 @@ inline void read(storage::tar::FileReader &reader, { // read actual data storage::serialization::read(reader, name + "/nodes", node_data_container.nodes); - storage::serialization::read(reader, name + "/annotations", node_data_container.annotation_data); + storage::serialization::read( + reader, name + "/annotations", node_data_container.annotation_data); } template inline void write(storage::tar::FileWriter &writer, - const std::string& name, + const std::string &name, const detail::EdgeBasedNodeDataContainerImpl &node_data_container) { storage::serialization::write(writer, name + "/nodes", node_data_container.nodes); - storage::serialization::write(writer, name + "/annotations", node_data_container.annotation_data); + storage::serialization::write( + writer, name + "/annotations", node_data_container.annotation_data); } inline void read(storage::io::FileReader &reader, NodeRestriction &restriction) @@ -307,7 +310,8 @@ inline void write(storage::io::BufferWriter &writer, } } -inline void read(storage::io::BufferReader &reader, std::vector &conditional_penalties) +inline void read(storage::io::BufferReader &reader, + std::vector &conditional_penalties) { auto num_elements = reader.ReadElementCount64(); conditional_penalties.resize(num_elements); @@ -318,8 +322,8 @@ inline void read(storage::io::BufferReader &reader, std::vector &conditional_penalties) + const std::string &name, + const std::vector &conditional_penalties) { storage::io::BufferWriter buffer_writer; write(buffer_writer, conditional_penalties); @@ -328,7 +332,7 @@ inline void write(storage::tar::FileWriter &writer, } inline void read(storage::tar::FileReader &reader, - const std::string& name, + const std::string &name, std::vector &conditional_penalties) { std::string buffer; @@ -336,9 +340,25 @@ inline void read(storage::tar::FileReader &reader, storage::io::BufferReader buffer_reader{buffer}; read(buffer_reader, conditional_penalties); - } +template +inline void write(storage::tar::FileWriter &writer, + const std::string &name, + const detail::NameTableImpl &name_table) +{ + storage::io::BufferWriter buffer_writer; + util::serialization::write(writer, name, name_table.indexed_data); +} + +template +inline void read(storage::tar::FileReader &reader, + const std::string &name, + detail::NameTableImpl &name_table) +{ + std::string buffer; + util::serialization::read(reader, name, name_table.indexed_data); +} } } } diff --git a/include/guidance/driveway_handler.hpp b/include/guidance/driveway_handler.hpp index 9c247dccc..1ac0d401c 100644 --- a/include/guidance/driveway_handler.hpp +++ b/include/guidance/driveway_handler.hpp @@ -19,7 +19,7 @@ class DrivewayHandler final : public IntersectionHandler const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table); ~DrivewayHandler() override final = default; diff --git a/include/guidance/guidance_processing.hpp b/include/guidance/guidance_processing.hpp index 0b4034e24..12b97c361 100644 --- a/include/guidance/guidance_processing.hpp +++ b/include/guidance/guidance_processing.hpp @@ -8,12 +8,12 @@ #include "extractor/suffix_table.hpp" #include "extractor/turn_lane_types.hpp" #include "extractor/way_restriction_map.hpp" +#include "extractor/name_table.hpp" #include "util/coordinate.hpp" #include "util/guidance/bearing_class.hpp" #include "util/guidance/entry_class.hpp" #include "util/guidance/turn_lanes.hpp" -#include "util/name_table.hpp" #include "util/node_based_graph.hpp" #include @@ -33,7 +33,7 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph, const std::unordered_set &barrier_nodes, const extractor::RestrictionMap &node_restriction_map, const extractor::WayRestrictionMap &way_restriction_map, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &suffix_table, const extractor::TurnLanesIndexedArray &turn_lanes_data, extractor::LaneDescriptionMap &lane_description_map, diff --git a/include/guidance/intersection_handler.hpp b/include/guidance/intersection_handler.hpp index 30b087bc8..5860cd4ae 100644 --- a/include/guidance/intersection_handler.hpp +++ b/include/guidance/intersection_handler.hpp @@ -4,13 +4,13 @@ #include "extractor/intersection/intersection_analysis.hpp" #include "extractor/intersection/node_based_graph_walker.hpp" #include "extractor/suffix_table.hpp" +#include "extractor/name_table.hpp" #include "guidance/constants.hpp" #include "guidance/intersection.hpp" #include "util/assert.hpp" #include "util/coordinate_calculation.hpp" #include "util/guidance/name_announcements.hpp" -#include "util/name_table.hpp" #include "util/node_based_graph.hpp" #include @@ -38,7 +38,7 @@ class IntersectionHandler const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table); virtual ~IntersectionHandler() = default; @@ -59,7 +59,7 @@ class IntersectionHandler const extractor::RestrictionMap &node_restriction_map; const std::unordered_set &barrier_nodes; const extractor::TurnLanesIndexedArray &turn_lanes_data; - const util::NameTable &name_table; + const extractor::NameTable &name_table; const extractor::SuffixTable &street_name_suffix_table; const extractor::intersection::NodeBasedGraphWalker graph_walker; // for skipping traffic signal, distances etc. diff --git a/include/guidance/is_through_street.hpp b/include/guidance/is_through_street.hpp index b91ff5956..456ac839e 100644 --- a/include/guidance/is_through_street.hpp +++ b/include/guidance/is_through_street.hpp @@ -7,6 +7,9 @@ #include "extractor/node_data_container.hpp" #include "extractor/suffix_table.hpp" +#include "util/node_based_graph.hpp" +#include "util/bearing.hpp" + #include "util/guidance/name_announcements.hpp" namespace osrm @@ -19,7 +22,7 @@ inline bool isThroughStreet(const std::size_t index, const IntersectionType &intersection, const util::NodeBasedDynamicGraph &node_based_graph, const extractor::EdgeBasedNodeDataContainer &node_data_container, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table) { using osrm::util::angularDeviation; diff --git a/include/guidance/motorway_handler.hpp b/include/guidance/motorway_handler.hpp index 110f3716e..5ecac8170 100644 --- a/include/guidance/motorway_handler.hpp +++ b/include/guidance/motorway_handler.hpp @@ -1,12 +1,13 @@ #ifndef OSRM_GUIDANCE_MOTORWAY_HANDLER_HPP_ #define OSRM_GUIDANCE_MOTORWAY_HANDLER_HPP_ +#include "extractor/name_table.hpp" + #include "guidance/intersection.hpp" #include "guidance/intersection_handler.hpp" #include "guidance/is_through_street.hpp" #include "util/attributes.hpp" -#include "util/name_table.hpp" #include "util/node_based_graph.hpp" #include @@ -27,7 +28,7 @@ class MotorwayHandler : public IntersectionHandler const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table); ~MotorwayHandler() override final = default; diff --git a/include/guidance/roundabout_handler.hpp b/include/guidance/roundabout_handler.hpp index 7ba90d7d5..fc0036e7c 100644 --- a/include/guidance/roundabout_handler.hpp +++ b/include/guidance/roundabout_handler.hpp @@ -4,12 +4,13 @@ #include "extractor/compressed_edge_container.hpp" #include "extractor/intersection/coordinate_extractor.hpp" #include "extractor/query_node.hpp" +#include "extractor/name_table.hpp" + #include "guidance/intersection.hpp" #include "guidance/intersection_handler.hpp" #include "guidance/is_through_street.hpp" #include "guidance/roundabout_type.hpp" -#include "util/name_table.hpp" #include "util/node_based_graph.hpp" #include "util/typedefs.hpp" @@ -44,7 +45,7 @@ class RoundaboutHandler : public IntersectionHandler const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table); ~RoundaboutHandler() override final = default; diff --git a/include/guidance/segregated_intersection_classification.hpp b/include/guidance/segregated_intersection_classification.hpp index 25d39d929..363192921 100644 --- a/include/guidance/segregated_intersection_classification.hpp +++ b/include/guidance/segregated_intersection_classification.hpp @@ -1,14 +1,11 @@ +#include "extractor/name_table.hpp" + #include "util/typedefs.hpp" #include namespace osrm { -namespace util -{ -class NameTable; -} - namespace extractor { class NodeBasedGraphFactory; @@ -22,6 +19,6 @@ namespace guidance // - staggered intersections (X-cross) // - square/circle intersections std::unordered_set findSegregatedNodes(const extractor::NodeBasedGraphFactory &factory, - const util::NameTable &names); + const extractor::NameTable &names); } } diff --git a/include/guidance/sliproad_handler.hpp b/include/guidance/sliproad_handler.hpp index b51442db2..bed9d7424 100644 --- a/include/guidance/sliproad_handler.hpp +++ b/include/guidance/sliproad_handler.hpp @@ -1,11 +1,12 @@ #ifndef OSRM_GUIDANCE_SLIPROAD_HANDLER_HPP_ #define OSRM_GUIDANCE_SLIPROAD_HANDLER_HPP_ +#include "extractor/name_table.hpp" + #include "guidance/intersection.hpp" #include "guidance/intersection_handler.hpp" #include "guidance/is_through_street.hpp" -#include "util/name_table.hpp" #include "util/node_based_graph.hpp" #include @@ -28,7 +29,7 @@ class SliproadHandler final : public IntersectionHandler const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table); ~SliproadHandler() override final = default; diff --git a/include/guidance/statistics_handler.hpp b/include/guidance/statistics_handler.hpp index c550a08c7..6145228ec 100644 --- a/include/guidance/statistics_handler.hpp +++ b/include/guidance/statistics_handler.hpp @@ -32,7 +32,7 @@ class StatisticsHandler final : public IntersectionHandler const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table) : IntersectionHandler(node_based_graph, node_data_container, diff --git a/include/guidance/suppress_mode_handler.hpp b/include/guidance/suppress_mode_handler.hpp index e047a5828..bebc79401 100644 --- a/include/guidance/suppress_mode_handler.hpp +++ b/include/guidance/suppress_mode_handler.hpp @@ -1,8 +1,11 @@ #ifndef OSRM_GUIDANCE_SUPPRESS_MODE_HANDLER_HPP_ #define OSRM_GUIDANCE_SUPPRESS_MODE_HANDLER_HPP_ +#include "extractor/name_table.hpp" + #include "guidance/intersection.hpp" #include "guidance/intersection_handler.hpp" + #include "util/node_based_graph.hpp" namespace osrm @@ -23,7 +26,7 @@ class SuppressModeHandler final : public IntersectionHandler const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table); ~SuppressModeHandler() override final = default; diff --git a/include/guidance/turn_analysis.hpp b/include/guidance/turn_analysis.hpp index 4fdf67c4d..957f05f15 100644 --- a/include/guidance/turn_analysis.hpp +++ b/include/guidance/turn_analysis.hpp @@ -5,6 +5,8 @@ #include "extractor/intersection/intersection_view.hpp" #include "extractor/restriction_index.hpp" #include "extractor/suffix_table.hpp" +#include "extractor/name_table.hpp" + #include "guidance/driveway_handler.hpp" #include "guidance/intersection.hpp" #include "guidance/motorway_handler.hpp" @@ -16,7 +18,6 @@ #include "guidance/turn_handler.hpp" #include "util/attributes.hpp" -#include "util/name_table.hpp" #include "util/node_based_graph.hpp" #include @@ -43,7 +44,7 @@ class TurnAnalysis const extractor::RestrictionMap &restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table); /* Full Analysis Process for a single node/edge combination. Use with caution, as the process is diff --git a/include/guidance/turn_handler.hpp b/include/guidance/turn_handler.hpp index 6c8ef9977..eac7f00d1 100644 --- a/include/guidance/turn_handler.hpp +++ b/include/guidance/turn_handler.hpp @@ -2,12 +2,13 @@ #define OSRM_GUIDANCE_TURN_HANDLER_HPP_ #include "extractor/query_node.hpp" +#include "extractor/name_table.hpp" + #include "guidance/intersection.hpp" #include "guidance/intersection_handler.hpp" #include "guidance/is_through_street.hpp" #include "util/attributes.hpp" -#include "util/name_table.hpp" #include "util/node_based_graph.hpp" #include @@ -32,7 +33,7 @@ class TurnHandler : public IntersectionHandler const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table); ~TurnHandler() override final = default; diff --git a/include/guidance/turn_lane_handler.hpp b/include/guidance/turn_lane_handler.hpp index 1ec860013..ad9a759a8 100644 --- a/include/guidance/turn_lane_handler.hpp +++ b/include/guidance/turn_lane_handler.hpp @@ -1,15 +1,16 @@ #ifndef OSRM_GUIDANCE_TURN_LANE_HANDLER_HPP_ #define OSRM_GUIDANCE_TURN_LANE_HANDLER_HPP_ +#include "extractor/name_table.hpp" #include "extractor/query_node.hpp" #include "extractor/turn_lane_types.hpp" + #include "guidance/intersection.hpp" #include "guidance/turn_analysis.hpp" #include "guidance/turn_lane_data.hpp" #include "util/attributes.hpp" #include "util/guidance/turn_lanes.hpp" -#include "util/name_table.hpp" #include "util/node_based_graph.hpp" #include "util/typedefs.hpp" diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp index 5788c9996..eaec021f9 100644 --- a/include/storage/shared_datatype.hpp +++ b/include/storage/shared_datatype.hpp @@ -20,7 +20,9 @@ namespace storage // Added at the start and end of each block as sanity check const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'}; -const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA", +const constexpr char *block_id_to_name[] = {"IGNORE_BLOCK", + "NAME_BLOCKS", + "NAME_VALUES", "EDGE_BASED_NODE_DATA", "ANNOTATION_DATA", "CH_GRAPH_NODE_LIST", @@ -92,15 +94,16 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA", "MLD_GRAPH_EDGE_LIST", "MLD_GRAPH_NODE_TO_OFFSET", "MANEUVER_OVERRIDES", - "MANEUVER_OVERRIDE_NODE_SEQUENCES", - "IGNORE_BLOCK"}; + "MANEUVER_OVERRIDE_NODE_SEQUENCES"}; class DataLayout { public: enum BlockID { - NAME_CHAR_DATA = 0, + IGNORE_BLOCK = 0, + NAME_BLOCKS, + NAME_VALUES, EDGE_BASED_NODE_DATA_LIST, ANNOTATION_DATA_LIST, CH_GRAPH_NODE_LIST, @@ -173,7 +176,6 @@ class DataLayout MLD_GRAPH_NODE_TO_OFFSET, MANEUVER_OVERRIDES, MANEUVER_OVERRIDE_NODE_SEQUENCES, - IGNORE_BLOCK, NUM_BLOCKS }; diff --git a/include/util/guidance/name_announcements.hpp b/include/util/guidance/name_announcements.hpp index f8e6761ff..2a3953ead 100644 --- a/include/util/guidance/name_announcements.hpp +++ b/include/util/guidance/name_announcements.hpp @@ -3,9 +3,10 @@ /* A set of tools required for guidance in both pre and post-processing */ +#include "extractor/name_table.hpp" #include "extractor/suffix_table.hpp" + #include "util/attributes.hpp" -#include "util/name_table.hpp" #include "util/typedefs.hpp" #include @@ -216,7 +217,7 @@ inline bool requiresNameAnnounced(const std::string &from_name, inline bool requiresNameAnnounced(const NameID from_name_id, const NameID to_name_id, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &suffix_table) { if (from_name_id == to_name_id) diff --git a/include/util/indexed_data.hpp b/include/util/indexed_data.hpp index 64ba1a4f3..b652e0be4 100644 --- a/include/util/indexed_data.hpp +++ b/include/util/indexed_data.hpp @@ -1,12 +1,14 @@ #ifndef OSRM_INDEXED_DATA_HPP #define OSRM_INDEXED_DATA_HPP -#include "storage/io.hpp" +#include "storage/tar_fwd.hpp" #include "util/exception.hpp" #include "util/string_view.hpp" +#include "util/vector_view.hpp" #include +#include #include #include @@ -18,6 +20,23 @@ namespace osrm { namespace util { +namespace detail +{ +template struct IndexedDataImpl; +} + +namespace serialization +{ +template +inline void read(storage::tar::FileReader &reader, + const std::string &name, + detail::IndexedDataImpl &index_data); + +template +inline void write(storage::tar::FileWriter &writer, + const std::string &name, + const detail::IndexedDataImpl &index_data); +} template struct VariableGroupBlock { @@ -85,11 +104,11 @@ template struct VariableGroupBlock /// of prefix length. sum(descriptor) equals to the block /// prefix length. /// Returns the block prefix length. - template - Offset WriteBlockReference(storage::io::FileWriter &out, - Offset data_offset, - OffsetIterator first, - OffsetIterator last) const + template + OutIter WriteBlockReference(OffsetIterator first, + OffsetIterator last, + Offset &data_offset, + OutIter out) const { BOOST_ASSERT(data_offset <= std::numeric_limits::max()); @@ -106,9 +125,9 @@ template struct VariableGroupBlock prefix_length += byte_length; } - out.WriteFrom(refernce); - - return prefix_length; + data_offset += prefix_length; + *out++ = refernce; + return out; } /// Write a block prefix that is an array of variable encoded data lengths: @@ -118,9 +137,8 @@ template struct VariableGroupBlock /// 65536..16777215 is 3 bytes. /// [first..last] is an inclusive range of block data. /// The length of the last item in the block is not stored. - template - void - WriteBlockPrefix(storage::io::FileWriter &out, OffsetIterator first, OffsetIterator last) const + template + OutByteIter WriteBlockPrefix(OffsetIterator first, OffsetIterator last, OutByteIter out) const { for (OffsetIterator curr = first, next = std::next(first); curr != last; ++curr, ++next) { @@ -131,8 +149,9 @@ template struct VariableGroupBlock // Here, we're only writing a few bytes from the 4-byte std::uint32_t, // so we need to cast to (char *) - out.WriteFrom((const char *)&data_length, byte_length); + out = std::copy_n((const char *)&data_length, byte_length, out); } + return out; } /// Advances the range to an item stored in the referenced block. @@ -178,36 +197,39 @@ template struct FixedGroupBlock /// Write a block reference {offset}, where offset is a global block offset /// Returns the fixed block prefix length. - template - Offset WriteBlockReference(storage::io::FileWriter &out, - Offset data_offset, - OffsetIterator, - OffsetIterator) const + template + OutIterator + WriteBlockReference(OffsetIterator, OffsetIterator, Offset &data_offset, OutIterator out) const { BOOST_ASSERT(data_offset <= std::numeric_limits::max()); BlockReference refernce{static_cast(data_offset)}; - out.WriteFrom(refernce); + data_offset += BLOCK_SIZE; + *out++ = refernce; - return BLOCK_SIZE; + return out; } /// Write a fixed length block prefix. - template - void - WriteBlockPrefix(storage::io::FileWriter &out, OffsetIterator first, OffsetIterator last) const + template + OutByteIter WriteBlockPrefix(OffsetIterator first, OffsetIterator last, OutByteIter out) const { - std::uint32_t index = 0; - std::array block_prefix; + constexpr std::size_t MAX_LENGTH = std::numeric_limits>::max(); + + auto index = 0; + std::array prefix; + for (OffsetIterator curr = first, next = std::next(first); curr != last; ++curr, ++next) { const std::uint32_t data_length = *next - *curr; - if (data_length >= 0x100) - throw util::exception(boost::format("too large data length %1%") % data_length); + if (data_length > MAX_LENGTH) + throw util::exception(boost::format("too large data length %1% > %2%") % data_length % MAX_LENGTH); - block_prefix[index++] = static_cast(data_length); + prefix[index++] = data_length; } - out.WriteFrom(block_prefix.data(), block_prefix.size()); + + out = std::copy_n((const char *)prefix.data(), sizeof(ValueType)*BLOCK_SIZE, out); + return out; } /// Advances the range to an item stored in the referenced block. @@ -233,28 +255,31 @@ template struct FixedGroupBlock } }; -template struct IndexedData +namespace detail { - static constexpr std::uint32_t BLOCK_SIZE = GroupBlock::BLOCK_SIZE; +template struct IndexedDataImpl +{ + static constexpr std::uint32_t BLOCK_SIZE = GroupBlockPolicy::BLOCK_SIZE; using BlocksNumberType = std::uint32_t; using DataSizeType = std::uint64_t; - using BlockReference = typename GroupBlock::BlockReference; - using ResultType = typename GroupBlock::ResultType; - using ValueType = typename GroupBlock::ValueType; + using BlockReference = typename GroupBlockPolicy::BlockReference; + using ResultType = typename GroupBlockPolicy::ResultType; + using ValueType = typename GroupBlockPolicy::ValueType; static_assert(sizeof(ValueType) == 1, "data basic type must char"); - IndexedData() : blocks_number{0}, block_references{nullptr}, begin{nullptr}, end{nullptr} {} + IndexedDataImpl() = default; + IndexedDataImpl(util::vector_view blocks_, util::vector_view values_) + : blocks(std::move(blocks_)), values(std::move(values_)) + { + } - bool empty() const { return blocks_number == 0; } + bool empty() const { return blocks.empty(); } template - void write(storage::io::FileWriter &out, - OffsetIterator first, - OffsetIterator last, - DataIterator data) const + IndexedDataImpl(OffsetIterator first, OffsetIterator last, DataIterator data) { static_assert(sizeof(typename DataIterator::value_type) == 1, "data basic type must char"); @@ -268,69 +293,37 @@ template struct IndexedData const BlocksNumberType number_of_blocks = number_of_elements == 0 ? 0 : 1 + (std::distance(first, sentinel) - 1) / (BLOCK_SIZE + 1); - out.WriteFrom(number_of_blocks); + blocks.resize(number_of_blocks); // Write block references and compute the total data size that includes prefix and data - const GroupBlock block; + const GroupBlockPolicy block; + + auto block_iter = blocks.begin(); DataSizeType data_size = 0; for (OffsetIterator curr = first, next = first; next != sentinel; curr = next) { std::advance(next, std::min(BLOCK_SIZE, std::distance(next, sentinel))); - data_size += block.WriteBlockReference(out, data_size, curr, next); + block_iter = block.WriteBlockReference(curr, next, data_size, block_iter); std::advance(next, std::min(1, std::distance(next, sentinel))); data_size += *next - *curr; } - // Write the total data size - out.WriteFrom(data_size); - + values.resize(data_size); + auto values_byte_iter = reinterpret_cast(values.data()); // Write data blocks that are (prefix, data) for (OffsetIterator curr = first, next = first; next != sentinel; curr = next) { std::advance(next, std::min(BLOCK_SIZE, std::distance(next, sentinel))); - block.WriteBlockPrefix(out, curr, next); + values_byte_iter = block.WriteBlockPrefix(curr, next, values_byte_iter); std::advance(next, std::min(1, std::distance(next, sentinel))); - std::for_each( - data + *curr, data + *next, [&out](const auto &element) { out.WriteFrom(element); }); + + auto to_bytes = [&](const auto &data) { + values_byte_iter = std::copy_n(&data, sizeof(ValueType), values_byte_iter); + }; + std::copy(data + *curr, data + *next, boost::make_function_output_iterator(to_bytes)); } } - /// Set internal pointers from the buffer [first, last). - /// Data buffer pointed by ptr must exists during IndexedData life-time. - /// No ownership is transferred. - void reset(const ValueType *first, const ValueType *last) - { - // Read blocks number - if (first + sizeof(BlocksNumberType) > last) - throw util::exception("incorrect memory block"); - - blocks_number = *reinterpret_cast(first); - first += sizeof(BlocksNumberType); - - // Get block references pointer - if (first + sizeof(BlockReference) * blocks_number > last) - throw util::exception("incorrect memory block"); - - block_references = reinterpret_cast(first); - first += sizeof(BlockReference) * blocks_number; - - // Read total data size - if (first + sizeof(DataSizeType) > last) - throw util::exception("incorrect memory block"); - - auto data_size = *reinterpret_cast(first); - first += sizeof(DataSizeType); - - // Get data blocks begin and end iterators - begin = reinterpret_cast(first); - first += sizeof(ValueType) * data_size; - - if (first > last) - throw util::exception("incorrect memory block"); - - end = reinterpret_cast(first); - } - // Return value at the given index ResultType at(std::uint32_t index) const { @@ -338,20 +331,29 @@ template struct IndexedData const BlocksNumberType block_idx = index / (BLOCK_SIZE + 1); const std::uint32_t internal_idx = index % (BLOCK_SIZE + 1); - if (block_idx >= blocks_number) + if (block_idx >= blocks.size()) return ResultType(); // Get block first and last iterators - auto first = begin + block_references[block_idx].offset; - auto last = - block_idx + 1 == blocks_number ? end : begin + block_references[block_idx + 1].offset; + auto first = values.begin() + blocks[block_idx].offset; + auto last = block_idx + 1 == blocks.size() ? values.end() + : values.begin() + blocks[block_idx + 1].offset; - const GroupBlock block; - block.ReadRefrencedBlock(block_references[block_idx], internal_idx, first, last); + const GroupBlockPolicy block; + block.ReadRefrencedBlock(blocks[block_idx], internal_idx, first, last); - return adapt(first, last); + return adapt(&*first, &*last); } + friend void serialization::read(storage::tar::FileReader &reader, + const std::string &name, + IndexedDataImpl &index_data); + + friend void + serialization::write(storage::tar::FileWriter &writer, + const std::string &name, + const IndexedDataImpl &index_data); + private: template typename std::enable_if::value, T>::type @@ -367,10 +369,16 @@ template struct IndexedData return ResultType(first, std::distance(first, last)); } - BlocksNumberType blocks_number; - const BlockReference *block_references; - const ValueType *begin, *end; + template using Vector = util::ViewOrVector; + Vector blocks; + Vector values; }; } + +template +using IndexedData = detail::IndexedDataImpl; +template +using IndexedDataView = detail::IndexedDataImpl; +} } #endif // OSRM_INDEXED_DATA_HPP diff --git a/include/util/name_table.hpp b/include/util/name_table.hpp deleted file mode 100644 index 2e76be928..000000000 --- a/include/util/name_table.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef OSRM_UTIL_NAME_TABLE_HPP -#define OSRM_UTIL_NAME_TABLE_HPP - -#include "util/indexed_data.hpp" -#include "util/string_view.hpp" -#include "util/typedefs.hpp" - -#include - -namespace osrm -{ -namespace util -{ - -// While this could, theoretically, hold any names in the fitting format, -// the NameTable allows access to a part of the Datafacade to allow -// processing based on name indices. -class NameTable -{ - public: - using IndexedData = util::IndexedData>; - using ResultType = IndexedData::ResultType; - using ValueType = IndexedData::ValueType; - - NameTable() {} - - // Read filename and store own data in m_buffer - NameTable(const std::string &filename); - - // Keep pointers only in m_name_table and don't own data in m_buffer - void reset(ValueType *begin, ValueType *end); - - // This class provides a limited view over all the string data we serialize out. - // The following functions are a subset of what is available. - // See the data facades for they provide full access to this serialized string data. - util::StringView GetNameForID(const NameID id) const; - util::StringView GetDestinationsForID(const NameID id) const; - util::StringView GetExitsForID(const NameID id) const; - util::StringView GetRefForID(const NameID id) const; - util::StringView GetPronunciationForID(const NameID id) const; - - private: - using BufferType = std::unique_ptr>; - - BufferType m_buffer; - IndexedData m_name_table; -}; -} // namespace util -} // namespace osrm - -#endif // OSRM_UTIL_NAME_TABLE_HPP diff --git a/include/util/serialization.hpp b/include/util/serialization.hpp index 62a3986c9..7210ce63a 100644 --- a/include/util/serialization.hpp +++ b/include/util/serialization.hpp @@ -5,6 +5,7 @@ #include "util/packed_vector.hpp" #include "util/range_table.hpp" #include "util/static_graph.hpp" +#include "util/indexed_data.hpp" #include "storage/io.hpp" #include "storage/serialization.hpp" @@ -168,6 +169,22 @@ inline void write(storage::tar::FileWriter &writer, writer.WriteStreaming::Edge>( name + "/edge_list", graph.edge_list.begin(), graph.number_of_edges); } + +template +inline void +read(storage::tar::FileReader &reader, const std::string &name, detail::IndexedDataImpl &index_data) +{ + storage::serialization::read(reader, name + "/blocks", index_data.blocks); + storage::serialization::read(reader, name + "/values", index_data.values); +} + +template +inline void write(storage::tar::FileWriter &writer, + const std::string &name, const detail::IndexedDataImpl &index_data) +{ + storage::serialization::write(writer, name + "/blocks", index_data.blocks); + storage::serialization::write(writer, name + "/values", index_data.values); +} } } } diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index e05a699a5..ddebc6875 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -69,7 +69,7 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory( const std::unordered_set &barrier_nodes, const std::unordered_set &traffic_lights, const std::vector &coordinates, - const util::NameTable &name_table, + const NameTable &name_table, const std::unordered_set &segregated_edges, const extractor::LaneDescriptionMap &lane_description_map) : m_edge_based_node_container(node_data_container), m_connectivity_checksum(0), diff --git a/src/extractor/extraction_containers.cpp b/src/extractor/extraction_containers.cpp index 6336d9890..3c4878d47 100644 --- a/src/extractor/extraction_containers.cpp +++ b/src/extractor/extraction_containers.cpp @@ -3,6 +3,8 @@ #include "extractor/extraction_way.hpp" #include "extractor/restriction.hpp" #include "extractor/serialization.hpp" +#include "extractor/name_table.hpp" +#include "extractor/files.hpp" #include "util/coordinate_calculation.hpp" @@ -10,7 +12,6 @@ #include "util/exception_utils.hpp" #include "util/fingerprint.hpp" #include "util/log.hpp" -#include "util/name_table.hpp" #include "util/timing_util.hpp" #include "storage/io.hpp" @@ -163,10 +164,8 @@ void ExtractionContainers::WriteCharData(const std::string &file_name) util::UnbufferedLog log; log << "writing street name index ... "; TIMER_START(write_index); - storage::io::FileWriter file(file_name, storage::io::FileWriter::GenerateFingerprint); - const util::NameTable::IndexedData indexed_data; - indexed_data.write(file, name_offsets.begin(), name_offsets.end(), name_char_data.begin()); + files::writeNames(file_name, NameTable {NameTable::IndexedData (name_offsets.begin(), name_offsets.end(), name_char_data.begin())}); TIMER_STOP(write_index); log << "ok, after " << TIMER_SEC(write_index) << "s"; diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index fe0d02e7f..07b80a8ca 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -13,6 +13,7 @@ #include "extractor/restriction_filter.hpp" #include "extractor/restriction_parser.hpp" #include "extractor/scripting_environment.hpp" +#include "extractor/name_table.hpp" #include "guidance/files.hpp" #include "guidance/guidance_processing.hpp" @@ -25,7 +26,6 @@ #include "util/exception_utils.hpp" #include "util/integer_range.hpp" #include "util/log.hpp" -#include "util/name_table.hpp" #include "util/range_table.hpp" #include "util/timing_util.hpp" @@ -230,11 +230,13 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) conditional_turn_restrictions, unresolved_maneuver_overrides); + NameTable name_table; + files::readNames(config.GetPath(".osrm.names"), name_table); + util::Log() << "Find segregated edges in node-based graph ..." << std::flush; TIMER_START(segregated); - util::NameTable names(config.GetPath(".osrm.names").string()); - auto segregated_edges = guidance::findSegregatedNodes(node_based_graph_factory, names); + auto segregated_edges = guidance::findSegregatedNodes(node_based_graph_factory, name_table); TIMER_STOP(segregated); util::Log() << "ok, after " << TIMER_SEC(segregated) << "s"; @@ -283,8 +285,6 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) const auto number_of_node_based_nodes = node_based_graph.GetNumberOfNodes(); - const util::NameTable name_table(config.GetPath(".osrm.names").string()); - const auto number_of_edge_based_nodes = BuildEdgeExpandedGraph(node_based_graph, coordinates, @@ -713,7 +713,7 @@ EdgeID Extractor::BuildEdgeExpandedGraph( const std::vector &turn_restrictions, const std::vector &conditional_turn_restrictions, const std::unordered_set &segregated_edges, - const util::NameTable &name_table, + const NameTable &name_table, const std::vector &maneuver_overrides, const LaneDescriptionMap &turn_lane_map, // for calculating turn penalties @@ -884,7 +884,7 @@ void Extractor::ProcessGuidanceTurns( const std::unordered_set &barrier_nodes, const std::vector &turn_restrictions, const std::vector &conditional_turn_restrictions, - const util::NameTable &name_table, + const NameTable &name_table, LaneDescriptionMap lane_description_map, ScriptingEnvironment &scripting_environment) { diff --git a/src/extractor/intersection/have_identical_names.cpp b/src/extractor/intersection/have_identical_names.cpp index 7cebd6eef..0dffc12fe 100644 --- a/src/extractor/intersection/have_identical_names.cpp +++ b/src/extractor/intersection/have_identical_names.cpp @@ -1,3 +1,5 @@ +#include "extractor/intersection/have_identical_names.hpp" + #include "util/guidance/name_announcements.hpp" namespace osrm @@ -12,7 +14,7 @@ namespace intersection // rhs->lhs) bool HaveIdenticalNames(const NameID lhs, const NameID rhs, - const util::NameTable &name_table, + const NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table) { const auto non_empty = (lhs != EMPTY_NAMEID) && (rhs != EMPTY_NAMEID); diff --git a/src/extractor/intersection/mergable_road_detector.cpp b/src/extractor/intersection/mergable_road_detector.cpp index 0c74ff4b1..e14c147eb 100644 --- a/src/extractor/intersection/mergable_road_detector.cpp +++ b/src/extractor/intersection/mergable_road_detector.cpp @@ -3,12 +3,12 @@ #include "extractor/intersection/node_based_graph_walker.hpp" #include "extractor/query_node.hpp" #include "extractor/suffix_table.hpp" +#include "extractor/name_table.hpp" #include "guidance/constants.hpp" #include "util/bearing.hpp" #include "util/coordinate_calculation.hpp" #include "util/guidance/name_announcements.hpp" -#include "util/name_table.hpp" using osrm::util::angularDeviation; @@ -27,7 +27,7 @@ namespace inline auto makeCheckRoadForName(const NameID name_id, const util::NodeBasedDynamicGraph &node_based_graph, const EdgeBasedNodeDataContainer &node_data_container, - const util::NameTable &name_table, + const NameTable &name_table, const SuffixTable &suffix_table) { return [name_id, &node_based_graph, &node_data_container, &name_table, &suffix_table]( @@ -59,7 +59,7 @@ MergableRoadDetector::MergableRoadDetector( const RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const NameTable &name_table, const SuffixTable &street_name_suffix_table) : node_based_graph(node_based_graph), node_data_container(node_data_container), node_coordinates(node_coordinates), compressed_geometries(compressed_geometries), diff --git a/src/guidance/driveway_handler.cpp b/src/guidance/driveway_handler.cpp index 3c37ae262..81984b474 100644 --- a/src/guidance/driveway_handler.cpp +++ b/src/guidance/driveway_handler.cpp @@ -17,7 +17,7 @@ DrivewayHandler::DrivewayHandler(const util::NodeBasedDynamicGraph &node_based_g const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table) : IntersectionHandler(node_based_graph, node_data_container, diff --git a/src/guidance/guidance_processing.cpp b/src/guidance/guidance_processing.cpp index 8cf52ac79..f2f014122 100644 --- a/src/guidance/guidance_processing.cpp +++ b/src/guidance/guidance_processing.cpp @@ -24,7 +24,7 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph, const std::unordered_set &barrier_nodes, const extractor::RestrictionMap &node_restriction_map, const extractor::WayRestrictionMap &way_restriction_map, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &suffix_table, const extractor::TurnLanesIndexedArray &turn_lanes_data, extractor::LaneDescriptionMap &lane_description_map, diff --git a/src/guidance/intersection_handler.cpp b/src/guidance/intersection_handler.cpp index eb9252e46..0dedc8f9a 100644 --- a/src/guidance/intersection_handler.cpp +++ b/src/guidance/intersection_handler.cpp @@ -52,7 +52,7 @@ IntersectionHandler::IntersectionHandler( const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table) : node_based_graph(node_based_graph), node_data_container(node_data_container), node_coordinates(node_coordinates), compressed_geometries(compressed_geometries), diff --git a/src/guidance/motorway_handler.cpp b/src/guidance/motorway_handler.cpp index 1e00b8a03..116369d10 100644 --- a/src/guidance/motorway_handler.cpp +++ b/src/guidance/motorway_handler.cpp @@ -49,7 +49,7 @@ MotorwayHandler::MotorwayHandler(const util::NodeBasedDynamicGraph &node_based_g const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table) : IntersectionHandler(node_based_graph, node_data_container, diff --git a/src/guidance/roundabout_handler.cpp b/src/guidance/roundabout_handler.cpp index 577dbf61e..a375c2ff8 100644 --- a/src/guidance/roundabout_handler.cpp +++ b/src/guidance/roundabout_handler.cpp @@ -30,7 +30,7 @@ RoundaboutHandler::RoundaboutHandler( const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table) : IntersectionHandler(node_based_graph, node_data_container, diff --git a/src/guidance/segregated_intersection_classification.cpp b/src/guidance/segregated_intersection_classification.cpp index 08e8f97fc..d8aaadafd 100644 --- a/src/guidance/segregated_intersection_classification.cpp +++ b/src/guidance/segregated_intersection_classification.cpp @@ -4,7 +4,6 @@ #include "guidance/turn_instruction.hpp" #include "util/coordinate_calculation.hpp" -#include "util/name_table.hpp" #include using osrm::guidance::getTurnDirection; @@ -42,7 +41,7 @@ struct EdgeInfo }; std::unordered_set findSegregatedNodes(const extractor::NodeBasedGraphFactory &factory, - const util::NameTable &names) + const extractor::NameTable &names) { auto const &graph = factory.GetGraph(); auto const &annotation = factory.GetAnnotationData(); diff --git a/src/guidance/sliproad_handler.cpp b/src/guidance/sliproad_handler.cpp index 493e9d7c2..08f8febf2 100644 --- a/src/guidance/sliproad_handler.cpp +++ b/src/guidance/sliproad_handler.cpp @@ -27,7 +27,7 @@ SliproadHandler::SliproadHandler(const util::NodeBasedDynamicGraph &node_based_g const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table) : IntersectionHandler(node_based_graph, node_data_container, diff --git a/src/guidance/suppress_mode_handler.cpp b/src/guidance/suppress_mode_handler.cpp index ee842f0c5..c50579f22 100644 --- a/src/guidance/suppress_mode_handler.cpp +++ b/src/guidance/suppress_mode_handler.cpp @@ -17,7 +17,7 @@ SuppressModeHandler::SuppressModeHandler( const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table) : IntersectionHandler(node_based_graph, node_data_container, diff --git a/src/guidance/turn_analysis.cpp b/src/guidance/turn_analysis.cpp index 4e56fed06..0c7463bfe 100644 --- a/src/guidance/turn_analysis.cpp +++ b/src/guidance/turn_analysis.cpp @@ -26,7 +26,7 @@ TurnAnalysis::TurnAnalysis(const util::NodeBasedDynamicGraph &node_based_graph, const extractor::RestrictionMap &restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table) : node_based_graph(node_based_graph), roundabout_handler(node_based_graph, node_data_container, diff --git a/src/guidance/turn_handler.cpp b/src/guidance/turn_handler.cpp index 579cd4364..d7644bf7d 100644 --- a/src/guidance/turn_handler.cpp +++ b/src/guidance/turn_handler.cpp @@ -115,7 +115,7 @@ TurnHandler::TurnHandler(const util::NodeBasedDynamicGraph &node_based_graph, const extractor::RestrictionMap &node_restriction_map, const std::unordered_set &barrier_nodes, const extractor::TurnLanesIndexedArray &turn_lanes_data, - const util::NameTable &name_table, + const extractor::NameTable &name_table, const extractor::SuffixTable &street_name_suffix_table) : IntersectionHandler(node_based_graph, node_data_container, diff --git a/src/osrm/osrm.cpp b/src/osrm/osrm.cpp index 0c49502c6..26f4187b3 100644 --- a/src/osrm/osrm.cpp +++ b/src/osrm/osrm.cpp @@ -35,7 +35,7 @@ OSRM::OSRM(engine::EngineConfig &config) auto mem = storage::makeSharedMemory(barrier.data().region); auto layout = reinterpret_cast(mem->Ptr()); - if (layout->GetBlockSize(storage::DataLayout::NAME_CHAR_DATA) == 0) + if (layout->GetBlockSize(storage::DataLayout::NAME_VALUES) == 0) throw util::exception( "No name data loaded, cannot continue. Have you run osrm-datastore to load data?"); } diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index e025e2cf0..8a8f5050d 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -18,6 +18,7 @@ #include "extractor/edge_based_node.hpp" #include "extractor/files.hpp" #include "extractor/maneuver_override.hpp" +#include "extractor/name_table.hpp" #include "extractor/packed_osm_ids.hpp" #include "extractor/profile_properties.hpp" #include "extractor/query_node.hpp" @@ -243,13 +244,6 @@ void Storage::PopulateLayout(DataLayout &layout) make_block(absolute_file_index_path.string().length() + 1)); } - { - util::Log() << "load names from: " << config.GetPath(".osrm.names"); - // number of entries in name index - io::FileReader name_file(config.GetPath(".osrm.names"), io::FileReader::VerifyFingerprint); - layout.SetBlock(DataLayout::NAME_CHAR_DATA, make_block(name_file.GetSize())); - } - // load rsearch tree size { io::FileReader tree_node_file(config.GetPath(".osrm.ramIndex"), @@ -345,6 +339,8 @@ void Storage::PopulateLayout(DataLayout &layout) {"/common/turn_data/lane_data_ids", DataLayout::LANE_DATA_ID}, {"/common/turn_data/entry_class_ids", DataLayout::ENTRY_CLASSID}, {"/common/turn_data/connectivity_checksum", DataLayout::IGNORE_BLOCK}, + {"/common/names/blocks", DataLayout::NAME_BLOCKS}, + {"/common/names/values", DataLayout::NAME_VALUES}, }; std::vector blocks; @@ -368,6 +364,7 @@ void Storage::PopulateLayout(DataLayout &layout) {REQUIRED, config.GetPath(".osrm.turn_weight_penalties")}, {REQUIRED, config.GetPath(".osrm.turn_duration_penalties")}, {REQUIRED, config.GetPath(".osrm.edges")}, + {REQUIRED, config.GetPath(".osrm.names")}, }; for (const auto &file : tar_files) @@ -425,14 +422,21 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // Name data { - io::FileReader name_file(config.GetPath(".osrm.names"), io::FileReader::VerifyFingerprint); - std::size_t name_file_size = name_file.GetSize(); + const auto name_blocks_ptr = + layout.GetBlockPtr( + memory_ptr, DataLayout::NAME_BLOCKS); + const auto name_values_ptr = + layout.GetBlockPtr( + memory_ptr, DataLayout::NAME_VALUES); - BOOST_ASSERT(name_file_size == layout.GetBlockSize(DataLayout::NAME_CHAR_DATA)); - const auto name_char_ptr = - layout.GetBlockPtr(memory_ptr, DataLayout::NAME_CHAR_DATA); + util::vector_view blocks( + name_blocks_ptr, layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS)); + util::vector_view values( + name_values_ptr, layout.GetBlockEntries(storage::DataLayout::NAME_VALUES)); - name_file.ReadInto(name_char_ptr, name_file_size); + extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)}; + extractor::NameTableView name_table{index_data_view}; + extractor::files::readNames(config.GetPath(".osrm.names"), name_table); } // Turn lane data @@ -624,7 +628,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) turn_duration_penalties_ptr, layout.GetBlockEntries(storage::DataLayout::TURN_WEIGHT_PENALTIES)); extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"), - turn_duration_penalties); + turn_duration_penalties); } // load turn duration penalties diff --git a/src/util/name_table.cpp b/src/util/name_table.cpp deleted file mode 100644 index f6ef670ba..000000000 --- a/src/util/name_table.cpp +++ /dev/null @@ -1,100 +0,0 @@ -#include "util/name_table.hpp" -#include "storage/io.hpp" -#include "util/log.hpp" - -namespace osrm -{ -namespace util -{ - -NameTable::NameTable(const std::string &file_name) -{ - using FileReader = storage::io::FileReader; - - FileReader name_stream_file_reader(file_name, FileReader::VerifyFingerprint); - const auto file_size = name_stream_file_reader.GetSize(); - - m_buffer = BufferType(static_cast(::operator new(file_size)), - [](void *ptr) { ::operator delete(ptr); }); - name_stream_file_reader.ReadInto(m_buffer.get(), file_size); - m_name_table.reset(m_buffer.get(), m_buffer.get() + file_size); - - if (m_name_table.empty()) - { - util::Log() << "list of street names is empty in construction of name table from: \"" - << file_name << "\""; - } -} - -void NameTable::reset(ValueType *begin, ValueType *end) -{ - m_buffer.reset(); - m_name_table.reset(begin, end); -} - -StringView NameTable::GetNameForID(const NameID id) const -{ - if (id == INVALID_NAMEID) - return {}; - - return m_name_table.at(id + 0); -} - -StringView NameTable::GetDestinationsForID(const NameID id) const -{ - if (id == INVALID_NAMEID) - return {}; - - return m_name_table.at(id + 1); -} - -StringView NameTable::GetExitsForID(const NameID id) const -{ - if (id == INVALID_NAMEID) - return {}; - - return m_name_table.at(id + 4); -} - -StringView NameTable::GetRefForID(const NameID id) const -{ - if (id == INVALID_NAMEID) - return {}; - - // Way string data is stored in blocks based on `id` as follows: - // - // | name | destination | pronunciation | ref | exits - // ^ ^ - // [range) - // ^ id + 3 - // - // `id + offset` gives us the range of chars. - // - // Offset 0 is name, 1 is destination, 2 is pronunciation, 3 is ref, 4 is exits - // See datafacades and extractor callbacks for details. - const constexpr auto OFFSET_REF = 3u; - return m_name_table.at(id + OFFSET_REF); -} - -StringView NameTable::GetPronunciationForID(const NameID id) const -{ - if (id == INVALID_NAMEID) - return {}; - - // Way string data is stored in blocks based on `id` as follows: - // - // | name | destination | pronunciation | ref | exits - // ^ ^ - // [range) - // ^ id + 2 - // - // `id + offset` gives us the range of chars. - // - // Offset 0 is name, 1 is destination, 2 is pronunciation, 3 is ref, 4 is exits - // See datafacades and extractor callbacks for details. - const constexpr auto OFFSET_PRONUNCIATION = 2u; - return m_name_table.at(id + OFFSET_PRONUNCIATION); -} - -} // namespace util -} // namespace osrm diff --git a/unit_tests/util/name_table.cpp b/unit_tests/extractor/name_table.cpp similarity index 83% rename from unit_tests/util/name_table.cpp rename to unit_tests/extractor/name_table.cpp index a7731dbac..8ed750c37 100644 --- a/unit_tests/util/name_table.cpp +++ b/unit_tests/extractor/name_table.cpp @@ -1,4 +1,4 @@ -#include "util/name_table.hpp" +#include "extractor/name_table.hpp" #include "common/temporary_file.hpp" #include "util/exception.hpp" @@ -12,14 +12,12 @@ #include #include -//#include - BOOST_AUTO_TEST_SUITE(name_table) using namespace osrm; -using namespace osrm::util; +using namespace osrm::extractor; -std::string PrapareNameTableData(std::vector &data, bool fill_all) +NameTable::IndexedData PrapareNameTableData(std::vector &data, bool fill_all) { NameTable::IndexedData indexed_data; std::vector name_char_data; @@ -60,19 +58,7 @@ std::string PrapareNameTableData(std::vector &data, bool fill_all) } name_offsets.push_back(name_char_data.size()); - TemporaryFile file; - { - storage::io::FileWriter writer(file.path, storage::io::FileWriter::HasNoFingerprint); - indexed_data.write( - writer, name_offsets.begin(), name_offsets.end(), name_char_data.begin()); - } - - storage::io::FileReader reader(file.path, storage::io::FileReader::HasNoFingerprint); - auto length = reader.GetSize(); - std::string str(length, '\0'); - reader.ReadInto(const_cast(str.data()), length); - - return str; + return NameTable::IndexedData(name_offsets.begin(), name_offsets.end(), name_char_data.begin()); } BOOST_AUTO_TEST_CASE(check_name_table_fill) @@ -83,9 +69,7 @@ BOOST_AUTO_TEST_CASE(check_name_table_fill) "X", "Y", "Z", "", "", "", "", "", "", "", "0", ""}; auto data = PrapareNameTableData(expected_names, true); - - NameTable name_table; - name_table.reset(&data[0], &data[data.size()]); + NameTable name_table {data}; for (std::size_t index = 0; index < expected_names.size(); ++index) { @@ -106,9 +90,7 @@ BOOST_AUTO_TEST_CASE(check_name_table_nofill) "X", "Y", "Z", "", "", "", "", "", "", "", "0", ""}; auto data = PrapareNameTableData(expected_names, false); - - NameTable name_table; - name_table.reset(&data[0], &data[data.size()]); + NameTable name_table {data}; // CALLGRIND_START_INSTRUMENTATION; for (std::size_t index = 0; index < expected_names.size(); ++index) diff --git a/unit_tests/util/indexed_data.cpp b/unit_tests/util/indexed_data.cpp index ad5878d3b..ec489550b 100644 --- a/unit_tests/util/indexed_data.cpp +++ b/unit_tests/util/indexed_data.cpp @@ -35,29 +35,7 @@ BOOST_AUTO_TEST_CASE(check_variable_group_block_bitops) template void test_rw(const Offsets &offsets, const Data &data) { - TemporaryFile file; - - IndexedData indexed_data; - - { - storage::io::FileWriter writer(file.path, storage::io::FileWriter::HasNoFingerprint); - indexed_data.write(writer, offsets.begin(), offsets.end(), data.begin()); - } - - storage::io::FileReader reader(file.path, storage::io::FileReader::HasNoFingerprint); - auto length = reader.GetSize(); - std::string str(length, '\0'); - reader.ReadInto(const_cast(str.data()), length); - -#if 0 - std::cout << "\n" << typeid(IndexedData).name() << "\nsaved size = " << str.size() << "\n"; - for (auto c : str) - std::cout << std::hex << std::setw(2) << std::setfill('0') - << (int)((unsigned char)c) << " "; - std::cout << std::dec << "\n"; -#endif - - indexed_data.reset(str.c_str(), str.c_str() + str.size()); + IndexedData indexed_data(offsets.begin(), offsets.end(), data.begin()); for (std::size_t index = 0; index < offsets.size() - 1; ++index) { @@ -151,56 +129,16 @@ BOOST_AUTO_TEST_CASE(check_max_size) name_offsets = {0, 256}; BOOST_CHECK_THROW(test_fixed(), osrm::util::exception); name_offsets = {0, 255}; - test_fixed(); -} - -BOOST_AUTO_TEST_CASE(check_corrupted_memory) -{ - std::vector buf; - - auto test_variable = [&buf]() { - IndexedData>> indexed_data; - indexed_data.reset(&buf[0], &buf[buf.size()]); - const auto result = indexed_data.at(0); - return std::string(reinterpret_cast(&result[0]), result.size()); - }; - - // Use LE internal representation - buf = {0, 42}; - BOOST_CHECK_THROW(test_variable(), osrm::util::exception); - - buf = {1, 0, 0, 0, 0}; - BOOST_CHECK_THROW(test_variable(), osrm::util::exception); - - buf = {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42}; - BOOST_CHECK_THROW(test_variable(), osrm::util::exception); - - buf = {1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 42}; - BOOST_CHECK_THROW(test_variable(), osrm::util::exception); - - buf = {1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0xF0, 0x9F, 0x90, 0xBC}; - BOOST_CHECK_EQUAL(test_variable(), "🐼"); + BOOST_CHECK_NO_THROW(test_fixed()); } BOOST_AUTO_TEST_CASE(check_string_view) { - TemporaryFile file; - std::string name_data = "hellostringview"; std::vector name_offsets = {0, 5, 11, 15}; - IndexedData> indexed_data; - { - storage::io::FileWriter writer(file.path, storage::io::FileWriter::HasNoFingerprint); - indexed_data.write(writer, name_offsets.begin(), name_offsets.end(), name_data.begin()); - } - - storage::io::FileReader reader(file.path, storage::io::FileReader::HasNoFingerprint); - auto length = reader.GetSize(); - std::string str(length, '\0'); - reader.ReadInto(const_cast(str.data()), length); - - indexed_data.reset(str.c_str(), str.c_str() + str.size()); + IndexedData> indexed_data( + name_offsets.begin(), name_offsets.end(), name_data.begin()); BOOST_CHECK_EQUAL(indexed_data.at(0), "hello"); BOOST_CHECK_EQUAL(indexed_data.at(1), "string"); diff --git a/unit_tests/util/serialization.cpp b/unit_tests/util/serialization.cpp index caf402dd5..038f12f03 100644 --- a/unit_tests/util/serialization.cpp +++ b/unit_tests/util/serialization.cpp @@ -71,6 +71,46 @@ BOOST_AUTO_TEST_CASE(tar_serialize_packed_vector) TestPackedVector result; storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint); util::serialization::read(reader, "my_packed_vector", result); + + CHECK_EQUAL_COLLECTIONS(result, v); + } + } +} + +BOOST_AUTO_TEST_CASE(tar_serialize_variable_indexed_data) +{ + TemporaryFile tmp; + { + using TestIndexedData = IndexedData>; + + std::vector> offset_data = { + {5, 8, 8}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + }; + std::vector char_data = { + "HalloFoo", + "ABCDEFGHIJKLMNOPQR", + "ABCDEFGHIJKLMNOP", + }; + + for (const auto i : util::irange(0, offset_data.size())) + { + TestIndexedData indexed {offset_data[i].begin(), offset_data[i].end(), char_data[i].begin()}; + { + storage::tar::FileWriter writer(tmp.path, + storage::tar::FileWriter::GenerateFingerprint); + util::serialization::write(writer, "my_indexed_data", indexed); + } + + TestIndexedData result; + storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint); + util::serialization::read(reader, "my_indexed_data", result); + + for(auto j : util::irange(0, offset_data[i].size() - 1)) + { + BOOST_CHECK_EQUAL(indexed.at(j), result.at(j)); + } } } }