Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Port .names file to tar

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-21 11:10:02 +00:00
parent a594008e57
commit b8260e44fa
47 changed files with 459 additions and 438 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View File

@ -15,6 +15,7 @@
#include "extractor/edge_based_node.hpp"
#include "extractor/intersection_bearings_container.hpp"
#include "extractor/maneuver_override.hpp"
#include "extractor/name_table.hpp"
#include "extractor/node_data_container.hpp"
#include "extractor/packed_osm_ids.hpp"
#include "extractor/profile_properties.hpp"
@ -40,7 +41,6 @@
#include "util/guidance/entry_class.hpp"
#include "util/guidance/turn_lanes.hpp"
#include "util/log.hpp"
#include "util/name_table.hpp"
#include "util/packed_vector.hpp"
#include "util/range_table.hpp"
#include "util/rectangle.hpp"
@ -215,7 +215,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
extractor::IntersectionBearingsView intersection_bearings_view;
util::NameTable m_name_table;
extractor::NameTableView m_name_table;
// the look-up table for entry classes. An entry class lists the possibility of entry for all
// available turns. Such a class id is stored with every edge.
util::vector_view<util::guidance::EntryClass> m_entry_class_table;
@ -353,11 +353,20 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
void InitializeNamePointers(storage::DataLayout &data_layout, char *memory_block)
{
auto name_data_ptr =
data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::NAME_CHAR_DATA);
const auto name_data_size =
data_layout.GetBlockEntries(storage::DataLayout::NAME_CHAR_DATA);
m_name_table.reset(name_data_ptr, name_data_ptr + name_data_size);
const auto name_blocks_ptr =
data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference>(
memory_block, storage::DataLayout::NAME_BLOCKS);
const auto name_values_ptr =
data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType>(
memory_block, storage::DataLayout::NAME_VALUES);
util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
name_blocks_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS));
util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
name_values_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_VALUES));
extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
m_name_table = extractor::NameTableView{std::move(index_data_view)};
}
void InitializeTurnLaneDescriptionsPointers(storage::DataLayout &data_layout,
@ -938,7 +947,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
auto found_range = std::equal_range(
m_maneuver_overrides.begin(), m_maneuver_overrides.end(), edge_based_node_id, Comp{});
std::for_each(found_range.first, found_range.second, [&](const auto & override) {
std::for_each(found_range.first, found_range.second, [&](const auto &override) {
std::vector<NodeID> sequence(
m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_begin,
m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_end);

6
include/extractor/edge_based_graph_factory.hpp
View File

@ -9,6 +9,7 @@
#include "extractor/edge_based_node_segment.hpp"
#include "extractor/extraction_turn.hpp"
#include "extractor/maneuver_override.hpp"
#include "extractor/name_table.hpp"
#include "extractor/nbg_to_ebg.hpp"
#include "extractor/node_data_container.hpp"
#include "extractor/query_node.hpp"
@ -18,7 +19,6 @@
#include "util/concurrent_id_map.hpp"
#include "util/deallocating_vector.hpp"
#include "util/name_table.hpp"
#include "util/node_based_graph.hpp"
#include "util/typedefs.hpp"
@ -70,7 +70,7 @@ class EdgeBasedGraphFactory
const std::unordered_set<NodeID> &barrier_nodes,
const std::unordered_set<NodeID> &traffic_lights,
const std::vector<util::Coordinate> &coordinates,
const util::NameTable &name_table,
const NameTable &name_table,
const std::unordered_set<EdgeID> &segregated_edges,
const LaneDescriptionMap &lane_description_map);
@ -138,7 +138,7 @@ class EdgeBasedGraphFactory
const std::unordered_set<NodeID> &m_traffic_lights;
const CompressedEdgeContainer &m_compressed_edge_container;
const util::NameTable &name_table;
const NameTable &name_table;
const std::unordered_set<EdgeID> &segregated_edges;
const LaneDescriptionMap &lane_description_map;

4
include/extractor/extractor.hpp
View File

@ -77,7 +77,7 @@ class Extractor
const std::vector<TurnRestriction> &turn_restrictions,
const std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions,
const std::unordered_set<EdgeID> &segregated_edges,
const util::NameTable &name_table,
const NameTable &name_table,
const std::vector<UnresolvedManeuverOverride> &maneuver_overrides,
const LaneDescriptionMap &turn_lane_map,
// for calculating turn penalties
@ -116,7 +116,7 @@ class Extractor
const std::unordered_set<NodeID> &barrier_nodes,
const std::vector<TurnRestriction> &turn_restrictions,
const std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions,
const util::NameTable &name_table,
const NameTable &name_table,
LaneDescriptionMap lane_description_map,
ScriptingEnvironment &scripting_environment);
};

21
include/extractor/files.hpp
View File

@ -411,7 +411,8 @@ void readRawNBGraph(const boost::filesystem::path &path,
osm_node_ids.push_back(current_node.node_id);
index++;
};
reader.ReadStreaming<extractor::QueryNode>("/extractor/nodes", boost::make_function_output_iterator(decode));
reader.ReadStreaming<extractor::QueryNode>("/extractor/nodes",
boost::make_function_output_iterator(decode));
reader.ReadStreaming<NodeID>("/extractor/barriers", barriers);
@ -420,6 +421,24 @@ void readRawNBGraph(const boost::filesystem::path &path,
storage::serialization::read(reader, "/extractor/edges", edge_list);
storage::serialization::read(reader, "/extractor/annotations", annotations);
}
template <typename NameTableT>
void readNames(const boost::filesystem::path &path, NameTableT &table)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
serialization::read(reader, "/common/names", table);
}
template <typename NameTableT>
void writeNames(const boost::filesystem::path &path, const NameTableT &table)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
serialization::write(writer, "/common/names", table);
}
}
}
}

4
include/extractor/intersection/have_identical_names.hpp
View File

@ -1,9 +1,9 @@
#ifndef OSRM_EXTRACTOR_INTERSECTION_HAVE_IDENTICAL_NAMES_HPP_
#define OSRM_EXTRACTOR_INTERSECTION_HAVE_IDENTICAL_NAMES_HPP_
#include "extractor/name_table.hpp"
#include "extractor/suffix_table.hpp"
#include "guidance/constants.hpp"
#include "util/name_table.hpp"
namespace osrm
{
@ -17,7 +17,7 @@ namespace intersection
// rhs->lhs)
bool HaveIdenticalNames(const NameID lhs,
const NameID rhs,
const util::NameTable &name_table,
const NameTable &name_table,
const SuffixTable &street_name_suffix_table);
} // namespace intersection

7
include/extractor/intersection/mergable_road_detector.hpp
View File

@ -6,7 +6,10 @@
#include "extractor/intersection/have_identical_names.hpp"
#include "extractor/restriction_index.hpp"
#include "extractor/turn_lane_types.hpp"
#include "extractor/name_table.hpp"
#include "guidance/intersection.hpp"
#include "util/coordinate.hpp"
#include "util/node_based_graph.hpp"
#include "util/typedefs.hpp"
@ -49,7 +52,7 @@ class MergableRoadDetector
const RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const SuffixTable &street_name_suffix_table);
// OSM ways tend to be modelled as separate ways for different directions. This is often due to
@ -168,7 +171,7 @@ class MergableRoadDetector
const TurnLanesIndexedArray &turn_lanes_data;
// name detection
const util::NameTable &name_table;
const extractor::NameTable &name_table;
const SuffixTable &street_name_suffix_table;
const CoordinateExtractor coordinate_extractor;

120
include/extractor/name_table.hpp Normal file
View File

@ -0,0 +1,120 @@
#ifndef OSRM_EXTRACTOR_NAME_TABLE_HPP
#define OSRM_EXTRACTOR_NAME_TABLE_HPP
#include "util/indexed_data.hpp"
#include "util/string_view.hpp"
#include "util/typedefs.hpp"
#include <string>
namespace osrm
{
namespace extractor
{
namespace detail
{
template <storage::Ownership Ownership> class NameTableImpl;
}
namespace serialization
{
template <storage::Ownership Ownership>
inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::NameTableImpl<Ownership> &index_data);
template <storage::Ownership Ownership>
inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::NameTableImpl<Ownership> &index_data);
}
namespace detail
{
// This class provides a limited view over all the string data we serialize out.
// The following functions are a subset of what is available.
// See the data facades for they provide full access to this serialized string data.
// Way string data is stored in blocks based on `id` as follows:
//
// | name | destination | pronunciation | ref | exits
// ^ ^
// [range)
// ^ id + 2
//
// `id + offset` gives us the range of chars.
//
// Offset 0 is name, 1 is destination, 2 is pronunciation, 3 is ref, 4 is exits
// See datafacades and extractor callbacks for details.
template <storage::Ownership Ownership> class NameTableImpl
{
public:
using IndexedData =
util::detail::IndexedDataImpl<util::VariableGroupBlock<16, util::StringView>, Ownership>;
using ResultType = typename IndexedData::ResultType;
using ValueType = typename IndexedData::ValueType;
NameTableImpl() {}
NameTableImpl(IndexedData indexed_data_) : indexed_data{std::move(indexed_data_)} {}
util::StringView GetNameForID(const NameID id) const
{
if (id == INVALID_NAMEID)
return {};
return indexed_data.at(id + 0);
}
util::StringView GetDestinationsForID(const NameID id) const
{
if (id == INVALID_NAMEID)
return {};
return indexed_data.at(id + 1);
}
util::StringView GetExitsForID(const NameID id) const
{
if (id == INVALID_NAMEID)
return {};
return indexed_data.at(id + 4);
}
util::StringView GetRefForID(const NameID id) const
{
if (id == INVALID_NAMEID)
return {};
const constexpr auto OFFSET_REF = 3u;
return indexed_data.at(id + OFFSET_REF);
}
util::StringView GetPronunciationForID(const NameID id) const
{
if (id == INVALID_NAMEID)
return {};
const constexpr auto OFFSET_PRONUNCIATION = 2u;
return indexed_data.at(id + OFFSET_PRONUNCIATION);
}
friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
const std::string &name,
NameTableImpl &index_data);
friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
const std::string &name,
const NameTableImpl &index_data);
private:
IndexedData indexed_data;
};
}
using NameTable = detail::NameTableImpl<storage::Ownership::Container>;
using NameTableView = detail::NameTableImpl<storage::Ownership::View>;
} // namespace extractor
} // namespace osrm
#endif // OSRM_EXTRACTOR_NAME_TABLE_HPP

38
include/extractor/serialization.hpp
View File

@ -1,10 +1,11 @@
#ifndef OSRM_EXTRACTOR_IO_HPP
#define OSRM_EXTRACTOR_IO_HPP
#include "conditional_turn_penalty.hpp"
#include "extractor/conditional_turn_penalty.hpp"
#include "extractor/datasources.hpp"
#include "extractor/intersection_bearings_container.hpp"
#include "extractor/maneuver_override.hpp"
#include "extractor/name_table.hpp"
#include "extractor/nbg_to_ebg.hpp"
#include "extractor/node_data_container.hpp"
#include "extractor/profile_properties.hpp"
@ -117,16 +118,18 @@ inline void read(storage::tar::FileReader &reader,
{
// read actual data
storage::serialization::read(reader, name + "/nodes", node_data_container.nodes);
storage::serialization::read(reader, name + "/annotations", node_data_container.annotation_data);
storage::serialization::read(
reader, name + "/annotations", node_data_container.annotation_data);
}
template <storage::Ownership Ownership>
inline void write(storage::tar::FileWriter &writer,
const std::string& name,
const std::string &name,
const detail::EdgeBasedNodeDataContainerImpl<Ownership> &node_data_container)
{
storage::serialization::write(writer, name + "/nodes", node_data_container.nodes);
storage::serialization::write(writer, name + "/annotations", node_data_container.annotation_data);
storage::serialization::write(
writer, name + "/annotations", node_data_container.annotation_data);
}
inline void read(storage::io::FileReader &reader, NodeRestriction &restriction)
@ -307,7 +310,8 @@ inline void write(storage::io::BufferWriter &writer,
}
}
inline void read(storage::io::BufferReader &reader, std::vector<ConditionalTurnPenalty> &conditional_penalties)
inline void read(storage::io::BufferReader &reader,
std::vector<ConditionalTurnPenalty> &conditional_penalties)
{
auto num_elements = reader.ReadElementCount64();
conditional_penalties.resize(num_elements);
@ -318,8 +322,8 @@ inline void read(storage::io::BufferReader &reader, std::vector<ConditionalTurnP
}
inline void write(storage::tar::FileWriter &writer,
const std::string& name,
const std::vector<ConditionalTurnPenalty> &conditional_penalties)
const std::string &name,
const std::vector<ConditionalTurnPenalty> &conditional_penalties)
{
storage::io::BufferWriter buffer_writer;
write(buffer_writer, conditional_penalties);
@ -328,7 +332,7 @@ inline void write(storage::tar::FileWriter &writer,
}
inline void read(storage::tar::FileReader &reader,
const std::string& name,
const std::string &name,
std::vector<ConditionalTurnPenalty> &conditional_penalties)
{
std::string buffer;
@ -336,9 +340,25 @@ inline void read(storage::tar::FileReader &reader,
storage::io::BufferReader buffer_reader{buffer};
read(buffer_reader, conditional_penalties);
}
template <storage::Ownership Ownership>
inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::NameTableImpl<Ownership> &name_table)
{
storage::io::BufferWriter buffer_writer;
util::serialization::write(writer, name, name_table.indexed_data);
}
template <storage::Ownership Ownership>
inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::NameTableImpl<Ownership> &name_table)
{
std::string buffer;
util::serialization::read(reader, name, name_table.indexed_data);
}
}
}
}

2
include/guidance/driveway_handler.hpp
View File

@ -19,7 +19,7 @@ class DrivewayHandler final : public IntersectionHandler
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table);
~DrivewayHandler() override final = default;

4
include/guidance/guidance_processing.hpp
View File

@ -8,12 +8,12 @@
#include "extractor/suffix_table.hpp"
#include "extractor/turn_lane_types.hpp"
#include "extractor/way_restriction_map.hpp"
#include "extractor/name_table.hpp"
#include "util/coordinate.hpp"
#include "util/guidance/bearing_class.hpp"
#include "util/guidance/entry_class.hpp"
#include "util/guidance/turn_lanes.hpp"
#include "util/name_table.hpp"
#include "util/node_based_graph.hpp"
#include <unordered_set>
@ -33,7 +33,7 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::RestrictionMap &node_restriction_map,
const extractor::WayRestrictionMap &way_restriction_map,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &suffix_table,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
extractor::LaneDescriptionMap &lane_description_map,

6
include/guidance/intersection_handler.hpp
View File

@ -4,13 +4,13 @@
#include "extractor/intersection/intersection_analysis.hpp"
#include "extractor/intersection/node_based_graph_walker.hpp"
#include "extractor/suffix_table.hpp"
#include "extractor/name_table.hpp"
#include "guidance/constants.hpp"
#include "guidance/intersection.hpp"
#include "util/assert.hpp"
#include "util/coordinate_calculation.hpp"
#include "util/guidance/name_announcements.hpp"
#include "util/name_table.hpp"
#include "util/node_based_graph.hpp"
#include <algorithm>
@ -38,7 +38,7 @@ class IntersectionHandler
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table);
virtual ~IntersectionHandler() = default;
@ -59,7 +59,7 @@ class IntersectionHandler
const extractor::RestrictionMap &node_restriction_map;
const std::unordered_set<NodeID> &barrier_nodes;
const extractor::TurnLanesIndexedArray &turn_lanes_data;
const util::NameTable &name_table;
const extractor::NameTable &name_table;
const extractor::SuffixTable &street_name_suffix_table;
const extractor::intersection::NodeBasedGraphWalker
graph_walker; // for skipping traffic signal, distances etc.

5
include/guidance/is_through_street.hpp
View File

@ -7,6 +7,9 @@
#include "extractor/node_data_container.hpp"
#include "extractor/suffix_table.hpp"
#include "util/node_based_graph.hpp"
#include "util/bearing.hpp"
#include "util/guidance/name_announcements.hpp"
namespace osrm
@ -19,7 +22,7 @@ inline bool isThroughStreet(const std::size_t index,
const IntersectionType &intersection,
const util::NodeBasedDynamicGraph &node_based_graph,
const extractor::EdgeBasedNodeDataContainer &node_data_container,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table)
{
using osrm::util::angularDeviation;

5
include/guidance/motorway_handler.hpp
View File

@ -1,12 +1,13 @@
#ifndef OSRM_GUIDANCE_MOTORWAY_HANDLER_HPP_
#define OSRM_GUIDANCE_MOTORWAY_HANDLER_HPP_
#include "extractor/name_table.hpp"
#include "guidance/intersection.hpp"
#include "guidance/intersection_handler.hpp"
#include "guidance/is_through_street.hpp"
#include "util/attributes.hpp"
#include "util/name_table.hpp"
#include "util/node_based_graph.hpp"
#include <vector>
@ -27,7 +28,7 @@ class MotorwayHandler : public IntersectionHandler
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table);
~MotorwayHandler() override final = default;

5
include/guidance/roundabout_handler.hpp
View File

@ -4,12 +4,13 @@
#include "extractor/compressed_edge_container.hpp"
#include "extractor/intersection/coordinate_extractor.hpp"
#include "extractor/query_node.hpp"
#include "extractor/name_table.hpp"
#include "guidance/intersection.hpp"
#include "guidance/intersection_handler.hpp"
#include "guidance/is_through_street.hpp"
#include "guidance/roundabout_type.hpp"
#include "util/name_table.hpp"
#include "util/node_based_graph.hpp"
#include "util/typedefs.hpp"
@ -44,7 +45,7 @@ class RoundaboutHandler : public IntersectionHandler
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table);
~RoundaboutHandler() override final = default;

9
include/guidance/segregated_intersection_classification.hpp
View File

@ -1,14 +1,11 @@
#include "extractor/name_table.hpp"
#include "util/typedefs.hpp"
#include <unordered_set>
namespace osrm
{
namespace util
{
class NameTable;
}
namespace extractor
{
class NodeBasedGraphFactory;
@ -22,6 +19,6 @@ namespace guidance
// - staggered intersections (X-cross)
// - square/circle intersections
std::unordered_set<EdgeID> findSegregatedNodes(const extractor::NodeBasedGraphFactory &factory,
const util::NameTable &names);
const extractor::NameTable &names);
}
}

5
include/guidance/sliproad_handler.hpp
View File

@ -1,11 +1,12 @@
#ifndef OSRM_GUIDANCE_SLIPROAD_HANDLER_HPP_
#define OSRM_GUIDANCE_SLIPROAD_HANDLER_HPP_
#include "extractor/name_table.hpp"
#include "guidance/intersection.hpp"
#include "guidance/intersection_handler.hpp"
#include "guidance/is_through_street.hpp"
#include "util/name_table.hpp"
#include "util/node_based_graph.hpp"
#include <vector>
@ -28,7 +29,7 @@ class SliproadHandler final : public IntersectionHandler
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table);
~SliproadHandler() override final = default;

2
include/guidance/statistics_handler.hpp
View File

@ -32,7 +32,7 @@ class StatisticsHandler final : public IntersectionHandler
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table)
: IntersectionHandler(node_based_graph,
node_data_container,

5
include/guidance/suppress_mode_handler.hpp
View File

@ -1,8 +1,11 @@
#ifndef OSRM_GUIDANCE_SUPPRESS_MODE_HANDLER_HPP_
#define OSRM_GUIDANCE_SUPPRESS_MODE_HANDLER_HPP_
#include "extractor/name_table.hpp"
#include "guidance/intersection.hpp"
#include "guidance/intersection_handler.hpp"
#include "util/node_based_graph.hpp"
namespace osrm
@ -23,7 +26,7 @@ class SuppressModeHandler final : public IntersectionHandler
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table);
~SuppressModeHandler() override final = default;

5
include/guidance/turn_analysis.hpp
View File

@ -5,6 +5,8 @@
#include "extractor/intersection/intersection_view.hpp"
#include "extractor/restriction_index.hpp"
#include "extractor/suffix_table.hpp"
#include "extractor/name_table.hpp"
#include "guidance/driveway_handler.hpp"
#include "guidance/intersection.hpp"
#include "guidance/motorway_handler.hpp"
@ -16,7 +18,6 @@
#include "guidance/turn_handler.hpp"
#include "util/attributes.hpp"
#include "util/name_table.hpp"
#include "util/node_based_graph.hpp"
#include <cstdint>
@ -43,7 +44,7 @@ class TurnAnalysis
const extractor::RestrictionMap &restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table);
/* Full Analysis Process for a single node/edge combination. Use with caution, as the process is

5
include/guidance/turn_handler.hpp
View File

@ -2,12 +2,13 @@
#define OSRM_GUIDANCE_TURN_HANDLER_HPP_
#include "extractor/query_node.hpp"
#include "extractor/name_table.hpp"
#include "guidance/intersection.hpp"
#include "guidance/intersection_handler.hpp"
#include "guidance/is_through_street.hpp"
#include "util/attributes.hpp"
#include "util/name_table.hpp"
#include "util/node_based_graph.hpp"
#include <boost/optional.hpp>
@ -32,7 +33,7 @@ class TurnHandler : public IntersectionHandler
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table);
~TurnHandler() override final = default;

3
include/guidance/turn_lane_handler.hpp
View File

@ -1,15 +1,16 @@
#ifndef OSRM_GUIDANCE_TURN_LANE_HANDLER_HPP_
#define OSRM_GUIDANCE_TURN_LANE_HANDLER_HPP_
#include "extractor/name_table.hpp"
#include "extractor/query_node.hpp"
#include "extractor/turn_lane_types.hpp"
#include "guidance/intersection.hpp"
#include "guidance/turn_analysis.hpp"
#include "guidance/turn_lane_data.hpp"
#include "util/attributes.hpp"
#include "util/guidance/turn_lanes.hpp"
#include "util/name_table.hpp"
#include "util/node_based_graph.hpp"
#include "util/typedefs.hpp"

12
include/storage/shared_datatype.hpp
View File

@ -20,7 +20,9 @@ namespace storage
// Added at the start and end of each block as sanity check
const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'};
const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
const constexpr char *block_id_to_name[] = {"IGNORE_BLOCK",
"NAME_BLOCKS",
"NAME_VALUES",
"EDGE_BASED_NODE_DATA",
"ANNOTATION_DATA",
"CH_GRAPH_NODE_LIST",
@ -92,15 +94,16 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
"MLD_GRAPH_EDGE_LIST",
"MLD_GRAPH_NODE_TO_OFFSET",
"MANEUVER_OVERRIDES",
"MANEUVER_OVERRIDE_NODE_SEQUENCES",
"IGNORE_BLOCK"};
"MANEUVER_OVERRIDE_NODE_SEQUENCES"};
class DataLayout
{
public:
enum BlockID
{
NAME_CHAR_DATA = 0,
IGNORE_BLOCK = 0,
NAME_BLOCKS,
NAME_VALUES,
EDGE_BASED_NODE_DATA_LIST,
ANNOTATION_DATA_LIST,
CH_GRAPH_NODE_LIST,
@ -173,7 +176,6 @@ class DataLayout
MLD_GRAPH_NODE_TO_OFFSET,
MANEUVER_OVERRIDES,
MANEUVER_OVERRIDE_NODE_SEQUENCES,
IGNORE_BLOCK,
NUM_BLOCKS
};

5
include/util/guidance/name_announcements.hpp
View File

@ -3,9 +3,10 @@
/* A set of tools required for guidance in both pre and post-processing */
#include "extractor/name_table.hpp"
#include "extractor/suffix_table.hpp"
#include "util/attributes.hpp"
#include "util/name_table.hpp"
#include "util/typedefs.hpp"
#include <algorithm>
@ -216,7 +217,7 @@ inline bool requiresNameAnnounced(const std::string &from_name,
inline bool requiresNameAnnounced(const NameID from_name_id,
const NameID to_name_id,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &suffix_table)
{
if (from_name_id == to_name_id)

198
include/util/indexed_data.hpp
View File

@ -1,12 +1,14 @@
#ifndef OSRM_INDEXED_DATA_HPP
#define OSRM_INDEXED_DATA_HPP
#include "storage/io.hpp"
#include "storage/tar_fwd.hpp"
#include "util/exception.hpp"
#include "util/string_view.hpp"
#include "util/vector_view.hpp"
#include <boost/assert.hpp>
#include <boost/function_output_iterator.hpp>
#include <array>
#include <iterator>
@ -18,6 +20,23 @@ namespace osrm
{
namespace util
{
namespace detail
{
template <typename GroupBlockPolicy, storage::Ownership Ownership> struct IndexedDataImpl;
}
namespace serialization
{
template <typename BlockPolicy, storage::Ownership Ownership>
inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::IndexedDataImpl<BlockPolicy, Ownership> &index_data);
template <typename BlockPolicy, storage::Ownership Ownership>
inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::IndexedDataImpl<BlockPolicy, Ownership> &index_data);
}
template <int N, typename T = std::string> struct VariableGroupBlock
{
@ -85,11 +104,11 @@ template <int N, typename T = std::string> struct VariableGroupBlock
/// of prefix length. sum(descriptor) equals to the block
/// prefix length.
/// Returns the block prefix length.
template <typename Offset, typename OffsetIterator>
Offset WriteBlockReference(storage::io::FileWriter &out,
Offset data_offset,
OffsetIterator first,
OffsetIterator last) const
template <typename Offset, typename OffsetIterator, typename OutIter>
OutIter WriteBlockReference(OffsetIterator first,
OffsetIterator last,
Offset &data_offset,
OutIter out) const
{
BOOST_ASSERT(data_offset <= std::numeric_limits<decltype(BlockReference::offset)>::max());
@ -106,9 +125,9 @@ template <int N, typename T = std::string> struct VariableGroupBlock
prefix_length += byte_length;
}
out.WriteFrom(refernce);
return prefix_length;
data_offset += prefix_length;
*out++ = refernce;
return out;
}
/// Write a block prefix that is an array of variable encoded data lengths:
@ -118,9 +137,8 @@ template <int N, typename T = std::string> struct VariableGroupBlock
/// 65536..16777215 is 3 bytes.
/// [first..last] is an inclusive range of block data.
/// The length of the last item in the block is not stored.
template <typename OffsetIterator>
void
WriteBlockPrefix(storage::io::FileWriter &out, OffsetIterator first, OffsetIterator last) const
template <typename OffsetIterator, typename OutByteIter>
OutByteIter WriteBlockPrefix(OffsetIterator first, OffsetIterator last, OutByteIter out) const
{
for (OffsetIterator curr = first, next = std::next(first); curr != last; ++curr, ++next)
{
@ -131,8 +149,9 @@ template <int N, typename T = std::string> struct VariableGroupBlock
// Here, we're only writing a few bytes from the 4-byte std::uint32_t,
// so we need to cast to (char *)
out.WriteFrom((const char *)&data_length, byte_length);
out = std::copy_n((const char *)&data_length, byte_length, out);
}
return out;
}
/// Advances the range to an item stored in the referenced block.
@ -178,36 +197,39 @@ template <int N, typename T = std::string> struct FixedGroupBlock
/// Write a block reference {offset}, where offset is a global block offset
/// Returns the fixed block prefix length.
template <typename Offset, typename OffsetIterator>
Offset WriteBlockReference(storage::io::FileWriter &out,
Offset data_offset,
OffsetIterator,
OffsetIterator) const
template <typename Offset, typename OffsetIterator, typename OutIterator>
OutIterator
WriteBlockReference(OffsetIterator, OffsetIterator, Offset &data_offset, OutIterator out) const
{
BOOST_ASSERT(data_offset <= std::numeric_limits<decltype(BlockReference::offset)>::max());
BlockReference refernce{static_cast<decltype(BlockReference::offset)>(data_offset)};
out.WriteFrom(refernce);
data_offset += BLOCK_SIZE;
*out++ = refernce;
return BLOCK_SIZE;
return out;
}
/// Write a fixed length block prefix.
template <typename OffsetIterator>
void
WriteBlockPrefix(storage::io::FileWriter &out, OffsetIterator first, OffsetIterator last) const
template <typename OffsetIterator, typename OutByteIter>
OutByteIter WriteBlockPrefix(OffsetIterator first, OffsetIterator last, OutByteIter out) const
{
std::uint32_t index = 0;
std::array<ValueType, BLOCK_SIZE> block_prefix;
constexpr std::size_t MAX_LENGTH = std::numeric_limits<std::make_unsigned_t<ValueType>>::max();
auto index = 0;
std::array<ValueType, BLOCK_SIZE> prefix;
for (OffsetIterator curr = first, next = std::next(first); curr != last; ++curr, ++next)
{
const std::uint32_t data_length = *next - *curr;
if (data_length >= 0x100)
throw util::exception(boost::format("too large data length %1%") % data_length);
if (data_length > MAX_LENGTH)
throw util::exception(boost::format("too large data length %1% > %2%") % data_length % MAX_LENGTH);
block_prefix[index++] = static_cast<ValueType>(data_length);
prefix[index++] = data_length;
}
out.WriteFrom(block_prefix.data(), block_prefix.size());
out = std::copy_n((const char *)prefix.data(), sizeof(ValueType)*BLOCK_SIZE, out);
return out;
}
/// Advances the range to an item stored in the referenced block.
@ -233,28 +255,31 @@ template <int N, typename T = std::string> struct FixedGroupBlock
}
};
template <typename GroupBlock> struct IndexedData
namespace detail
{
static constexpr std::uint32_t BLOCK_SIZE = GroupBlock::BLOCK_SIZE;
template <typename GroupBlockPolicy, storage::Ownership Ownership> struct IndexedDataImpl
{
static constexpr std::uint32_t BLOCK_SIZE = GroupBlockPolicy::BLOCK_SIZE;
using BlocksNumberType = std::uint32_t;
using DataSizeType = std::uint64_t;
using BlockReference = typename GroupBlock::BlockReference;
using ResultType = typename GroupBlock::ResultType;
using ValueType = typename GroupBlock::ValueType;
using BlockReference = typename GroupBlockPolicy::BlockReference;
using ResultType = typename GroupBlockPolicy::ResultType;
using ValueType = typename GroupBlockPolicy::ValueType;
static_assert(sizeof(ValueType) == 1, "data basic type must char");
IndexedData() : blocks_number{0}, block_references{nullptr}, begin{nullptr}, end{nullptr} {}
IndexedDataImpl() = default;
IndexedDataImpl(util::vector_view<BlockReference> blocks_, util::vector_view<ValueType> values_)
: blocks(std::move(blocks_)), values(std::move(values_))
{
}
bool empty() const { return blocks_number == 0; }
bool empty() const { return blocks.empty(); }
template <typename OffsetIterator, typename DataIterator>
void write(storage::io::FileWriter &out,
OffsetIterator first,
OffsetIterator last,
DataIterator data) const
IndexedDataImpl(OffsetIterator first, OffsetIterator last, DataIterator data)
{
static_assert(sizeof(typename DataIterator::value_type) == 1, "data basic type must char");
@ -268,69 +293,37 @@ template <typename GroupBlock> struct IndexedData
const BlocksNumberType number_of_blocks =
number_of_elements == 0 ? 0
: 1 + (std::distance(first, sentinel) - 1) / (BLOCK_SIZE + 1);
out.WriteFrom(number_of_blocks);
blocks.resize(number_of_blocks);
// Write block references and compute the total data size that includes prefix and data
const GroupBlock block;
const GroupBlockPolicy block;
auto block_iter = blocks.begin();
DataSizeType data_size = 0;
for (OffsetIterator curr = first, next = first; next != sentinel; curr = next)
{
std::advance(next, std::min<diff_type>(BLOCK_SIZE, std::distance(next, sentinel)));
data_size += block.WriteBlockReference(out, data_size, curr, next);
block_iter = block.WriteBlockReference(curr, next, data_size, block_iter);
std::advance(next, std::min<diff_type>(1, std::distance(next, sentinel)));
data_size += *next - *curr;
}
// Write the total data size
out.WriteFrom(data_size);
values.resize(data_size);
auto values_byte_iter = reinterpret_cast<char *>(values.data());
// Write data blocks that are (prefix, data)
for (OffsetIterator curr = first, next = first; next != sentinel; curr = next)
{
std::advance(next, std::min<diff_type>(BLOCK_SIZE, std::distance(next, sentinel)));
block.WriteBlockPrefix(out, curr, next);
values_byte_iter = block.WriteBlockPrefix(curr, next, values_byte_iter);
std::advance(next, std::min<diff_type>(1, std::distance(next, sentinel)));
std::for_each(
data + *curr, data + *next, [&out](const auto &element) { out.WriteFrom(element); });
auto to_bytes = [&](const auto &data) {
values_byte_iter = std::copy_n(&data, sizeof(ValueType), values_byte_iter);
};
std::copy(data + *curr, data + *next, boost::make_function_output_iterator(to_bytes));
}
}
/// Set internal pointers from the buffer [first, last).
/// Data buffer pointed by ptr must exists during IndexedData life-time.
/// No ownership is transferred.
void reset(const ValueType *first, const ValueType *last)
{
// Read blocks number
if (first + sizeof(BlocksNumberType) > last)
throw util::exception("incorrect memory block");
blocks_number = *reinterpret_cast<const BlocksNumberType *>(first);
first += sizeof(BlocksNumberType);
// Get block references pointer
if (first + sizeof(BlockReference) * blocks_number > last)
throw util::exception("incorrect memory block");
block_references = reinterpret_cast<const BlockReference *>(first);
first += sizeof(BlockReference) * blocks_number;
// Read total data size
if (first + sizeof(DataSizeType) > last)
throw util::exception("incorrect memory block");
auto data_size = *reinterpret_cast<const DataSizeType *>(first);
first += sizeof(DataSizeType);
// Get data blocks begin and end iterators
begin = reinterpret_cast<const ValueType *>(first);
first += sizeof(ValueType) * data_size;
if (first > last)
throw util::exception("incorrect memory block");
end = reinterpret_cast<const ValueType *>(first);
}
// Return value at the given index
ResultType at(std::uint32_t index) const
{
@ -338,20 +331,29 @@ template <typename GroupBlock> struct IndexedData
const BlocksNumberType block_idx = index / (BLOCK_SIZE + 1);
const std::uint32_t internal_idx = index % (BLOCK_SIZE + 1);
if (block_idx >= blocks_number)
if (block_idx >= blocks.size())
return ResultType();
// Get block first and last iterators
auto first = begin + block_references[block_idx].offset;
auto last =
block_idx + 1 == blocks_number ? end : begin + block_references[block_idx + 1].offset;
auto first = values.begin() + blocks[block_idx].offset;
auto last = block_idx + 1 == blocks.size() ? values.end()
: values.begin() + blocks[block_idx + 1].offset;
const GroupBlock block;
block.ReadRefrencedBlock(block_references[block_idx], internal_idx, first, last);
const GroupBlockPolicy block;
block.ReadRefrencedBlock(blocks[block_idx], internal_idx, first, last);
return adapt(first, last);
return adapt(&*first, &*last);
}
friend void serialization::read<GroupBlockPolicy, Ownership>(storage::tar::FileReader &reader,
const std::string &name,
IndexedDataImpl &index_data);
friend void
serialization::write<GroupBlockPolicy, Ownership>(storage::tar::FileWriter &writer,
const std::string &name,
const IndexedDataImpl &index_data);
private:
template <class T = ResultType>
typename std::enable_if<!std::is_same<T, StringView>::value, T>::type
@ -367,10 +369,16 @@ template <typename GroupBlock> struct IndexedData
return ResultType(first, std::distance(first, last));
}
BlocksNumberType blocks_number;
const BlockReference *block_references;
const ValueType *begin, *end;
template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
Vector<BlockReference> blocks;
Vector<ValueType> values;
};
}
template <typename GroupBlockPolicy>
using IndexedData = detail::IndexedDataImpl<GroupBlockPolicy, storage::Ownership::Container>;
template <typename GroupBlockPolicy>
using IndexedDataView = detail::IndexedDataImpl<GroupBlockPolicy, storage::Ownership::View>;
}
}
#endif // OSRM_INDEXED_DATA_HPP

51
include/util/name_table.hpp
View File

@ -1,51 +0,0 @@
#ifndef OSRM_UTIL_NAME_TABLE_HPP
#define OSRM_UTIL_NAME_TABLE_HPP
#include "util/indexed_data.hpp"
#include "util/string_view.hpp"
#include "util/typedefs.hpp"
#include <string>
namespace osrm
{
namespace util
{
// While this could, theoretically, hold any names in the fitting format,
// the NameTable allows access to a part of the Datafacade to allow
// processing based on name indices.
class NameTable
{
public:
using IndexedData = util::IndexedData<util::VariableGroupBlock<16, util::StringView>>;
using ResultType = IndexedData::ResultType;
using ValueType = IndexedData::ValueType;
NameTable() {}
// Read filename and store own data in m_buffer
NameTable(const std::string &filename);
// Keep pointers only in m_name_table and don't own data in m_buffer
void reset(ValueType *begin, ValueType *end);
// This class provides a limited view over all the string data we serialize out.
// The following functions are a subset of what is available.
// See the data facades for they provide full access to this serialized string data.
util::StringView GetNameForID(const NameID id) const;
util::StringView GetDestinationsForID(const NameID id) const;
util::StringView GetExitsForID(const NameID id) const;
util::StringView GetRefForID(const NameID id) const;
util::StringView GetPronunciationForID(const NameID id) const;
private:
using BufferType = std::unique_ptr<ValueType, std::function<void(void *)>>;
BufferType m_buffer;
IndexedData m_name_table;
};
} // namespace util
} // namespace osrm
#endif // OSRM_UTIL_NAME_TABLE_HPP

17
include/util/serialization.hpp
View File

@ -5,6 +5,7 @@
#include "util/packed_vector.hpp"
#include "util/range_table.hpp"
#include "util/static_graph.hpp"
#include "util/indexed_data.hpp"
#include "storage/io.hpp"
#include "storage/serialization.hpp"
@ -168,6 +169,22 @@ inline void write(storage::tar::FileWriter &writer,
writer.WriteStreaming<typename std::remove_reference_t<decltype(graph)>::Edge>(
name + "/edge_list", graph.edge_list.begin(), graph.number_of_edges);
}
template <typename BlockPolicy, storage::Ownership Ownership>
inline void
read(storage::tar::FileReader &reader, const std::string &name, detail::IndexedDataImpl<BlockPolicy, Ownership> &index_data)
{
storage::serialization::read(reader, name + "/blocks", index_data.blocks);
storage::serialization::read(reader, name + "/values", index_data.values);
}
template <typename BlockPolicy, storage::Ownership Ownership>
inline void write(storage::tar::FileWriter &writer,
const std::string &name, const detail::IndexedDataImpl<BlockPolicy, Ownership> &index_data)
{
storage::serialization::write(writer, name + "/blocks", index_data.blocks);
storage::serialization::write(writer, name + "/values", index_data.values);
}
}
}
}

2
src/extractor/edge_based_graph_factory.cpp
View File

@ -69,7 +69,7 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory(
const std::unordered_set<NodeID> &barrier_nodes,
const std::unordered_set<NodeID> &traffic_lights,
const std::vector<util::Coordinate> &coordinates,
const util::NameTable &name_table,
const NameTable &name_table,
const std::unordered_set<EdgeID> &segregated_edges,
const extractor::LaneDescriptionMap &lane_description_map)
: m_edge_based_node_container(node_data_container), m_connectivity_checksum(0),

7
src/extractor/extraction_containers.cpp
View File

@ -3,6 +3,8 @@
#include "extractor/extraction_way.hpp"
#include "extractor/restriction.hpp"
#include "extractor/serialization.hpp"
#include "extractor/name_table.hpp"
#include "extractor/files.hpp"
#include "util/coordinate_calculation.hpp"
@ -10,7 +12,6 @@
#include "util/exception_utils.hpp"
#include "util/fingerprint.hpp"
#include "util/log.hpp"
#include "util/name_table.hpp"
#include "util/timing_util.hpp"
#include "storage/io.hpp"
@ -163,10 +164,8 @@ void ExtractionContainers::WriteCharData(const std::string &file_name)
util::UnbufferedLog log;
log << "writing street name index ... ";
TIMER_START(write_index);
storage::io::FileWriter file(file_name, storage::io::FileWriter::GenerateFingerprint);
const util::NameTable::IndexedData indexed_data;
indexed_data.write(file, name_offsets.begin(), name_offsets.end(), name_char_data.begin());
files::writeNames(file_name, NameTable {NameTable::IndexedData (name_offsets.begin(), name_offsets.end(), name_char_data.begin())});
TIMER_STOP(write_index);
log << "ok, after " << TIMER_SEC(write_index) << "s";

14
src/extractor/extractor.cpp
View File

@ -13,6 +13,7 @@
#include "extractor/restriction_filter.hpp"
#include "extractor/restriction_parser.hpp"
#include "extractor/scripting_environment.hpp"
#include "extractor/name_table.hpp"
#include "guidance/files.hpp"
#include "guidance/guidance_processing.hpp"
@ -25,7 +26,6 @@
#include "util/exception_utils.hpp"
#include "util/integer_range.hpp"
#include "util/log.hpp"
#include "util/name_table.hpp"
#include "util/range_table.hpp"
#include "util/timing_util.hpp"
@ -230,11 +230,13 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
conditional_turn_restrictions,
unresolved_maneuver_overrides);
NameTable name_table;
files::readNames(config.GetPath(".osrm.names"), name_table);
util::Log() << "Find segregated edges in node-based graph ..." << std::flush;
TIMER_START(segregated);
util::NameTable names(config.GetPath(".osrm.names").string());
auto segregated_edges = guidance::findSegregatedNodes(node_based_graph_factory, names);
auto segregated_edges = guidance::findSegregatedNodes(node_based_graph_factory, name_table);
TIMER_STOP(segregated);
util::Log() << "ok, after " << TIMER_SEC(segregated) << "s";
@ -283,8 +285,6 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
const auto number_of_node_based_nodes = node_based_graph.GetNumberOfNodes();
const util::NameTable name_table(config.GetPath(".osrm.names").string());
const auto number_of_edge_based_nodes =
BuildEdgeExpandedGraph(node_based_graph,
coordinates,
@ -713,7 +713,7 @@ EdgeID Extractor::BuildEdgeExpandedGraph(
const std::vector<TurnRestriction> &turn_restrictions,
const std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions,
const std::unordered_set<EdgeID> &segregated_edges,
const util::NameTable &name_table,
const NameTable &name_table,
const std::vector<UnresolvedManeuverOverride> &maneuver_overrides,
const LaneDescriptionMap &turn_lane_map,
// for calculating turn penalties
@ -884,7 +884,7 @@ void Extractor::ProcessGuidanceTurns(
const std::unordered_set<NodeID> &barrier_nodes,
const std::vector<TurnRestriction> &turn_restrictions,
const std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions,
const util::NameTable &name_table,
const NameTable &name_table,
LaneDescriptionMap lane_description_map,
ScriptingEnvironment &scripting_environment)
{

4
src/extractor/intersection/have_identical_names.cpp
View File

@ -1,3 +1,5 @@
#include "extractor/intersection/have_identical_names.hpp"
#include "util/guidance/name_announcements.hpp"
namespace osrm
@ -12,7 +14,7 @@ namespace intersection
// rhs->lhs)
bool HaveIdenticalNames(const NameID lhs,
const NameID rhs,
const util::NameTable &name_table,
const NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table)
{
const auto non_empty = (lhs != EMPTY_NAMEID) && (rhs != EMPTY_NAMEID);

6
src/extractor/intersection/mergable_road_detector.cpp
View File

@ -3,12 +3,12 @@
#include "extractor/intersection/node_based_graph_walker.hpp"
#include "extractor/query_node.hpp"
#include "extractor/suffix_table.hpp"
#include "extractor/name_table.hpp"
#include "guidance/constants.hpp"
#include "util/bearing.hpp"
#include "util/coordinate_calculation.hpp"
#include "util/guidance/name_announcements.hpp"
#include "util/name_table.hpp"
using osrm::util::angularDeviation;
@ -27,7 +27,7 @@ namespace
inline auto makeCheckRoadForName(const NameID name_id,
const util::NodeBasedDynamicGraph &node_based_graph,
const EdgeBasedNodeDataContainer &node_data_container,
const util::NameTable &name_table,
const NameTable &name_table,
const SuffixTable &suffix_table)
{
return [name_id, &node_based_graph, &node_data_container, &name_table, &suffix_table](
@ -59,7 +59,7 @@ MergableRoadDetector::MergableRoadDetector(
const RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const NameTable &name_table,
const SuffixTable &street_name_suffix_table)
: node_based_graph(node_based_graph), node_data_container(node_data_container),
node_coordinates(node_coordinates), compressed_geometries(compressed_geometries),

2
src/guidance/driveway_handler.cpp
View File

@ -17,7 +17,7 @@ DrivewayHandler::DrivewayHandler(const util::NodeBasedDynamicGraph &node_based_g
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table)
: IntersectionHandler(node_based_graph,
node_data_container,

2
src/guidance/guidance_processing.cpp
View File

@ -24,7 +24,7 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::RestrictionMap &node_restriction_map,
const extractor::WayRestrictionMap &way_restriction_map,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &suffix_table,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
extractor::LaneDescriptionMap &lane_description_map,

2
src/guidance/intersection_handler.cpp
View File

@ -52,7 +52,7 @@ IntersectionHandler::IntersectionHandler(
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table)
: node_based_graph(node_based_graph), node_data_container(node_data_container),
node_coordinates(node_coordinates), compressed_geometries(compressed_geometries),

2
src/guidance/motorway_handler.cpp
View File

@ -49,7 +49,7 @@ MotorwayHandler::MotorwayHandler(const util::NodeBasedDynamicGraph &node_based_g
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table)
: IntersectionHandler(node_based_graph,
node_data_container,

2
src/guidance/roundabout_handler.cpp
View File

@ -30,7 +30,7 @@ RoundaboutHandler::RoundaboutHandler(
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table)
: IntersectionHandler(node_based_graph,
node_data_container,

3
src/guidance/segregated_intersection_classification.cpp
View File

@ -4,7 +4,6 @@
#include "guidance/turn_instruction.hpp"
#include "util/coordinate_calculation.hpp"
#include "util/name_table.hpp"
#include <set>
using osrm::guidance::getTurnDirection;
@ -42,7 +41,7 @@ struct EdgeInfo
};
std::unordered_set<EdgeID> findSegregatedNodes(const extractor::NodeBasedGraphFactory &factory,
const util::NameTable &names)
const extractor::NameTable &names)
{
auto const &graph = factory.GetGraph();
auto const &annotation = factory.GetAnnotationData();

2
src/guidance/sliproad_handler.cpp
View File

@ -27,7 +27,7 @@ SliproadHandler::SliproadHandler(const util::NodeBasedDynamicGraph &node_based_g
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table)
: IntersectionHandler(node_based_graph,
node_data_container,

2
src/guidance/suppress_mode_handler.cpp
View File

@ -17,7 +17,7 @@ SuppressModeHandler::SuppressModeHandler(
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table)
: IntersectionHandler(node_based_graph,
node_data_container,

2
src/guidance/turn_analysis.cpp
View File

@ -26,7 +26,7 @@ TurnAnalysis::TurnAnalysis(const util::NodeBasedDynamicGraph &node_based_graph,
const extractor::RestrictionMap &restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table)
: node_based_graph(node_based_graph), roundabout_handler(node_based_graph,
node_data_container,

2
src/guidance/turn_handler.cpp
View File

@ -115,7 +115,7 @@ TurnHandler::TurnHandler(const util::NodeBasedDynamicGraph &node_based_graph,
const extractor::RestrictionMap &node_restriction_map,
const std::unordered_set<NodeID> &barrier_nodes,
const extractor::TurnLanesIndexedArray &turn_lanes_data,
const util::NameTable &name_table,
const extractor::NameTable &name_table,
const extractor::SuffixTable &street_name_suffix_table)
: IntersectionHandler(node_based_graph,
node_data_container,

2
src/osrm/osrm.cpp
View File

@ -35,7 +35,7 @@ OSRM::OSRM(engine::EngineConfig &config)
auto mem = storage::makeSharedMemory(barrier.data().region);
auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
if (layout->GetBlockSize(storage::DataLayout::NAME_CHAR_DATA) == 0)
if (layout->GetBlockSize(storage::DataLayout::NAME_VALUES) == 0)
throw util::exception(
"No name data loaded, cannot continue. Have you run osrm-datastore to load data?");
}

32
src/storage/storage.cpp
View File

@ -18,6 +18,7 @@
#include "extractor/edge_based_node.hpp"
#include "extractor/files.hpp"
#include "extractor/maneuver_override.hpp"
#include "extractor/name_table.hpp"
#include "extractor/packed_osm_ids.hpp"
#include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp"
@ -243,13 +244,6 @@ void Storage::PopulateLayout(DataLayout &layout)
make_block<char>(absolute_file_index_path.string().length() + 1));
}
{
util::Log() << "load names from: " << config.GetPath(".osrm.names");
// number of entries in name index
io::FileReader name_file(config.GetPath(".osrm.names"), io::FileReader::VerifyFingerprint);
layout.SetBlock(DataLayout::NAME_CHAR_DATA, make_block<char>(name_file.GetSize()));
}
// load rsearch tree size
{
io::FileReader tree_node_file(config.GetPath(".osrm.ramIndex"),
@ -345,6 +339,8 @@ void Storage::PopulateLayout(DataLayout &layout)
{"/common/turn_data/lane_data_ids", DataLayout::LANE_DATA_ID},
{"/common/turn_data/entry_class_ids", DataLayout::ENTRY_CLASSID},
{"/common/turn_data/connectivity_checksum", DataLayout::IGNORE_BLOCK},
{"/common/names/blocks", DataLayout::NAME_BLOCKS},
{"/common/names/values", DataLayout::NAME_VALUES},
};
std::vector<NamedBlock> blocks;
@ -368,6 +364,7 @@ void Storage::PopulateLayout(DataLayout &layout)
{REQUIRED, config.GetPath(".osrm.turn_weight_penalties")},
{REQUIRED, config.GetPath(".osrm.turn_duration_penalties")},
{REQUIRED, config.GetPath(".osrm.edges")},
{REQUIRED, config.GetPath(".osrm.names")},
};
for (const auto &file : tar_files)
@ -425,14 +422,21 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Name data
{
io::FileReader name_file(config.GetPath(".osrm.names"), io::FileReader::VerifyFingerprint);
std::size_t name_file_size = name_file.GetSize();
const auto name_blocks_ptr =
layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference, true>(
memory_ptr, DataLayout::NAME_BLOCKS);
const auto name_values_ptr =
layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType, true>(
memory_ptr, DataLayout::NAME_VALUES);
BOOST_ASSERT(name_file_size == layout.GetBlockSize(DataLayout::NAME_CHAR_DATA));
const auto name_char_ptr =
layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::NAME_CHAR_DATA);
util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
name_blocks_ptr, layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS));
util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
name_values_ptr, layout.GetBlockEntries(storage::DataLayout::NAME_VALUES));
name_file.ReadInto<char>(name_char_ptr, name_file_size);
extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
extractor::NameTableView name_table{index_data_view};
extractor::files::readNames(config.GetPath(".osrm.names"), name_table);
}
// Turn lane data
@ -624,7 +628,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
turn_duration_penalties_ptr,
layout.GetBlockEntries(storage::DataLayout::TURN_WEIGHT_PENALTIES));
extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"),
turn_duration_penalties);
turn_duration_penalties);
}
// load turn duration penalties

100
src/util/name_table.cpp
View File

@ -1,100 +0,0 @@
#include "util/name_table.hpp"
#include "storage/io.hpp"
#include "util/log.hpp"
namespace osrm
{
namespace util
{
NameTable::NameTable(const std::string &file_name)
{
using FileReader = storage::io::FileReader;
FileReader name_stream_file_reader(file_name, FileReader::VerifyFingerprint);
const auto file_size = name_stream_file_reader.GetSize();
m_buffer = BufferType(static_cast<ValueType *>(::operator new(file_size)),
[](void *ptr) { ::operator delete(ptr); });
name_stream_file_reader.ReadInto<char>(m_buffer.get(), file_size);
m_name_table.reset(m_buffer.get(), m_buffer.get() + file_size);
if (m_name_table.empty())
{
util::Log() << "list of street names is empty in construction of name table from: \""
<< file_name << "\"";
}
}
void NameTable::reset(ValueType *begin, ValueType *end)
{
m_buffer.reset();
m_name_table.reset(begin, end);
}
StringView NameTable::GetNameForID(const NameID id) const
{
if (id == INVALID_NAMEID)
return {};
return m_name_table.at(id + 0);
}
StringView NameTable::GetDestinationsForID(const NameID id) const
{
if (id == INVALID_NAMEID)
return {};
return m_name_table.at(id + 1);
}
StringView NameTable::GetExitsForID(const NameID id) const
{
if (id == INVALID_NAMEID)
return {};
return m_name_table.at(id + 4);
}
StringView NameTable::GetRefForID(const NameID id) const
{
if (id == INVALID_NAMEID)
return {};
// Way string data is stored in blocks based on `id` as follows:
//
// | name | destination | pronunciation | ref | exits
// ^ ^
// [range)
// ^ id + 3
//
// `id + offset` gives us the range of chars.
//
// Offset 0 is name, 1 is destination, 2 is pronunciation, 3 is ref, 4 is exits
// See datafacades and extractor callbacks for details.
const constexpr auto OFFSET_REF = 3u;
return m_name_table.at(id + OFFSET_REF);
}
StringView NameTable::GetPronunciationForID(const NameID id) const
{
if (id == INVALID_NAMEID)
return {};
// Way string data is stored in blocks based on `id` as follows:
//
// | name | destination | pronunciation | ref | exits
// ^ ^
// [range)
// ^ id + 2
//
// `id + offset` gives us the range of chars.
//
// Offset 0 is name, 1 is destination, 2 is pronunciation, 3 is ref, 4 is exits
// See datafacades and extractor callbacks for details.
const constexpr auto OFFSET_PRONUNCIATION = 2u;
return m_name_table.at(id + OFFSET_PRONUNCIATION);
}
} // namespace util
} // namespace osrm

30
unit_tests/util/name_table.cpp → unit_tests/extractor/name_table.cpp
View File

@ -1,4 +1,4 @@
#include "util/name_table.hpp"
#include "extractor/name_table.hpp"
#include "common/temporary_file.hpp"
#include "util/exception.hpp"
@ -12,14 +12,12 @@
#include <typeinfo>
#include <vector>
//#include <valgrind/callgrind.h>
BOOST_AUTO_TEST_SUITE(name_table)
using namespace osrm;
using namespace osrm::util;
using namespace osrm::extractor;
std::string PrapareNameTableData(std::vector<std::string> &data, bool fill_all)
NameTable::IndexedData PrapareNameTableData(std::vector<std::string> &data, bool fill_all)
{
NameTable::IndexedData indexed_data;
std::vector<unsigned char> name_char_data;
@ -60,19 +58,7 @@ std::string PrapareNameTableData(std::vector<std::string> &data, bool fill_all)
}
name_offsets.push_back(name_char_data.size());
TemporaryFile file;
{
storage::io::FileWriter writer(file.path, storage::io::FileWriter::HasNoFingerprint);
indexed_data.write(
writer, name_offsets.begin(), name_offsets.end(), name_char_data.begin());
}
storage::io::FileReader reader(file.path, storage::io::FileReader::HasNoFingerprint);
auto length = reader.GetSize();
std::string str(length, '\0');
reader.ReadInto(const_cast<char *>(str.data()), length);
return str;
return NameTable::IndexedData(name_offsets.begin(), name_offsets.end(), name_char_data.begin());
}
BOOST_AUTO_TEST_CASE(check_name_table_fill)
@ -83,9 +69,7 @@ BOOST_AUTO_TEST_CASE(check_name_table_fill)
"X", "Y", "Z", "", "", "", "", "", "", "", "0", ""};
auto data = PrapareNameTableData(expected_names, true);
NameTable name_table;
name_table.reset(&data[0], &data[data.size()]);
NameTable name_table {data};
for (std::size_t index = 0; index < expected_names.size(); ++index)
{
@ -106,9 +90,7 @@ BOOST_AUTO_TEST_CASE(check_name_table_nofill)
"X", "Y", "Z", "", "", "", "", "", "", "", "0", ""};
auto data = PrapareNameTableData(expected_names, false);
NameTable name_table;
name_table.reset(&data[0], &data[data.size()]);
NameTable name_table {data};
// CALLGRIND_START_INSTRUMENTATION;
for (std::size_t index = 0; index < expected_names.size(); ++index)

70
unit_tests/util/indexed_data.cpp
View File

@ -35,29 +35,7 @@ BOOST_AUTO_TEST_CASE(check_variable_group_block_bitops)
template <typename IndexedData, typename Offsets, typename Data>
void test_rw(const Offsets &offsets, const Data &data)
{
TemporaryFile file;
IndexedData indexed_data;
{
storage::io::FileWriter writer(file.path, storage::io::FileWriter::HasNoFingerprint);
indexed_data.write(writer, offsets.begin(), offsets.end(), data.begin());
}
storage::io::FileReader reader(file.path, storage::io::FileReader::HasNoFingerprint);
auto length = reader.GetSize();
std::string str(length, '\0');
reader.ReadInto(const_cast<char *>(str.data()), length);
#if 0
std::cout << "\n" << typeid(IndexedData).name() << "\nsaved size = " << str.size() << "\n";
for (auto c : str)
std::cout << std::hex << std::setw(2) << std::setfill('0')
<< (int)((unsigned char)c) << " ";
std::cout << std::dec << "\n";
#endif
indexed_data.reset(str.c_str(), str.c_str() + str.size());
IndexedData indexed_data(offsets.begin(), offsets.end(), data.begin());
for (std::size_t index = 0; index < offsets.size() - 1; ++index)
{
@ -151,56 +129,16 @@ BOOST_AUTO_TEST_CASE(check_max_size)
name_offsets = {0, 256};
BOOST_CHECK_THROW(test_fixed(), osrm::util::exception);
name_offsets = {0, 255};
test_fixed();
}
BOOST_AUTO_TEST_CASE(check_corrupted_memory)
{
std::vector<unsigned char> buf;
auto test_variable = [&buf]() {
IndexedData<VariableGroupBlock<16, std::vector<unsigned char>>> indexed_data;
indexed_data.reset(&buf[0], &buf[buf.size()]);
const auto result = indexed_data.at(0);
return std::string(reinterpret_cast<const char *>(&result[0]), result.size());
};
// Use LE internal representation
buf = {0, 42};
BOOST_CHECK_THROW(test_variable(), osrm::util::exception);
buf = {1, 0, 0, 0, 0};
BOOST_CHECK_THROW(test_variable(), osrm::util::exception);
buf = {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42};
BOOST_CHECK_THROW(test_variable(), osrm::util::exception);
buf = {1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 42};
BOOST_CHECK_THROW(test_variable(), osrm::util::exception);
buf = {1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0xF0, 0x9F, 0x90, 0xBC};
BOOST_CHECK_EQUAL(test_variable(), "🐼");
BOOST_CHECK_NO_THROW(test_fixed());
}
BOOST_AUTO_TEST_CASE(check_string_view)
{
TemporaryFile file;
std::string name_data = "hellostringview";
std::vector<std::uint32_t> name_offsets = {0, 5, 11, 15};
IndexedData<VariableGroupBlock<16, StringView>> indexed_data;
{
storage::io::FileWriter writer(file.path, storage::io::FileWriter::HasNoFingerprint);
indexed_data.write(writer, name_offsets.begin(), name_offsets.end(), name_data.begin());
}
storage::io::FileReader reader(file.path, storage::io::FileReader::HasNoFingerprint);
auto length = reader.GetSize();
std::string str(length, '\0');
reader.ReadInto(const_cast<char *>(str.data()), length);
indexed_data.reset(str.c_str(), str.c_str() + str.size());
IndexedData<VariableGroupBlock<16, StringView>> indexed_data(
name_offsets.begin(), name_offsets.end(), name_data.begin());
BOOST_CHECK_EQUAL(indexed_data.at(0), "hello");
BOOST_CHECK_EQUAL(indexed_data.at(1), "string");

40
unit_tests/util/serialization.cpp
View File

@ -71,6 +71,46 @@ BOOST_AUTO_TEST_CASE(tar_serialize_packed_vector)
TestPackedVector result;
storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
util::serialization::read(reader, "my_packed_vector", result);
CHECK_EQUAL_COLLECTIONS(result, v);
}
}
}
BOOST_AUTO_TEST_CASE(tar_serialize_variable_indexed_data)
{
TemporaryFile tmp;
{
using TestIndexedData = IndexedData<VariableGroupBlock<16, std::string>>;
std::vector<std::vector<unsigned>> offset_data = {
{5, 8, 8},
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17},
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
};
std::vector<std::string> char_data = {
"HalloFoo",
"ABCDEFGHIJKLMNOPQR",
"ABCDEFGHIJKLMNOP",
};
for (const auto i : util::irange<std::size_t>(0, offset_data.size()))
{
TestIndexedData indexed {offset_data[i].begin(), offset_data[i].end(), char_data[i].begin()};
{
storage::tar::FileWriter writer(tmp.path,
storage::tar::FileWriter::GenerateFingerprint);
util::serialization::write(writer, "my_indexed_data", indexed);
}
TestIndexedData result;
storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
util::serialization::read(reader, "my_indexed_data", result);
for(auto j : util::irange<std::size_t>(0, offset_data[i].size() - 1))
{
BOOST_CHECK_EQUAL(indexed.at(j), result.at(j));
}
}
}
}