Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Consolidate datafacade file loading logic.

Browse Source 
Both datafacades now use a single large memory block and common
file loading logic.
This commit is contained in:
Daniel Patterson 2016-10-24 21:27:56 -06:00
parent 8f6dd805e5
commit bf6df74d44
10 changed files with 1054 additions and 1753 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/FingerPrint-Config.cmake
View File

@ -4,7 +4,7 @@ set(INFILE ${SOURCE_DIR}/include/util/fingerprint_impl.hpp.in)
file(MD5 ${SOURCE_DIR}/src/tools/contract.cpp MD5PREPARE)
file(MD5 ${SOURCE_DIR}/include/util/static_rtree.hpp MD5RTREE)
file(MD5 ${SOURCE_DIR}/include/util/graph_loader.hpp MD5GRAPH)
file(MD5 ${SOURCE_DIR}/include/engine/datafacade/internal_datafacade.hpp MD5OBJECTS)
file(MD5 ${SOURCE_DIR}/src/storage/storage.cpp MD5OBJECTS)
CONFIGURE_FILE(${INFILE} ${NEWFILE})

2
include/engine/data_watchdog.hpp
View File

@ -1,7 +1,7 @@
#ifndef OSRM_ENGINE_DATA_WATCHDOG_HPP
#define OSRM_ENGINE_DATA_WATCHDOG_HPP
#include "engine/datafacade/shared_datafacade.hpp"
#include "engine/datafacade/shared_memory_datafacade.hpp"
#include "storage/shared_barriers.hpp"
#include "storage/shared_datatype.hpp"

974
include/engine/datafacade/internal_datafacade.hpp
View File

@ -1,974 +0,0 @@
#ifndef INTERNAL_DATAFACADE_HPP
#define INTERNAL_DATAFACADE_HPP
// implements all data storage when shared memory is _NOT_ used
#include "engine/datafacade/datafacade_base.hpp"
#include "extractor/guidance/turn_instruction.hpp"
#include "util/guidance/bearing_class.hpp"
#include "util/guidance/entry_class.hpp"
#include "extractor/compressed_edge_container.hpp"
#include "extractor/original_edge_data.hpp"
#include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp"
#include "storage/io.hpp"
#include "storage/storage_config.hpp"
#include "engine/geospatial_query.hpp"
#include "util/graph_loader.hpp"
#include "util/guidance/turn_bearing.hpp"
#include "util/guidance/turn_lanes.hpp"
#include "util/io.hpp"
#include "util/packed_vector.hpp"
#include "util/range_table.hpp"
#include "util/rectangle.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/simple_logger.hpp"
#include "util/static_graph.hpp"
#include "util/static_rtree.hpp"
#include "util/typedefs.hpp"
#include "osrm/coordinate.hpp"
#include <cstddef>
#include <cstdlib>
#include <algorithm>
#include <fstream>
#include <ios>
#include <limits>
#include <memory>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include <boost/assert.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/thread/tss.hpp>
namespace osrm
{
namespace engine
{
namespace datafacade
{
class InternalDataFacade final : public BaseDataFacade
{
private:
using super = BaseDataFacade;
using QueryGraph = util::StaticGraph<typename super::EdgeData>;
using InputEdge = QueryGraph::InputEdge;
using RTreeLeaf = super::RTreeLeaf;
using InternalRTree =
util::StaticRTree<RTreeLeaf, util::ShM<util::Coordinate, false>::vector, false>;
using InternalGeospatialQuery = GeospatialQuery<InternalRTree, BaseDataFacade>;
InternalDataFacade() {}
unsigned m_check_sum;
std::unique_ptr<QueryGraph> m_query_graph;
std::string m_timestamp;
util::ShM<util::Coordinate, false>::vector m_coordinate_list;
util::PackedVector<OSMNodeID, false> m_osmnodeid_list;
util::ShM<GeometryID, false>::vector m_via_geometry_list;
util::ShM<unsigned, false>::vector m_name_ID_list;
util::ShM<extractor::guidance::TurnInstruction, false>::vector m_turn_instruction_list;
util::ShM<LaneDataID, false>::vector m_lane_data_id;
util::ShM<util::guidance::LaneTupleIdPair, false>::vector m_lane_tuple_id_pairs;
util::ShM<extractor::TravelMode, false>::vector m_travel_mode_list;
util::ShM<char, false>::vector m_names_char_list;
util::ShM<unsigned, false>::vector m_geometry_indices;
util::ShM<NodeID, false>::vector m_geometry_node_list;
util::ShM<EdgeWeight, false>::vector m_geometry_fwd_weight_list;
util::ShM<EdgeWeight, false>::vector m_geometry_rev_weight_list;
util::ShM<bool, false>::vector m_is_core_node;
util::ShM<uint8_t, false>::vector m_datasource_list;
util::ShM<std::string, false>::vector m_datasource_names;
util::ShM<std::uint32_t, false>::vector m_lane_description_offsets;
util::ShM<extractor::guidance::TurnLaneType::Mask, false>::vector m_lane_description_masks;
extractor::ProfileProperties m_profile_properties;
std::unique_ptr<InternalRTree> m_static_rtree;
std::unique_ptr<InternalGeospatialQuery> m_geospatial_query;
boost::filesystem::path ram_index_path;
boost::filesystem::path file_index_path;
util::RangeTable<16, false> m_name_table;
// bearing classes by node based node
util::ShM<BearingClassID, false>::vector m_bearing_class_id_table;
// entry class IDs by edge based egde
util::ShM<EntryClassID, false>::vector m_entry_class_id_list;
// bearings pre/post turn
util::ShM<util::guidance::TurnBearing, false>::vector m_pre_turn_bearing;
util::ShM<util::guidance::TurnBearing, false>::vector m_post_turn_bearing;
// the look-up table for entry classes. An entry class lists the possibility of entry for all
// available turns. For every turn, there is an associated entry class.
util::ShM<util::guidance::EntryClass, false>::vector m_entry_class_table;
// the look-up table for distinct bearing classes. A bearing class lists the available bearings
// at an intersection
util::RangeTable<16, false> m_bearing_ranges_table;
util::ShM<DiscreteBearing, false>::vector m_bearing_values_table;
void LoadProfileProperties(const boost::filesystem::path &properties_path)
{
boost::filesystem::ifstream in_stream(properties_path);
if (!in_stream)
{
throw util::exception("Could not open " + properties_path.string() + " for reading.");
}
const auto properties_size = storage::io::readPropertiesCount();
storage::io::readProperties(in_stream, &m_profile_properties, properties_size);
}
void LoadLaneTupleIdPairs(const boost::filesystem::path &lane_data_path)
{
boost::filesystem::ifstream in_stream(lane_data_path);
if (!in_stream)
{
throw util::exception("Could not open " + lane_data_path.string() + " for reading.");
}
std::uint64_t size;
in_stream.read(reinterpret_cast<char *>(&size), sizeof(size));
m_lane_tuple_id_pairs.resize(size);
in_stream.read(reinterpret_cast<char *>(&m_lane_tuple_id_pairs[0]),
sizeof(m_lane_tuple_id_pairs) * size);
}
void LoadTimestamp(const boost::filesystem::path &timestamp_path)
{
util::SimpleLogger().Write() << "Loading Timestamp";
boost::filesystem::ifstream timestamp_stream(timestamp_path);
if (!timestamp_stream)
{
throw util::exception("Could not open " + timestamp_path.string() + " for reading.");
}
const auto timestamp_size = storage::io::readNumberOfBytes(timestamp_stream);
m_timestamp.resize(timestamp_size);
storage::io::readTimestamp(timestamp_stream, &m_timestamp.front(), timestamp_size);
}
void LoadGraph(const boost::filesystem::path &hsgr_path)
{
boost::filesystem::ifstream hsgr_input_stream(hsgr_path);
if (!hsgr_input_stream)
{
throw util::exception("Could not open " + hsgr_path.string() + " for reading.");
}
const auto header = storage::io::readHSGRHeader(hsgr_input_stream);
m_check_sum = header.checksum;
util::ShM<QueryGraph::NodeArrayEntry, false>::vector node_list(header.number_of_nodes);
util::ShM<QueryGraph::EdgeArrayEntry, false>::vector edge_list(header.number_of_edges);
storage::io::readHSGR(hsgr_input_stream,
node_list.data(),
header.number_of_nodes,
edge_list.data(),
header.number_of_edges);
m_query_graph = std::unique_ptr<QueryGraph>(new QueryGraph(node_list, edge_list));
util::SimpleLogger().Write() << "Data checksum is " << m_check_sum;
}
void LoadNodeAndEdgeInformation(const boost::filesystem::path &nodes_file_path,
const boost::filesystem::path &edges_file_path)
{
boost::filesystem::ifstream nodes_input_stream(nodes_file_path, std::ios::binary);
if (!nodes_input_stream)
{
throw util::exception("Could not open " + nodes_file_path.string() + " for reading.");
}
const auto number_of_coordinates = storage::io::readElementCount(nodes_input_stream);
m_coordinate_list.resize(number_of_coordinates);
m_osmnodeid_list.reserve(number_of_coordinates);
storage::io::readNodes(
nodes_input_stream, m_coordinate_list.data(), m_osmnodeid_list, number_of_coordinates);
boost::filesystem::ifstream edges_input_stream(edges_file_path, std::ios::binary);
if (!edges_input_stream)
{
throw util::exception("Could not open " + edges_file_path.string() + " for reading.");
}
const auto number_of_edges = storage::io::readElementCount(edges_input_stream);
m_via_geometry_list.resize(number_of_edges);
m_name_ID_list.resize(number_of_edges);
m_turn_instruction_list.resize(number_of_edges);
m_lane_data_id.resize(number_of_edges);
m_travel_mode_list.resize(number_of_edges);
m_entry_class_id_list.resize(number_of_edges);
m_pre_turn_bearing.resize(number_of_edges);
m_post_turn_bearing.resize(number_of_edges);
storage::io::readEdges(edges_input_stream,
m_via_geometry_list.data(),
m_name_ID_list.data(),
m_turn_instruction_list.data(),
m_lane_data_id.data(),
m_travel_mode_list.data(),
m_entry_class_id_list.data(),
m_pre_turn_bearing.data(),
m_post_turn_bearing.data(),
number_of_edges);
}
void LoadCoreInformation(const boost::filesystem::path &core_data_file)
{
std::ifstream core_stream(core_data_file.string().c_str(), std::ios::binary);
unsigned number_of_markers;
core_stream.read((char *)&number_of_markers, sizeof(unsigned));
std::vector<char> unpacked_core_markers(number_of_markers);
core_stream.read((char *)unpacked_core_markers.data(), sizeof(char) * number_of_markers);
// in this case we have nothing to do
if (number_of_markers <= 0)
{
return;
}
m_is_core_node.resize(number_of_markers);
for (auto i = 0u; i < number_of_markers; ++i)
{
BOOST_ASSERT(unpacked_core_markers[i] == 0 || unpacked_core_markers[i] == 1);
m_is_core_node[i] = unpacked_core_markers[i] == 1;
}
}
void LoadGeometries(const boost::filesystem::path &geometry_file)
{
std::ifstream geometry_stream(geometry_file.string().c_str(), std::ios::binary);
unsigned number_of_indices = 0;
unsigned number_of_compressed_geometries = 0;
geometry_stream.read((char *)&number_of_indices, sizeof(unsigned));
m_geometry_indices.resize(number_of_indices);
if (number_of_indices > 0)
{
geometry_stream.read((char *)&(m_geometry_indices[0]),
number_of_indices * sizeof(unsigned));
}
geometry_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned));
BOOST_ASSERT(m_geometry_indices.back() == number_of_compressed_geometries);
m_geometry_node_list.resize(number_of_compressed_geometries);
m_geometry_fwd_weight_list.resize(number_of_compressed_geometries);
m_geometry_rev_weight_list.resize(number_of_compressed_geometries);
if (number_of_compressed_geometries > 0)
{
geometry_stream.read((char *)&(m_geometry_node_list[0]),
number_of_compressed_geometries * sizeof(NodeID));
geometry_stream.read((char *)&(m_geometry_fwd_weight_list[0]),
number_of_compressed_geometries * sizeof(EdgeWeight));
geometry_stream.read((char *)&(m_geometry_rev_weight_list[0]),
number_of_compressed_geometries * sizeof(EdgeWeight));
}
}
void LoadDatasourceInfo(const boost::filesystem::path &datasource_names_file,
const boost::filesystem::path &datasource_indexes_file)
{
boost::filesystem::ifstream datasources_stream(datasource_indexes_file, std::ios::binary);
if (!datasources_stream)
{
throw util::exception("Could not open " + datasource_indexes_file.string() +
" for reading!");
}
BOOST_ASSERT(datasources_stream);
const auto number_of_datasources = storage::io::readElementCount(datasources_stream);
if (number_of_datasources > 0)
{
m_datasource_list.resize(number_of_datasources);
storage::io::readDatasourceIndexes(
datasources_stream, &m_datasource_list.front(), number_of_datasources);
}
boost::filesystem::ifstream datasourcenames_stream(datasource_names_file, std::ios::binary);
if (!datasourcenames_stream)
{
throw util::exception("Could not open " + datasource_names_file.string() +
" for reading!");
}
BOOST_ASSERT(datasourcenames_stream);
const auto datasource_names_data = storage::io::readDatasourceNames(datasourcenames_stream);
m_datasource_names.resize(datasource_names_data.lengths.size());
for (std::size_t i = 0; i < datasource_names_data.lengths.size(); ++i)
{
auto name_begin =
datasource_names_data.names.begin() + datasource_names_data.offsets[i];
auto name_end = datasource_names_data.names.begin() + datasource_names_data.offsets[i] +
datasource_names_data.lengths[i];
m_datasource_names[i] = std::string(name_begin, name_end);
}
}
void LoadRTree()
{
BOOST_ASSERT_MSG(!m_coordinate_list.empty(), "coordinates must be loaded before r-tree");
m_static_rtree.reset(new InternalRTree(ram_index_path, file_index_path, m_coordinate_list));
m_geospatial_query.reset(
new InternalGeospatialQuery(*m_static_rtree, m_coordinate_list, *this));
}
void LoadLaneDescriptions(const boost::filesystem::path &lane_description_file)
{
if (!util::deserializeAdjacencyArray(lane_description_file.string(),
m_lane_description_offsets,
m_lane_description_masks))
util::SimpleLogger().Write(logWARNING) << "Failed to read turn lane descriptions from "
<< lane_description_file.string();
}
void LoadStreetNames(const boost::filesystem::path &names_file)
{
boost::filesystem::ifstream name_stream(names_file, std::ios::binary);
name_stream >> m_name_table;
unsigned number_of_chars = 0;
name_stream.read((char *)&number_of_chars, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != number_of_chars, "name file broken");
m_names_char_list.resize(number_of_chars + 1); //+1 gives sentinel element
name_stream.read((char *)&m_names_char_list[0], number_of_chars * sizeof(char));
if (0 == m_names_char_list.size())
{
util::SimpleLogger().Write(logWARNING) << "list of street names is empty";
}
}
void LoadIntersectionClasses(const boost::filesystem::path &intersection_class_file)
{
std::ifstream intersection_stream(intersection_class_file.string(), std::ios::binary);
if (!intersection_stream)
throw util::exception("Could not open " + intersection_class_file.string() +
" for reading.");
if (!util::readAndCheckFingerprint(intersection_stream))
throw util::exception("Fingeprint does not match in " +
intersection_class_file.string());
{
util::SimpleLogger().Write(logINFO) << "Loading Bearing Class IDs";
std::vector<BearingClassID> bearing_class_id;
if (!util::deserializeVector(intersection_stream, bearing_class_id))
throw util::exception("Reading from " + intersection_class_file.string() +
" failed.");
m_bearing_class_id_table.resize(bearing_class_id.size());
std::copy(
bearing_class_id.begin(), bearing_class_id.end(), &m_bearing_class_id_table[0]);
}
{
util::SimpleLogger().Write(logINFO) << "Loading Bearing Classes";
// read the range table
intersection_stream >> m_bearing_ranges_table;
std::vector<util::guidance::BearingClass> bearing_classes;
// and the actual bearing values
std::uint64_t num_bearings;
intersection_stream.read(reinterpret_cast<char *>(&num_bearings), sizeof(num_bearings));
m_bearing_values_table.resize(num_bearings);
intersection_stream.read(reinterpret_cast<char *>(&m_bearing_values_table[0]),
sizeof(m_bearing_values_table[0]) * num_bearings);
if (!static_cast<bool>(intersection_stream))
throw util::exception("Reading from " + intersection_class_file.string() +
" failed.");
}
{
util::SimpleLogger().Write(logINFO) << "Loading Entry Classes";
std::vector<util::guidance::EntryClass> entry_classes;
if (!util::deserializeVector(intersection_stream, entry_classes))
throw util::exception("Reading from " + intersection_class_file.string() +
" failed.");
m_entry_class_table.resize(entry_classes.size());
std::copy(entry_classes.begin(), entry_classes.end(), &m_entry_class_table[0]);
}
}
public:
virtual ~InternalDataFacade()
{
m_static_rtree.reset();
m_geospatial_query.reset();
}
explicit InternalDataFacade(const storage::StorageConfig &config)
{
ram_index_path = config.ram_index_path;
file_index_path = config.file_index_path;
util::SimpleLogger().Write() << "loading graph data";
LoadGraph(config.hsgr_data_path);
util::SimpleLogger().Write() << "loading edge information";
LoadNodeAndEdgeInformation(config.nodes_data_path, config.edges_data_path);
util::SimpleLogger().Write() << "loading core information";
LoadCoreInformation(config.core_data_path);
util::SimpleLogger().Write() << "loading geometries";
LoadGeometries(config.geometries_path);
util::SimpleLogger().Write() << "loading datasource info";
LoadDatasourceInfo(config.datasource_names_path, config.datasource_indexes_path);
util::SimpleLogger().Write() << "loading timestamp";
LoadTimestamp(config.timestamp_path);
util::SimpleLogger().Write() << "loading profile properties";
LoadProfileProperties(config.properties_path);
util::SimpleLogger().Write() << "loading street names";
LoadStreetNames(config.names_data_path);
util::SimpleLogger().Write() << "loading lane tags";
LoadLaneDescriptions(config.turn_lane_description_path);
util::SimpleLogger().Write() << "loading rtree";
LoadRTree();
util::SimpleLogger().Write() << "loading intersection class data";
LoadIntersectionClasses(config.intersection_class_path);
util::SimpleLogger().Write() << "Loading Lane Data Pairs";
LoadLaneTupleIdPairs(config.turn_lane_data_path);
}
// search graph access
unsigned GetNumberOfNodes() const override final { return m_query_graph->GetNumberOfNodes(); }
unsigned GetNumberOfEdges() const override final { return m_query_graph->GetNumberOfEdges(); }
unsigned GetOutDegree(const NodeID n) const override final
{
return m_query_graph->GetOutDegree(n);
}
NodeID GetTarget(const EdgeID e) const override final { return m_query_graph->GetTarget(e); }
EdgeData &GetEdgeData(const EdgeID e) const override final
{
return m_query_graph->GetEdgeData(e);
}
EdgeID BeginEdges(const NodeID n) const override final { return m_query_graph->BeginEdges(n); }
EdgeID EndEdges(const NodeID n) const override final { return m_query_graph->EndEdges(n); }
EdgeRange GetAdjacentEdgeRange(const NodeID node) const override final
{
return m_query_graph->GetAdjacentEdgeRange(node);
}
// searches for a specific edge
EdgeID FindEdge(const NodeID from, const NodeID to) const override final
{
return m_query_graph->FindEdge(from, to);
}
EdgeID FindEdgeInEitherDirection(const NodeID from, const NodeID to) const override final
{
return m_query_graph->FindEdgeInEitherDirection(from, to);
}
EdgeID
FindEdgeIndicateIfReverse(const NodeID from, const NodeID to, bool &result) const override final
{
return m_query_graph->FindEdgeIndicateIfReverse(from, to, result);
}
EdgeID FindSmallestEdge(const NodeID from,
const NodeID to,
std::function<bool(EdgeData)> filter) const override final
{
return m_query_graph->FindSmallestEdge(from, to, filter);
}
// node and edge information access
util::Coordinate GetCoordinateOfNode(const unsigned id) const override final
{
return m_coordinate_list[id];
}
OSMNodeID GetOSMNodeIDOfNode(const unsigned id) const override final
{
return m_osmnodeid_list.at(id);
}
extractor::guidance::TurnInstruction
GetTurnInstructionForEdgeID(const unsigned id) const override final
{
return m_turn_instruction_list.at(id);
}
extractor::TravelMode GetTravelModeForEdgeID(const unsigned id) const override final
{
return m_travel_mode_list.at(id);
}
std::vector<RTreeLeaf> GetEdgesInBox(const util::Coordinate south_west,
const util::Coordinate north_east) const override final
{
BOOST_ASSERT(m_geospatial_query.get());
const util::RectangleInt2D bbox{
south_west.lon, north_east.lon, south_west.lat, north_east.lat};
return m_geospatial_query->Search(bbox);
}
std::vector<PhantomNodeWithDistance>
NearestPhantomNodesInRange(const util::Coordinate input_coordinate,
const float max_distance) const override final
{
BOOST_ASSERT(m_geospatial_query.get());
return m_geospatial_query->NearestPhantomNodesInRange(input_coordinate, max_distance);
}
std::vector<PhantomNodeWithDistance>
NearestPhantomNodesInRange(const util::Coordinate input_coordinate,
const float max_distance,
const int bearing,
const int bearing_range) const override final
{
BOOST_ASSERT(m_geospatial_query.get());
return m_geospatial_query->NearestPhantomNodesInRange(
input_coordinate, max_distance, bearing, bearing_range);
}
std::vector<PhantomNodeWithDistance>
NearestPhantomNodes(const util::Coordinate input_coordinate,
const unsigned max_results) const override final
{
BOOST_ASSERT(m_geospatial_query.get());
return m_geospatial_query->NearestPhantomNodes(input_coordinate, max_results);
}
std::vector<PhantomNodeWithDistance>
NearestPhantomNodes(const util::Coordinate input_coordinate,
const unsigned max_results,
const double max_distance) const override final
{
BOOST_ASSERT(m_geospatial_query.get());
return m_geospatial_query->NearestPhantomNodes(input_coordinate, max_results, max_distance);
}
std::vector<PhantomNodeWithDistance>
NearestPhantomNodes(const util::Coordinate input_coordinate,
const unsigned max_results,
const int bearing,
const int bearing_range) const override final
{
BOOST_ASSERT(m_geospatial_query.get());
return m_geospatial_query->NearestPhantomNodes(
input_coordinate, max_results, bearing, bearing_range);
}
std::vector<PhantomNodeWithDistance>
NearestPhantomNodes(const util::Coordinate input_coordinate,
const unsigned max_results,
const double max_distance,
const int bearing,
const int bearing_range) const override final
{
BOOST_ASSERT(m_geospatial_query.get());
return m_geospatial_query->NearestPhantomNodes(
input_coordinate, max_results, max_distance, bearing, bearing_range);
}
std::pair<PhantomNode, PhantomNode> NearestPhantomNodeWithAlternativeFromBigComponent(
const util::Coordinate input_coordinate, const double max_distance) const override final
{
BOOST_ASSERT(m_geospatial_query.get());
return m_geospatial_query->NearestPhantomNodeWithAlternativeFromBigComponent(
input_coordinate, max_distance);
}
std::pair<PhantomNode, PhantomNode> NearestPhantomNodeWithAlternativeFromBigComponent(
const util::Coordinate input_coordinate) const override final
{
BOOST_ASSERT(m_geospatial_query.get());
return m_geospatial_query->NearestPhantomNodeWithAlternativeFromBigComponent(
input_coordinate);
}
std::pair<PhantomNode, PhantomNode>
NearestPhantomNodeWithAlternativeFromBigComponent(const util::Coordinate input_coordinate,
const double max_distance,
const int bearing,
const int bearing_range) const override final
{
BOOST_ASSERT(m_geospatial_query.get());
return m_geospatial_query->NearestPhantomNodeWithAlternativeFromBigComponent(
input_coordinate, max_distance, bearing, bearing_range);
}
std::pair<PhantomNode, PhantomNode>
NearestPhantomNodeWithAlternativeFromBigComponent(const util::Coordinate input_coordinate,
const int bearing,
const int bearing_range) const override final
{
BOOST_ASSERT(m_geospatial_query.get());
return m_geospatial_query->NearestPhantomNodeWithAlternativeFromBigComponent(
input_coordinate, bearing, bearing_range);
}
unsigned GetCheckSum() const override final { return m_check_sum; }
unsigned GetNameIndexFromEdgeID(const unsigned id) const override final
{
return m_name_ID_list.at(id);
}
std::string GetNameForID(const unsigned name_id) const override final
{
if (std::numeric_limits<unsigned>::max() == name_id)
{
return "";
}
auto range = m_name_table.GetRange(name_id);
std::string result;
result.reserve(range.size());
if (range.begin() != range.end())
{
result.resize(range.back() - range.front() + 1);
std::copy(m_names_char_list.begin() + range.front(),
m_names_char_list.begin() + range.back() + 1,
result.begin());
}
return result;
}
std::string GetRefForID(const unsigned name_id) const override final
{
// We store the ref after the name, destination and pronunciation of a street.
// We do this to get around the street length limit of 255 which would hit
// if we concatenate these. Order (see extractor_callbacks):
// name (0), destination (1), pronunciation (2), ref (3)
return GetNameForID(name_id + 3);
}
std::string GetPronunciationForID(const unsigned name_id) const override final
{
// We store the pronunciation after the name and destination of a street.
// We do this to get around the street length limit of 255 which would hit
// if we concatenate these. Order (see extractor_callbacks):
// name (0), destination (1), pronunciation (2)
return GetNameForID(name_id + 2);
}
std::string GetDestinationsForID(const unsigned name_id) const override final
{
// We store the destination after the name of a street.
// We do this to get around the street length limit of 255 which would hit
// if we concatenate these. Order (see extractor_callbacks):
// name (0), destination (1), pronunciation (2)
return GetNameForID(name_id + 1);
}
virtual GeometryID GetGeometryIndexForEdgeID(const unsigned id) const override final
{
return m_via_geometry_list.at(id);
}
virtual std::size_t GetCoreSize() const override final { return m_is_core_node.size(); }
virtual bool IsCoreNode(const NodeID id) const override final
{
if (m_is_core_node.size() > 0)
{
return m_is_core_node[id];
}
else
{
return false;
}
}
virtual std::vector<NodeID> GetUncompressedForwardGeometry(const EdgeID id) const override final
{
/*
* NodeID's for geometries are stored in one place for
* both forward and reverse segments along the same bi-
* directional edge. The m_geometry_indices stores
* refences to where to find the beginning of the bi-
* directional edge in the m_geometry_node_list vector. For
* forward geometries of bi-directional edges, edges 2 to
* n of that edge need to be read.
*/
const unsigned begin = m_geometry_indices.at(id);
const unsigned end = m_geometry_indices.at(id + 1);
std::vector<NodeID> result_nodes;
result_nodes.resize(end - begin);
std::copy(m_geometry_node_list.begin() + begin,
m_geometry_node_list.begin() + end,
result_nodes.begin());
return result_nodes;
}
virtual std::vector<NodeID> GetUncompressedReverseGeometry(const EdgeID id) const override final
{
/*
* NodeID's for geometries are stored in one place for
* both forward and reverse segments along the same bi-
* directional edge. The m_geometry_indices stores
* refences to where to find the beginning of the bi-
* directional edge in the m_geometry_node_list vector.
* */
const unsigned begin = m_geometry_indices.at(id);
const unsigned end = m_geometry_indices.at(id + 1);
std::vector<NodeID> result_nodes;
result_nodes.resize(end - begin);
std::copy(m_geometry_node_list.rbegin() + (m_geometry_node_list.size() - end),
m_geometry_node_list.rbegin() + (m_geometry_node_list.size() - begin),
result_nodes.begin());
return result_nodes;
}
virtual std::vector<EdgeWeight>
GetUncompressedForwardWeights(const EdgeID id) const override final
{
/*
* EdgeWeights's for geometries are stored in one place for
* both forward and reverse segments along the same bi-
* directional edge. The m_geometry_indices stores
* refences to where to find the beginning of the bi-
* directional edge in the m_geometry_fwd_weight_list vector.
* */
const unsigned begin = m_geometry_indices.at(id) + 1;
const unsigned end = m_geometry_indices.at(id + 1);
std::vector<EdgeWeight> result_weights;
result_weights.resize(end - begin);
std::copy(m_geometry_fwd_weight_list.begin() + begin,
m_geometry_fwd_weight_list.begin() + end,
result_weights.begin());
return result_weights;
}
virtual std::vector<EdgeWeight>
GetUncompressedReverseWeights(const EdgeID id) const override final
{
/*
* EdgeWeights for geometries are stored in one place for
* both forward and reverse segments along the same bi-
* directional edge. The m_geometry_indices stores
* refences to where to find the beginning of the bi-
* directional edge in the m_geometry_rev_weight_list vector. For
* reverse weights of bi-directional edges, edges 1 to
* n-1 of that edge need to be read in reverse.
*/
const unsigned begin = m_geometry_indices.at(id);
const unsigned end = m_geometry_indices.at(id + 1) - 1;
std::vector<EdgeWeight> result_weights;
result_weights.resize(end - begin);
std::copy(m_geometry_rev_weight_list.rbegin() + (m_geometry_rev_weight_list.size() - end),
m_geometry_rev_weight_list.rbegin() + (m_geometry_rev_weight_list.size() - begin),
result_weights.begin());
return result_weights;
}
// Returns the data source ids that were used to supply the edge
// weights.
virtual std::vector<uint8_t>
GetUncompressedForwardDatasources(const EdgeID id) const override final
{
/*
* Data sources for geometries are stored in one place for
* both forward and reverse segments along the same bi-
* directional edge. The m_geometry_indices stores
* refences to where to find the beginning of the bi-
* directional edge in the m_geometry_list vector. For
* forward datasources of bi-directional edges, edges 2 to
* n of that edge need to be read.
*/
const unsigned begin = m_geometry_indices.at(id) + 1;
const unsigned end = m_geometry_indices.at(id + 1);
std::vector<uint8_t> result_datasources;
result_datasources.resize(end - begin);
// If there was no datasource info, return an array of 0's.
if (m_datasource_list.empty())
{
for (unsigned i = 0; i < end - begin; ++i)
{
result_datasources.push_back(0);
}
}
else
{
std::copy(m_datasource_list.begin() + begin,
m_datasource_list.begin() + end,
result_datasources.begin());
}
return result_datasources;
}
// Returns the data source ids that were used to supply the edge
// weights.
virtual std::vector<uint8_t>
GetUncompressedReverseDatasources(const EdgeID id) const override final
{
/*
* Datasources for geometries are stored in one place for
* both forward and reverse segments along the same bi-
* directional edge. The m_geometry_indices stores
* refences to where to find the beginning of the bi-
* directional edge in the m_geometry_list vector. For
* reverse datasources of bi-directional edges, edges 1 to
* n-1 of that edge need to be read in reverse.
*/
const unsigned begin = m_geometry_indices.at(id);
const unsigned end = m_geometry_indices.at(id + 1) - 1;
std::vector<uint8_t> result_datasources;
result_datasources.resize(end - begin);
// If there was no datasource info, return an array of 0's.
if (m_datasource_list.empty())
{
for (unsigned i = 0; i < end - begin; ++i)
{
result_datasources.push_back(0);
}
}
else
{
std::copy(m_datasource_list.rbegin() + (m_datasource_list.size() - end),
m_datasource_list.rbegin() + (m_datasource_list.size() - begin),
result_datasources.begin());
}
return result_datasources;
}
virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const override final
{
BOOST_ASSERT(m_datasource_names.size() >= 1);
BOOST_ASSERT(m_datasource_names.size() > datasource_name_id);
return m_datasource_names[datasource_name_id];
}
std::string GetTimestamp() const override final { return m_timestamp; }
bool GetContinueStraightDefault() const override final
{
return m_profile_properties.continue_straight_at_waypoint;
}
double GetMapMatchingMaxSpeed() const override final
{
return m_profile_properties.max_speed_for_map_matching;
}
BearingClassID GetBearingClassID(const NodeID nid) const override final
{
return m_bearing_class_id_table.at(nid);
}
util::guidance::BearingClass
GetBearingClass(const BearingClassID bearing_class_id) const override final
{
BOOST_ASSERT(bearing_class_id != INVALID_BEARING_CLASSID);
auto range = m_bearing_ranges_table.GetRange(bearing_class_id);
util::guidance::BearingClass result;
for (auto itr = m_bearing_values_table.begin() + range.front();
itr != m_bearing_values_table.begin() + range.back() + 1;
++itr)
result.add(*itr);
return result;
}
EntryClassID GetEntryClassID(const EdgeID eid) const override final
{
return m_entry_class_id_list.at(eid);
}
util::guidance::TurnBearing PreTurnBearing(const EdgeID eid) const override final
{
return m_pre_turn_bearing.at(eid);
}
util::guidance::TurnBearing PostTurnBearing(const EdgeID eid) const override final
{
return m_post_turn_bearing.at(eid);
}
util::guidance::EntryClass GetEntryClass(const EntryClassID entry_class_id) const override final
{
return m_entry_class_table.at(entry_class_id);
}
bool hasLaneData(const EdgeID id) const override final
{
return m_lane_data_id[id] != INVALID_LANE_DATAID;
}
util::guidance::LaneTupleIdPair GetLaneData(const EdgeID id) const override final
{
BOOST_ASSERT(hasLaneData(id));
return m_lane_tuple_id_pairs[m_lane_data_id[id]];
}
extractor::guidance::TurnLaneDescription
GetTurnDescription(const LaneDescriptionID lane_description_id) const override final
{
if (lane_description_id == INVALID_LANE_DESCRIPTIONID)
return {};
else
return extractor::guidance::TurnLaneDescription(
m_lane_description_masks.begin() + m_lane_description_offsets[lane_description_id],
m_lane_description_masks.begin() +
m_lane_description_offsets[lane_description_id + 1]);
}
};
}
}
}
#endif // INTERNAL_DATAFACADE_HPP

56
include/engine/datafacade/internal_memory_datafacade.hpp Normal file
View File

@ -0,0 +1,56 @@
#ifndef INTERNAL_DATAFACADE_HPP
#define INTERNAL_DATAFACADE_HPP
// implements all data storage when shared memory is _NOT_ used
#include "engine/datafacade/memory_datafacade_base.hpp"
#include "storage/storage.hpp"
namespace osrm
{
namespace engine
{
namespace datafacade
{
/**
* This datafacade uses a process-local memory block to load
* data into. The structure and layout is the same as when using
* shared memory, so this class, and the SharedDatafacade both
* share a common base.
* This class holds a unique_ptr to the memory block, so it
* is auto-freed upon destruction.
*/
class InternalDataFacade final : public MemoryDataFacadeBase
{
private:
std::unique_ptr<char[]> internal_memory;
std::unique_ptr<storage::DataLayout> internal_layout;
public:
explicit InternalDataFacade(const storage::StorageConfig &config)
{
storage::Storage storage(config);
// Calculate the layout/size of the memory block
internal_layout = std::make_unique<storage::DataLayout>();
storage.LoadLayout(internal_layout.get());
// Allocate the memory block, then load data from files into it
internal_memory.reset(new char[internal_layout->GetSizeOfLayout()]);
storage.LoadData(internal_layout.get(), internal_memory.get());
// Set the common base-class pointers
data_layout = internal_layout.get();
memory_block = internal_memory.get();
// Adjust all the private m_* members to point to the right places
Init();
}
};
}
}
}
#endif // INTERNAL_DATAFACADE_HPP

310
include/engine/datafacade/shared_datafacade.hpp → include/engine/datafacade/memory_datafacade_base.hpp
View File

@ -3,9 +3,6 @@
// implements all data storage when shared memory _IS_ used
#include "storage/shared_barriers.hpp"
#include "storage/shared_datatype.hpp"
#include "storage/shared_memory.hpp"
#include "engine/datafacade/datafacade_base.hpp"
#include "extractor/compressed_edge_container.hpp"
@ -47,10 +44,10 @@ namespace engine
namespace datafacade
{
class SharedDataFacade final : public BaseDataFacade
class MemoryDataFacadeBase : public BaseDataFacade
{
private:
protected:
using super = BaseDataFacade;
using QueryGraph = util::StaticGraph<EdgeData, true>;
using GraphNode = QueryGraph::NodeArrayEntry;
@ -63,13 +60,8 @@ class SharedDataFacade final : public BaseDataFacade
using SharedGeospatialQuery = GeospatialQuery<SharedRTree, BaseDataFacade>;
using RTreeNode = SharedRTree::TreeNode;
storage::SharedDataLayout *data_layout;
char *shared_memory;
std::shared_ptr<storage::SharedBarriers> shared_barriers;
storage::SharedDataType layout_region;
storage::SharedDataType data_region;
unsigned shared_timestamp;
storage::DataLayout *data_layout;
char *memory_block;
unsigned m_check_sum;
std::unique_ptr<QueryGraph> m_query_graph;
@ -121,35 +113,35 @@ class SharedDataFacade final : public BaseDataFacade
std::shared_ptr<util::RangeTable<16, true>> m_bearing_ranges_table;
util::ShM<DiscreteBearing, true>::vector m_bearing_values_table;
void LoadChecksum()
void InitChecksum()
{
m_check_sum = *data_layout->GetBlockPtr<unsigned>(shared_memory,
storage::SharedDataLayout::HSGR_CHECKSUM);
m_check_sum =
*data_layout->GetBlockPtr<unsigned>(memory_block, storage::DataLayout::HSGR_CHECKSUM);
util::SimpleLogger().Write() << "set checksum: " << m_check_sum;
}
void LoadProfileProperties()
void InitProfileProperties()
{
m_profile_properties = data_layout->GetBlockPtr<extractor::ProfileProperties>(
shared_memory, storage::SharedDataLayout::PROPERTIES);
memory_block, storage::DataLayout::PROPERTIES);
}
void LoadTimestamp()
void InitTimestamp()
{
auto timestamp_ptr =
data_layout->GetBlockPtr<char>(shared_memory, storage::SharedDataLayout::TIMESTAMP);
m_timestamp.resize(data_layout->GetBlockSize(storage::SharedDataLayout::TIMESTAMP));
data_layout->GetBlockPtr<char>(memory_block, storage::DataLayout::TIMESTAMP);
m_timestamp.resize(data_layout->GetBlockSize(storage::DataLayout::TIMESTAMP));
std::copy(timestamp_ptr,
timestamp_ptr + data_layout->GetBlockSize(storage::SharedDataLayout::TIMESTAMP),
timestamp_ptr + data_layout->GetBlockSize(storage::DataLayout::TIMESTAMP),
m_timestamp.begin());
}
void LoadRTree()
void InitRTree()
{
BOOST_ASSERT_MSG(!m_coordinate_list.empty(), "coordinates must be loaded before r-tree");
const auto file_index_ptr = data_layout->GetBlockPtr<char>(
shared_memory, storage::SharedDataLayout::FILE_INDEX_PATH);
const auto file_index_ptr =
data_layout->GetBlockPtr<char>(memory_block, storage::DataLayout::FILE_INDEX_PATH);
file_index_path = boost::filesystem::path(file_index_ptr);
if (!boost::filesystem::exists(file_index_path))
{
@ -158,39 +150,38 @@ class SharedDataFacade final : public BaseDataFacade
"Is any data loaded into shared memory?");
}
auto tree_ptr = data_layout->GetBlockPtr<RTreeNode>(
shared_memory, storage::SharedDataLayout::R_SEARCH_TREE);
auto tree_ptr =
data_layout->GetBlockPtr<RTreeNode>(memory_block, storage::DataLayout::R_SEARCH_TREE);
m_static_rtree.reset(
new SharedRTree(tree_ptr,
data_layout->num_entries[storage::SharedDataLayout::R_SEARCH_TREE],
data_layout->num_entries[storage::DataLayout::R_SEARCH_TREE],
file_index_path,
m_coordinate_list));
m_geospatial_query.reset(
new SharedGeospatialQuery(*m_static_rtree, m_coordinate_list, *this));
}
void LoadGraph()
void InitGraph()
{
auto graph_nodes_ptr = data_layout->GetBlockPtr<GraphNode>(
shared_memory, storage::SharedDataLayout::GRAPH_NODE_LIST);
auto graph_nodes_ptr =
data_layout->GetBlockPtr<GraphNode>(memory_block, storage::DataLayout::GRAPH_NODE_LIST);
auto graph_edges_ptr = data_layout->GetBlockPtr<GraphEdge>(
shared_memory, storage::SharedDataLayout::GRAPH_EDGE_LIST);
auto graph_edges_ptr =
data_layout->GetBlockPtr<GraphEdge>(memory_block, storage::DataLayout::GRAPH_EDGE_LIST);
util::ShM<GraphNode, true>::vector node_list(
graph_nodes_ptr, data_layout->num_entries[storage::SharedDataLayout::GRAPH_NODE_LIST]);
graph_nodes_ptr, data_layout->num_entries[storage::DataLayout::GRAPH_NODE_LIST]);
util::ShM<GraphEdge, true>::vector edge_list(
graph_edges_ptr, data_layout->num_entries[storage::SharedDataLayout::GRAPH_EDGE_LIST]);
graph_edges_ptr, data_layout->num_entries[storage::DataLayout::GRAPH_EDGE_LIST]);
m_query_graph.reset(new QueryGraph(node_list, edge_list));
}
void LoadNodeAndEdgeInformation()
void InitNodeAndEdgeInformation()
{
const auto coordinate_list_ptr = data_layout->GetBlockPtr<util::Coordinate>(
shared_memory, storage::SharedDataLayout::COORDINATE_LIST);
m_coordinate_list.reset(
coordinate_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::COORDINATE_LIST]);
memory_block, storage::DataLayout::COORDINATE_LIST);
m_coordinate_list.reset(coordinate_list_ptr,
data_layout->num_entries[storage::DataLayout::COORDINATE_LIST]);
for (unsigned i = 0; i < m_coordinate_list.size(); ++i)
{
@ -198,285 +189,226 @@ class SharedDataFacade final : public BaseDataFacade
}
const auto osmnodeid_list_ptr = data_layout->GetBlockPtr<std::uint64_t>(
shared_memory, storage::SharedDataLayout::OSM_NODE_ID_LIST);
m_osmnodeid_list.reset(
osmnodeid_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::OSM_NODE_ID_LIST]);
memory_block, storage::DataLayout::OSM_NODE_ID_LIST);
m_osmnodeid_list.reset(osmnodeid_list_ptr,
data_layout->num_entries[storage::DataLayout::OSM_NODE_ID_LIST]);
// We (ab)use the number of coordinates here because we know we have the same amount of ids
m_osmnodeid_list.set_number_of_entries(
data_layout->num_entries[storage::SharedDataLayout::COORDINATE_LIST]);
data_layout->num_entries[storage::DataLayout::COORDINATE_LIST]);
const auto travel_mode_list_ptr = data_layout->GetBlockPtr<extractor::TravelMode>(
shared_memory, storage::SharedDataLayout::TRAVEL_MODE);
memory_block, storage::DataLayout::TRAVEL_MODE);
util::ShM<extractor::TravelMode, true>::vector travel_mode_list(
travel_mode_list_ptr, data_layout->num_entries[storage::SharedDataLayout::TRAVEL_MODE]);
travel_mode_list_ptr, data_layout->num_entries[storage::DataLayout::TRAVEL_MODE]);
m_travel_mode_list = std::move(travel_mode_list);
const auto lane_data_id_ptr = data_layout->GetBlockPtr<LaneDataID>(
shared_memory, storage::SharedDataLayout::LANE_DATA_ID);
const auto lane_data_id_ptr =
data_layout->GetBlockPtr<LaneDataID>(memory_block, storage::DataLayout::LANE_DATA_ID);
util::ShM<LaneDataID, true>::vector lane_data_id(
lane_data_id_ptr, data_layout->num_entries[storage::SharedDataLayout::LANE_DATA_ID]);
lane_data_id_ptr, data_layout->num_entries[storage::DataLayout::LANE_DATA_ID]);
m_lane_data_id = std::move(lane_data_id);
const auto lane_tupel_id_pair_ptr =
data_layout->GetBlockPtr<util::guidance::LaneTupleIdPair>(
shared_memory, storage::SharedDataLayout::TURN_LANE_DATA);
memory_block, storage::DataLayout::TURN_LANE_DATA);
util::ShM<util::guidance::LaneTupleIdPair, true>::vector lane_tupel_id_pair(
lane_tupel_id_pair_ptr,
data_layout->num_entries[storage::SharedDataLayout::TURN_LANE_DATA]);
lane_tupel_id_pair_ptr, data_layout->num_entries[storage::DataLayout::TURN_LANE_DATA]);
m_lane_tupel_id_pairs = std::move(lane_tupel_id_pair);
const auto turn_instruction_list_ptr =
data_layout->GetBlockPtr<extractor::guidance::TurnInstruction>(
shared_memory, storage::SharedDataLayout::TURN_INSTRUCTION);
memory_block, storage::DataLayout::TURN_INSTRUCTION);
util::ShM<extractor::guidance::TurnInstruction, true>::vector turn_instruction_list(
turn_instruction_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::TURN_INSTRUCTION]);
data_layout->num_entries[storage::DataLayout::TURN_INSTRUCTION]);
m_turn_instruction_list = std::move(turn_instruction_list);
const auto name_id_list_ptr = data_layout->GetBlockPtr<unsigned>(
shared_memory, storage::SharedDataLayout::NAME_ID_LIST);
const auto name_id_list_ptr =
data_layout->GetBlockPtr<unsigned>(memory_block, storage::DataLayout::NAME_ID_LIST);
util::ShM<unsigned, true>::vector name_id_list(
name_id_list_ptr, data_layout->num_entries[storage::SharedDataLayout::NAME_ID_LIST]);
name_id_list_ptr, data_layout->num_entries[storage::DataLayout::NAME_ID_LIST]);
m_name_ID_list = std::move(name_id_list);
const auto entry_class_id_list_ptr = data_layout->GetBlockPtr<EntryClassID>(
shared_memory, storage::SharedDataLayout::ENTRY_CLASSID);
memory_block, storage::DataLayout::ENTRY_CLASSID);
typename util::ShM<EntryClassID, true>::vector entry_class_id_list(
entry_class_id_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::ENTRY_CLASSID]);
entry_class_id_list_ptr, data_layout->num_entries[storage::DataLayout::ENTRY_CLASSID]);
m_entry_class_id_list = std::move(entry_class_id_list);
const auto pre_turn_bearing_ptr = data_layout->GetBlockPtr<util::guidance::TurnBearing>(
shared_memory, storage::SharedDataLayout::PRE_TURN_BEARING);
memory_block, storage::DataLayout::PRE_TURN_BEARING);
typename util::ShM<util::guidance::TurnBearing, true>::vector pre_turn_bearing(
pre_turn_bearing_ptr,
data_layout->num_entries[storage::SharedDataLayout::PRE_TURN_BEARING]);
pre_turn_bearing_ptr, data_layout->num_entries[storage::DataLayout::PRE_TURN_BEARING]);
m_pre_turn_bearing = std::move(pre_turn_bearing);
const auto post_turn_bearing_ptr = data_layout->GetBlockPtr<util::guidance::TurnBearing>(
shared_memory, storage::SharedDataLayout::POST_TURN_BEARING);
memory_block, storage::DataLayout::POST_TURN_BEARING);
typename util::ShM<util::guidance::TurnBearing, true>::vector post_turn_bearing(
post_turn_bearing_ptr,
data_layout->num_entries[storage::SharedDataLayout::POST_TURN_BEARING]);
data_layout->num_entries[storage::DataLayout::POST_TURN_BEARING]);
m_post_turn_bearing = std::move(post_turn_bearing);
}
void LoadViaNodeList()
void InitViaNodeList()
{
auto via_geometry_list_ptr = data_layout->GetBlockPtr<GeometryID>(
shared_memory, storage::SharedDataLayout::VIA_NODE_LIST);
auto via_geometry_list_ptr =
data_layout->GetBlockPtr<GeometryID>(memory_block, storage::DataLayout::VIA_NODE_LIST);
util::ShM<GeometryID, true>::vector via_geometry_list(
via_geometry_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::VIA_NODE_LIST]);
via_geometry_list_ptr, data_layout->num_entries[storage::DataLayout::VIA_NODE_LIST]);
m_via_geometry_list = std::move(via_geometry_list);
}
void LoadNames()
void InitNames()
{
auto offsets_ptr = data_layout->GetBlockPtr<unsigned>(
shared_memory, storage::SharedDataLayout::NAME_OFFSETS);
auto blocks_ptr = data_layout->GetBlockPtr<IndexBlock>(
shared_memory, storage::SharedDataLayout::NAME_BLOCKS);
auto offsets_ptr =
data_layout->GetBlockPtr<unsigned>(memory_block, storage::DataLayout::NAME_OFFSETS);
auto blocks_ptr =
data_layout->GetBlockPtr<IndexBlock>(memory_block, storage::DataLayout::NAME_BLOCKS);
util::ShM<unsigned, true>::vector name_offsets(
offsets_ptr, data_layout->num_entries[storage::SharedDataLayout::NAME_OFFSETS]);
offsets_ptr, data_layout->num_entries[storage::DataLayout::NAME_OFFSETS]);
util::ShM<IndexBlock, true>::vector name_blocks(
blocks_ptr, data_layout->num_entries[storage::SharedDataLayout::NAME_BLOCKS]);
blocks_ptr, data_layout->num_entries[storage::DataLayout::NAME_BLOCKS]);
auto names_list_ptr = data_layout->GetBlockPtr<char>(
shared_memory, storage::SharedDataLayout::NAME_CHAR_LIST);
auto names_list_ptr =
data_layout->GetBlockPtr<char>(memory_block, storage::DataLayout::NAME_CHAR_LIST);
util::ShM<char, true>::vector names_char_list(
names_list_ptr, data_layout->num_entries[storage::SharedDataLayout::NAME_CHAR_LIST]);
names_list_ptr, data_layout->num_entries[storage::DataLayout::NAME_CHAR_LIST]);
m_name_table = std::make_unique<util::RangeTable<16, true>>(
name_offsets, name_blocks, static_cast<unsigned>(names_char_list.size()));
m_names_char_list = std::move(names_char_list);
}
void LoadTurnLaneDescriptions()
void InitTurnLaneDescriptions()
{
auto offsets_ptr = data_layout->GetBlockPtr<std::uint32_t>(
shared_memory, storage::SharedDataLayout::LANE_DESCRIPTION_OFFSETS);
memory_block, storage::DataLayout::LANE_DESCRIPTION_OFFSETS);
util::ShM<std::uint32_t, true>::vector offsets(
offsets_ptr,
data_layout->num_entries[storage::SharedDataLayout::LANE_DESCRIPTION_OFFSETS]);
offsets_ptr, data_layout->num_entries[storage::DataLayout::LANE_DESCRIPTION_OFFSETS]);
m_lane_description_offsets = std::move(offsets);
auto masks_ptr = data_layout->GetBlockPtr<extractor::guidance::TurnLaneType::Mask>(
shared_memory, storage::SharedDataLayout::LANE_DESCRIPTION_MASKS);
memory_block, storage::DataLayout::LANE_DESCRIPTION_MASKS);
util::ShM<extractor::guidance::TurnLaneType::Mask, true>::vector masks(
masks_ptr, data_layout->num_entries[storage::SharedDataLayout::LANE_DESCRIPTION_MASKS]);
masks_ptr, data_layout->num_entries[storage::DataLayout::LANE_DESCRIPTION_MASKS]);
m_lane_description_masks = std::move(masks);
}
void LoadCoreInformation()
void InitCoreInformation()
{
auto core_marker_ptr = data_layout->GetBlockPtr<unsigned>(
shared_memory, storage::SharedDataLayout::CORE_MARKER);
auto core_marker_ptr =
data_layout->GetBlockPtr<unsigned>(memory_block, storage::DataLayout::CORE_MARKER);
util::ShM<bool, true>::vector is_core_node(
core_marker_ptr, data_layout->num_entries[storage::SharedDataLayout::CORE_MARKER]);
core_marker_ptr, data_layout->num_entries[storage::DataLayout::CORE_MARKER]);
m_is_core_node = std::move(is_core_node);
}
void LoadGeometries()
void InitGeometries()
{
auto geometries_index_ptr = data_layout->GetBlockPtr<unsigned>(
shared_memory, storage::SharedDataLayout::GEOMETRIES_INDEX);
auto geometries_index_ptr =
data_layout->GetBlockPtr<unsigned>(memory_block, storage::DataLayout::GEOMETRIES_INDEX);
util::ShM<unsigned, true>::vector geometry_begin_indices(
geometries_index_ptr,
data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_INDEX]);
geometries_index_ptr, data_layout->num_entries[storage::DataLayout::GEOMETRIES_INDEX]);
m_geometry_indices = std::move(geometry_begin_indices);
auto geometries_node_list_ptr = data_layout->GetBlockPtr<NodeID>(
shared_memory, storage::SharedDataLayout::GEOMETRIES_NODE_LIST);
memory_block, storage::DataLayout::GEOMETRIES_NODE_LIST);
util::ShM<NodeID, true>::vector geometry_node_list(
geometries_node_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_NODE_LIST]);
data_layout->num_entries[storage::DataLayout::GEOMETRIES_NODE_LIST]);
m_geometry_node_list = std::move(geometry_node_list);
auto geometries_fwd_weight_list_ptr = data_layout->GetBlockPtr<EdgeWeight>(
shared_memory, storage::SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST);
memory_block, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST);
util::ShM<EdgeWeight, true>::vector geometry_fwd_weight_list(
geometries_fwd_weight_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST]);
data_layout->num_entries[storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]);
m_geometry_fwd_weight_list = std::move(geometry_fwd_weight_list);
auto geometries_rev_weight_list_ptr = data_layout->GetBlockPtr<EdgeWeight>(
shared_memory, storage::SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST);
memory_block, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST);
util::ShM<EdgeWeight, true>::vector geometry_rev_weight_list(
geometries_rev_weight_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST]);
data_layout->num_entries[storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST]);
m_geometry_rev_weight_list = std::move(geometry_rev_weight_list);
auto datasources_list_ptr = data_layout->GetBlockPtr<uint8_t>(
shared_memory, storage::SharedDataLayout::DATASOURCES_LIST);
auto datasources_list_ptr =
data_layout->GetBlockPtr<uint8_t>(memory_block, storage::DataLayout::DATASOURCES_LIST);
util::ShM<uint8_t, true>::vector datasources_list(
datasources_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::DATASOURCES_LIST]);
datasources_list_ptr, data_layout->num_entries[storage::DataLayout::DATASOURCES_LIST]);
m_datasource_list = std::move(datasources_list);
auto datasource_name_data_ptr = data_layout->GetBlockPtr<char>(
shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_DATA);
auto datasource_name_data_ptr =
data_layout->GetBlockPtr<char>(memory_block, storage::DataLayout::DATASOURCE_NAME_DATA);
util::ShM<char, true>::vector datasource_name_data(
datasource_name_data_ptr,
data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_DATA]);
data_layout->num_entries[storage::DataLayout::DATASOURCE_NAME_DATA]);
m_datasource_name_data = std::move(datasource_name_data);
auto datasource_name_offsets_ptr = data_layout->GetBlockPtr<std::size_t>(
shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_OFFSETS);
memory_block, storage::DataLayout::DATASOURCE_NAME_OFFSETS);
util::ShM<std::size_t, true>::vector datasource_name_offsets(
datasource_name_offsets_ptr,
data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_OFFSETS]);
data_layout->num_entries[storage::DataLayout::DATASOURCE_NAME_OFFSETS]);
m_datasource_name_offsets = std::move(datasource_name_offsets);
auto datasource_name_lengths_ptr = data_layout->GetBlockPtr<std::size_t>(
shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_LENGTHS);
memory_block, storage::DataLayout::DATASOURCE_NAME_LENGTHS);
util::ShM<std::size_t, true>::vector datasource_name_lengths(
datasource_name_lengths_ptr,
data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_LENGTHS]);
data_layout->num_entries[storage::DataLayout::DATASOURCE_NAME_LENGTHS]);
m_datasource_name_lengths = std::move(datasource_name_lengths);
}
void LoadIntersectionClasses()
void InitIntersectionClasses()
{
auto bearing_class_id_ptr = data_layout->GetBlockPtr<BearingClassID>(
shared_memory, storage::SharedDataLayout::BEARING_CLASSID);
memory_block, storage::DataLayout::BEARING_CLASSID);
typename util::ShM<BearingClassID, true>::vector bearing_class_id_table(
bearing_class_id_ptr,
data_layout->num_entries[storage::SharedDataLayout::BEARING_CLASSID]);
bearing_class_id_ptr, data_layout->num_entries[storage::DataLayout::BEARING_CLASSID]);
m_bearing_class_id_table = std::move(bearing_class_id_table);
auto bearing_class_ptr = data_layout->GetBlockPtr<DiscreteBearing>(
shared_memory, storage::SharedDataLayout::BEARING_VALUES);
memory_block, storage::DataLayout::BEARING_VALUES);
typename util::ShM<DiscreteBearing, true>::vector bearing_class_table(
bearing_class_ptr, data_layout->num_entries[storage::SharedDataLayout::BEARING_VALUES]);
bearing_class_ptr, data_layout->num_entries[storage::DataLayout::BEARING_VALUES]);
m_bearing_values_table = std::move(bearing_class_table);
auto offsets_ptr = data_layout->GetBlockPtr<unsigned>(
shared_memory, storage::SharedDataLayout::BEARING_OFFSETS);
auto blocks_ptr = data_layout->GetBlockPtr<IndexBlock>(
shared_memory, storage::SharedDataLayout::BEARING_BLOCKS);
auto offsets_ptr =
data_layout->GetBlockPtr<unsigned>(memory_block, storage::DataLayout::BEARING_OFFSETS);
auto blocks_ptr =
data_layout->GetBlockPtr<IndexBlock>(memory_block, storage::DataLayout::BEARING_BLOCKS);
util::ShM<unsigned, true>::vector bearing_offsets(
offsets_ptr, data_layout->num_entries[storage::SharedDataLayout::BEARING_OFFSETS]);
offsets_ptr, data_layout->num_entries[storage::DataLayout::BEARING_OFFSETS]);
util::ShM<IndexBlock, true>::vector bearing_blocks(
blocks_ptr, data_layout->num_entries[storage::SharedDataLayout::BEARING_BLOCKS]);
blocks_ptr, data_layout->num_entries[storage::DataLayout::BEARING_BLOCKS]);
m_bearing_ranges_table = std::make_unique<util::RangeTable<16, true>>(
bearing_offsets, bearing_blocks, static_cast<unsigned>(m_bearing_values_table.size()));
auto entry_class_ptr = data_layout->GetBlockPtr<util::guidance::EntryClass>(
shared_memory, storage::SharedDataLayout::ENTRY_CLASS);
memory_block, storage::DataLayout::ENTRY_CLASS);
typename util::ShM<util::guidance::EntryClass, true>::vector entry_class_table(
entry_class_ptr, data_layout->num_entries[storage::SharedDataLayout::ENTRY_CLASS]);
entry_class_ptr, data_layout->num_entries[storage::DataLayout::ENTRY_CLASS]);
m_entry_class_table = std::move(entry_class_table);
}
public:
// this function handle the deallocation of the shared memory it we can prove it will not be
// used anymore
virtual ~SharedDataFacade()
void Init()
{
boost::interprocess::scoped_lock<boost::interprocess::named_sharable_mutex> exclusive_lock(
data_region == storage::DATA_1 ? shared_barriers->regions_1_mutex
: shared_barriers->regions_2_mutex,
boost::interprocess::defer_lock);
// if this returns false this is still in use
if (exclusive_lock.try_lock())
{
// Now check if this is still the newest dataset
const boost::interprocess::sharable_lock<boost::interprocess::named_upgradable_mutex>
lock(shared_barriers->current_regions_mutex);
auto shared_regions = storage::makeSharedMemory(storage::CURRENT_REGIONS);
const auto current_timestamp =
static_cast<const storage::SharedDataTimestamp *>(shared_regions->Ptr());
if (current_timestamp->timestamp == shared_timestamp)
{
util::SimpleLogger().Write(logDEBUG) << "Retaining data with shared timestamp "
<< shared_timestamp;
}
else
{
storage::SharedMemory::Remove(data_region);
storage::SharedMemory::Remove(layout_region);
}
}
}
SharedDataFacade(const std::shared_ptr<storage::SharedBarriers> &shared_barriers_,
storage::SharedDataType layout_region_,
storage::SharedDataType data_region_,
unsigned shared_timestamp_)
: shared_barriers(shared_barriers_), layout_region(layout_region_),
data_region(data_region_), shared_timestamp(shared_timestamp_)
{
util::SimpleLogger().Write(logDEBUG) << "Loading new data with shared timestamp "
<< shared_timestamp;
BOOST_ASSERT(storage::SharedMemory::RegionExists(layout_region));
m_layout_memory = storage::makeSharedMemory(layout_region);
data_layout = static_cast<storage::SharedDataLayout *>(m_layout_memory->Ptr());
BOOST_ASSERT(storage::SharedMemory::RegionExists(data_region));
m_large_memory = storage::makeSharedMemory(data_region);
shared_memory = (char *)(m_large_memory->Ptr());
LoadGraph();
LoadChecksum();
LoadNodeAndEdgeInformation();
LoadGeometries();
LoadTimestamp();
LoadViaNodeList();
LoadNames();
LoadTurnLaneDescriptions();
LoadCoreInformation();
LoadProfileProperties();
LoadRTree();
LoadIntersectionClasses();
InitGraph();
InitChecksum();
InitNodeAndEdgeInformation();
InitGeometries();
InitTimestamp();
InitViaNodeList();
InitNames();
InitTurnLaneDescriptions();
InitCoreInformation();
InitProfileProperties();
InitRTree();
InitIntersectionClasses();
}
// search graph access

88
include/engine/datafacade/shared_memory_datafacade.hpp Normal file
View File

@ -0,0 +1,88 @@
#ifndef SHARED_MEMORY_DATAFACADE_HPP
#define SHARED_MEMORY_DATAFACADE_HPP
// implements all data storage when shared memory _IS_ used
#include "storage/shared_barriers.hpp"
#include "storage/shared_datatype.hpp"
#include "storage/shared_memory.hpp"
#include "engine/datafacade/memory_datafacade_base.hpp"
namespace osrm
{
namespace engine
{
namespace datafacade
{
class SharedDataFacade : public MemoryDataFacadeBase
{
protected:
std::shared_ptr<storage::SharedBarriers> shared_barriers;
storage::SharedDataType layout_region;
storage::SharedDataType data_region;
unsigned shared_timestamp;
SharedDataFacade() {}
public:
// this function handle the deallocation of the shared memory it we can prove it will not be
// used anymore
virtual ~SharedDataFacade()
{
boost::interprocess::scoped_lock<boost::interprocess::named_sharable_mutex> exclusive_lock(
data_region == storage::DATA_1 ? shared_barriers->regions_1_mutex
: shared_barriers->regions_2_mutex,
boost::interprocess::defer_lock);
// if this returns false this is still in use
if (exclusive_lock.try_lock())
{
// Now check if this is still the newest dataset
const boost::interprocess::sharable_lock<boost::interprocess::named_upgradable_mutex>
lock(shared_barriers->current_regions_mutex);
auto shared_regions = storage::makeSharedMemory(storage::CURRENT_REGIONS);
const auto current_timestamp =
static_cast<const storage::SharedDataTimestamp *>(shared_regions->Ptr());
if (current_timestamp->timestamp == shared_timestamp)
{
util::SimpleLogger().Write(logDEBUG) << "Retaining data with shared timestamp "
<< shared_timestamp;
}
else
{
storage::SharedMemory::Remove(data_region);
storage::SharedMemory::Remove(layout_region);
}
}
}
SharedDataFacade(const std::shared_ptr<storage::SharedBarriers> &shared_barriers_,
storage::SharedDataType layout_region_,
storage::SharedDataType data_region_,
unsigned shared_timestamp_)
: shared_barriers(shared_barriers_), layout_region(layout_region_),
data_region(data_region_), shared_timestamp(shared_timestamp_)
{
util::SimpleLogger().Write(logDEBUG) << "Loading new data with shared timestamp "
<< shared_timestamp;
BOOST_ASSERT(storage::SharedMemory::RegionExists(layout_region));
m_layout_memory = storage::makeSharedMemory(layout_region);
data_layout = static_cast<storage::DataLayout *>(m_layout_memory->Ptr());
BOOST_ASSERT(storage::SharedMemory::RegionExists(data_region));
m_large_memory = storage::makeSharedMemory(data_region);
memory_block = (char *)(m_large_memory->Ptr());
Init();
}
};
}
}
}
#endif // SHARED_MEMORY_DATAFACADE_HPP

21
include/storage/shared_datatype.hpp
View File

@ -7,6 +7,7 @@
#include <cstdint>
#include <array>
#include <iostream>
namespace osrm
{
@ -54,7 +55,7 @@ const constexpr char *block_id_to_name[] = {"NAME_OFFSETS",
"LANE_DESCRIPTION_OFFSETS",
"LANE_DESCRIPTION_MASKS"};
struct SharedDataLayout
struct DataLayout
{
enum BlockID
{
@ -101,7 +102,7 @@ struct SharedDataLayout
std::array<uint64_t, NUM_BLOCKS> num_entries;
std::array<uint64_t, NUM_BLOCKS> entry_size;
SharedDataLayout() : num_entries(), entry_size() {}
DataLayout() : num_entries(), entry_size() {}
template <typename T> inline void SetBlockSize(BlockID bid, uint64_t entries)
{
@ -171,6 +172,20 @@ struct SharedDataLayout
return ptr;
}
friend std::ostream &operator<<(std::ostream &os, const DataLayout &layout)
{
os << "Memory layout: " << std::endl;
os << " Total size: " << layout.GetSizeOfLayout() << std::endl;
for (unsigned bid = 0; bid < BlockID::NUM_BLOCKS; bid++)
{
os << " " << block_id_to_name[bid]
<< " offset: " << layout.GetBlockOffset(BlockID(bid))
<< " size: " << layout.GetBlockSize(BlockID(bid)) << std::endl;
}
return os;
}
};
enum SharedDataType
@ -214,7 +229,7 @@ inline std::string regionToString(const SharedDataType region)
}
}
static_assert(sizeof(block_id_to_name) / sizeof(*block_id_to_name) == SharedDataLayout::NUM_BLOCKS,
static_assert(sizeof(block_id_to_name) / sizeof(*block_id_to_name) == DataLayout::NUM_BLOCKS,
"Number of blocks needs to match the number of Block names.");
}
}

4
include/storage/storage.hpp
View File

@ -29,6 +29,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define STORAGE_HPP
#include "storage/storage_config.hpp"
#include "storage/shared_datatype.hpp"
#include <boost/filesystem/path.hpp>
@ -52,6 +53,9 @@ class Storage
ReturnCode Run(int max_wait);
void LoadLayout(DataLayout *layout);
void LoadData(DataLayout *layout_ptr, char *memory_ptr);
private:
StorageConfig config;
};

4
src/engine/engine.cpp
View File

@ -3,8 +3,8 @@
#include "engine/engine_config.hpp"
#include "engine/status.hpp"
#include "engine/datafacade/internal_datafacade.hpp"
#include "engine/datafacade/shared_datafacade.hpp"
#include "engine/datafacade/internal_memory_datafacade.hpp"
#include "engine/datafacade/shared_memory_datafacade.hpp"
#include "storage/shared_barriers.hpp"
#include "util/simple_logger.hpp"

1346
src/storage/storage.cpp
View File

File diff suppressed because it is too large Load Diff