diff --git a/cmake/FingerPrint-Config.cmake b/cmake/FingerPrint-Config.cmake index 0672b3402..552659a8e 100644 --- a/cmake/FingerPrint-Config.cmake +++ b/cmake/FingerPrint-Config.cmake @@ -4,7 +4,7 @@ set(INFILE ${SOURCE_DIR}/include/util/fingerprint_impl.hpp.in) file(MD5 ${SOURCE_DIR}/src/tools/contract.cpp MD5PREPARE) file(MD5 ${SOURCE_DIR}/include/util/static_rtree.hpp MD5RTREE) file(MD5 ${SOURCE_DIR}/include/util/graph_loader.hpp MD5GRAPH) -file(MD5 ${SOURCE_DIR}/include/engine/datafacade/internal_datafacade.hpp MD5OBJECTS) +file(MD5 ${SOURCE_DIR}/src/storage/storage.cpp MD5OBJECTS) CONFIGURE_FILE(${INFILE} ${NEWFILE}) diff --git a/include/engine/data_watchdog.hpp b/include/engine/data_watchdog.hpp index 097687e82..3981a0039 100644 --- a/include/engine/data_watchdog.hpp +++ b/include/engine/data_watchdog.hpp @@ -1,7 +1,7 @@ #ifndef OSRM_ENGINE_DATA_WATCHDOG_HPP #define OSRM_ENGINE_DATA_WATCHDOG_HPP -#include "engine/datafacade/shared_datafacade.hpp" +#include "engine/datafacade/shared_memory_datafacade.hpp" #include "storage/shared_barriers.hpp" #include "storage/shared_datatype.hpp" diff --git a/include/engine/datafacade/internal_datafacade.hpp b/include/engine/datafacade/internal_datafacade.hpp deleted file mode 100644 index 82a5729e7..000000000 --- a/include/engine/datafacade/internal_datafacade.hpp +++ /dev/null @@ -1,974 +0,0 @@ -#ifndef INTERNAL_DATAFACADE_HPP -#define INTERNAL_DATAFACADE_HPP - -// implements all data storage when shared memory is _NOT_ used - -#include "engine/datafacade/datafacade_base.hpp" - -#include "extractor/guidance/turn_instruction.hpp" -#include "util/guidance/bearing_class.hpp" -#include "util/guidance/entry_class.hpp" - -#include "extractor/compressed_edge_container.hpp" -#include "extractor/original_edge_data.hpp" -#include "extractor/profile_properties.hpp" -#include "extractor/query_node.hpp" -#include "storage/io.hpp" -#include "storage/storage_config.hpp" -#include "engine/geospatial_query.hpp" -#include "util/graph_loader.hpp" -#include "util/guidance/turn_bearing.hpp" -#include "util/guidance/turn_lanes.hpp" -#include "util/io.hpp" -#include "util/packed_vector.hpp" -#include "util/range_table.hpp" -#include "util/rectangle.hpp" -#include "util/shared_memory_vector_wrapper.hpp" -#include "util/simple_logger.hpp" -#include "util/static_graph.hpp" -#include "util/static_rtree.hpp" -#include "util/typedefs.hpp" - -#include "osrm/coordinate.hpp" - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace osrm -{ -namespace engine -{ -namespace datafacade -{ - -class InternalDataFacade final : public BaseDataFacade -{ - - private: - using super = BaseDataFacade; - using QueryGraph = util::StaticGraph; - using InputEdge = QueryGraph::InputEdge; - using RTreeLeaf = super::RTreeLeaf; - using InternalRTree = - util::StaticRTree::vector, false>; - using InternalGeospatialQuery = GeospatialQuery; - - InternalDataFacade() {} - - unsigned m_check_sum; - std::unique_ptr m_query_graph; - std::string m_timestamp; - - util::ShM::vector m_coordinate_list; - util::PackedVector m_osmnodeid_list; - util::ShM::vector m_via_geometry_list; - util::ShM::vector m_name_ID_list; - util::ShM::vector m_turn_instruction_list; - util::ShM::vector m_lane_data_id; - util::ShM::vector m_lane_tuple_id_pairs; - util::ShM::vector m_travel_mode_list; - util::ShM::vector m_names_char_list; - util::ShM::vector m_geometry_indices; - util::ShM::vector m_geometry_node_list; - util::ShM::vector m_geometry_fwd_weight_list; - util::ShM::vector m_geometry_rev_weight_list; - util::ShM::vector m_is_core_node; - util::ShM::vector m_datasource_list; - util::ShM::vector m_datasource_names; - util::ShM::vector m_lane_description_offsets; - util::ShM::vector m_lane_description_masks; - extractor::ProfileProperties m_profile_properties; - - std::unique_ptr m_static_rtree; - std::unique_ptr m_geospatial_query; - boost::filesystem::path ram_index_path; - boost::filesystem::path file_index_path; - util::RangeTable<16, false> m_name_table; - - // bearing classes by node based node - util::ShM::vector m_bearing_class_id_table; - // entry class IDs by edge based egde - util::ShM::vector m_entry_class_id_list; - // bearings pre/post turn - util::ShM::vector m_pre_turn_bearing; - util::ShM::vector m_post_turn_bearing; - // the look-up table for entry classes. An entry class lists the possibility of entry for all - // available turns. For every turn, there is an associated entry class. - util::ShM::vector m_entry_class_table; - // the look-up table for distinct bearing classes. A bearing class lists the available bearings - // at an intersection - util::RangeTable<16, false> m_bearing_ranges_table; - util::ShM::vector m_bearing_values_table; - - void LoadProfileProperties(const boost::filesystem::path &properties_path) - { - boost::filesystem::ifstream in_stream(properties_path); - if (!in_stream) - { - throw util::exception("Could not open " + properties_path.string() + " for reading."); - } - const auto properties_size = storage::io::readPropertiesCount(); - storage::io::readProperties(in_stream, &m_profile_properties, properties_size); - } - - void LoadLaneTupleIdPairs(const boost::filesystem::path &lane_data_path) - { - boost::filesystem::ifstream in_stream(lane_data_path); - if (!in_stream) - { - throw util::exception("Could not open " + lane_data_path.string() + " for reading."); - } - std::uint64_t size; - in_stream.read(reinterpret_cast(&size), sizeof(size)); - m_lane_tuple_id_pairs.resize(size); - in_stream.read(reinterpret_cast(&m_lane_tuple_id_pairs[0]), - sizeof(m_lane_tuple_id_pairs) * size); - } - - void LoadTimestamp(const boost::filesystem::path ×tamp_path) - { - util::SimpleLogger().Write() << "Loading Timestamp"; - - boost::filesystem::ifstream timestamp_stream(timestamp_path); - if (!timestamp_stream) - { - throw util::exception("Could not open " + timestamp_path.string() + " for reading."); - } - - const auto timestamp_size = storage::io::readNumberOfBytes(timestamp_stream); - m_timestamp.resize(timestamp_size); - storage::io::readTimestamp(timestamp_stream, &m_timestamp.front(), timestamp_size); - } - - void LoadGraph(const boost::filesystem::path &hsgr_path) - { - boost::filesystem::ifstream hsgr_input_stream(hsgr_path); - if (!hsgr_input_stream) - { - throw util::exception("Could not open " + hsgr_path.string() + " for reading."); - } - - const auto header = storage::io::readHSGRHeader(hsgr_input_stream); - m_check_sum = header.checksum; - - util::ShM::vector node_list(header.number_of_nodes); - util::ShM::vector edge_list(header.number_of_edges); - - storage::io::readHSGR(hsgr_input_stream, - node_list.data(), - header.number_of_nodes, - edge_list.data(), - header.number_of_edges); - - m_query_graph = std::unique_ptr(new QueryGraph(node_list, edge_list)); - - util::SimpleLogger().Write() << "Data checksum is " << m_check_sum; - } - - void LoadNodeAndEdgeInformation(const boost::filesystem::path &nodes_file_path, - const boost::filesystem::path &edges_file_path) - { - boost::filesystem::ifstream nodes_input_stream(nodes_file_path, std::ios::binary); - if (!nodes_input_stream) - { - throw util::exception("Could not open " + nodes_file_path.string() + " for reading."); - } - - const auto number_of_coordinates = storage::io::readElementCount(nodes_input_stream); - m_coordinate_list.resize(number_of_coordinates); - m_osmnodeid_list.reserve(number_of_coordinates); - storage::io::readNodes( - nodes_input_stream, m_coordinate_list.data(), m_osmnodeid_list, number_of_coordinates); - - boost::filesystem::ifstream edges_input_stream(edges_file_path, std::ios::binary); - if (!edges_input_stream) - { - throw util::exception("Could not open " + edges_file_path.string() + " for reading."); - } - const auto number_of_edges = storage::io::readElementCount(edges_input_stream); - m_via_geometry_list.resize(number_of_edges); - m_name_ID_list.resize(number_of_edges); - m_turn_instruction_list.resize(number_of_edges); - m_lane_data_id.resize(number_of_edges); - m_travel_mode_list.resize(number_of_edges); - m_entry_class_id_list.resize(number_of_edges); - m_pre_turn_bearing.resize(number_of_edges); - m_post_turn_bearing.resize(number_of_edges); - - storage::io::readEdges(edges_input_stream, - m_via_geometry_list.data(), - m_name_ID_list.data(), - m_turn_instruction_list.data(), - m_lane_data_id.data(), - m_travel_mode_list.data(), - m_entry_class_id_list.data(), - m_pre_turn_bearing.data(), - m_post_turn_bearing.data(), - number_of_edges); - } - - void LoadCoreInformation(const boost::filesystem::path &core_data_file) - { - std::ifstream core_stream(core_data_file.string().c_str(), std::ios::binary); - unsigned number_of_markers; - core_stream.read((char *)&number_of_markers, sizeof(unsigned)); - - std::vector unpacked_core_markers(number_of_markers); - core_stream.read((char *)unpacked_core_markers.data(), sizeof(char) * number_of_markers); - - // in this case we have nothing to do - if (number_of_markers <= 0) - { - return; - } - - m_is_core_node.resize(number_of_markers); - for (auto i = 0u; i < number_of_markers; ++i) - { - BOOST_ASSERT(unpacked_core_markers[i] == 0 || unpacked_core_markers[i] == 1); - m_is_core_node[i] = unpacked_core_markers[i] == 1; - } - } - - void LoadGeometries(const boost::filesystem::path &geometry_file) - { - std::ifstream geometry_stream(geometry_file.string().c_str(), std::ios::binary); - unsigned number_of_indices = 0; - unsigned number_of_compressed_geometries = 0; - - geometry_stream.read((char *)&number_of_indices, sizeof(unsigned)); - - m_geometry_indices.resize(number_of_indices); - if (number_of_indices > 0) - { - geometry_stream.read((char *)&(m_geometry_indices[0]), - number_of_indices * sizeof(unsigned)); - } - - geometry_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned)); - - BOOST_ASSERT(m_geometry_indices.back() == number_of_compressed_geometries); - m_geometry_node_list.resize(number_of_compressed_geometries); - m_geometry_fwd_weight_list.resize(number_of_compressed_geometries); - m_geometry_rev_weight_list.resize(number_of_compressed_geometries); - - if (number_of_compressed_geometries > 0) - { - geometry_stream.read((char *)&(m_geometry_node_list[0]), - number_of_compressed_geometries * sizeof(NodeID)); - - geometry_stream.read((char *)&(m_geometry_fwd_weight_list[0]), - number_of_compressed_geometries * sizeof(EdgeWeight)); - - geometry_stream.read((char *)&(m_geometry_rev_weight_list[0]), - number_of_compressed_geometries * sizeof(EdgeWeight)); - } - } - - void LoadDatasourceInfo(const boost::filesystem::path &datasource_names_file, - const boost::filesystem::path &datasource_indexes_file) - { - boost::filesystem::ifstream datasources_stream(datasource_indexes_file, std::ios::binary); - if (!datasources_stream) - { - throw util::exception("Could not open " + datasource_indexes_file.string() + - " for reading!"); - } - BOOST_ASSERT(datasources_stream); - - const auto number_of_datasources = storage::io::readElementCount(datasources_stream); - if (number_of_datasources > 0) - { - m_datasource_list.resize(number_of_datasources); - storage::io::readDatasourceIndexes( - datasources_stream, &m_datasource_list.front(), number_of_datasources); - } - - boost::filesystem::ifstream datasourcenames_stream(datasource_names_file, std::ios::binary); - if (!datasourcenames_stream) - { - throw util::exception("Could not open " + datasource_names_file.string() + - " for reading!"); - } - BOOST_ASSERT(datasourcenames_stream); - - const auto datasource_names_data = storage::io::readDatasourceNames(datasourcenames_stream); - m_datasource_names.resize(datasource_names_data.lengths.size()); - for (std::size_t i = 0; i < datasource_names_data.lengths.size(); ++i) - { - auto name_begin = - datasource_names_data.names.begin() + datasource_names_data.offsets[i]; - auto name_end = datasource_names_data.names.begin() + datasource_names_data.offsets[i] + - datasource_names_data.lengths[i]; - m_datasource_names[i] = std::string(name_begin, name_end); - } - } - - void LoadRTree() - { - BOOST_ASSERT_MSG(!m_coordinate_list.empty(), "coordinates must be loaded before r-tree"); - - m_static_rtree.reset(new InternalRTree(ram_index_path, file_index_path, m_coordinate_list)); - m_geospatial_query.reset( - new InternalGeospatialQuery(*m_static_rtree, m_coordinate_list, *this)); - } - - void LoadLaneDescriptions(const boost::filesystem::path &lane_description_file) - { - if (!util::deserializeAdjacencyArray(lane_description_file.string(), - m_lane_description_offsets, - m_lane_description_masks)) - util::SimpleLogger().Write(logWARNING) << "Failed to read turn lane descriptions from " - << lane_description_file.string(); - } - - void LoadStreetNames(const boost::filesystem::path &names_file) - { - boost::filesystem::ifstream name_stream(names_file, std::ios::binary); - - name_stream >> m_name_table; - - unsigned number_of_chars = 0; - name_stream.read((char *)&number_of_chars, sizeof(unsigned)); - BOOST_ASSERT_MSG(0 != number_of_chars, "name file broken"); - m_names_char_list.resize(number_of_chars + 1); //+1 gives sentinel element - name_stream.read((char *)&m_names_char_list[0], number_of_chars * sizeof(char)); - if (0 == m_names_char_list.size()) - { - util::SimpleLogger().Write(logWARNING) << "list of street names is empty"; - } - } - - void LoadIntersectionClasses(const boost::filesystem::path &intersection_class_file) - { - std::ifstream intersection_stream(intersection_class_file.string(), std::ios::binary); - if (!intersection_stream) - throw util::exception("Could not open " + intersection_class_file.string() + - " for reading."); - - if (!util::readAndCheckFingerprint(intersection_stream)) - throw util::exception("Fingeprint does not match in " + - intersection_class_file.string()); - - { - util::SimpleLogger().Write(logINFO) << "Loading Bearing Class IDs"; - std::vector bearing_class_id; - if (!util::deserializeVector(intersection_stream, bearing_class_id)) - throw util::exception("Reading from " + intersection_class_file.string() + - " failed."); - - m_bearing_class_id_table.resize(bearing_class_id.size()); - std::copy( - bearing_class_id.begin(), bearing_class_id.end(), &m_bearing_class_id_table[0]); - } - { - util::SimpleLogger().Write(logINFO) << "Loading Bearing Classes"; - // read the range table - intersection_stream >> m_bearing_ranges_table; - std::vector bearing_classes; - // and the actual bearing values - std::uint64_t num_bearings; - intersection_stream.read(reinterpret_cast(&num_bearings), sizeof(num_bearings)); - m_bearing_values_table.resize(num_bearings); - intersection_stream.read(reinterpret_cast(&m_bearing_values_table[0]), - sizeof(m_bearing_values_table[0]) * num_bearings); - if (!static_cast(intersection_stream)) - throw util::exception("Reading from " + intersection_class_file.string() + - " failed."); - } - { - util::SimpleLogger().Write(logINFO) << "Loading Entry Classes"; - std::vector entry_classes; - if (!util::deserializeVector(intersection_stream, entry_classes)) - throw util::exception("Reading from " + intersection_class_file.string() + - " failed."); - - m_entry_class_table.resize(entry_classes.size()); - std::copy(entry_classes.begin(), entry_classes.end(), &m_entry_class_table[0]); - } - } - - public: - virtual ~InternalDataFacade() - { - m_static_rtree.reset(); - m_geospatial_query.reset(); - } - - explicit InternalDataFacade(const storage::StorageConfig &config) - { - ram_index_path = config.ram_index_path; - file_index_path = config.file_index_path; - - util::SimpleLogger().Write() << "loading graph data"; - LoadGraph(config.hsgr_data_path); - - util::SimpleLogger().Write() << "loading edge information"; - LoadNodeAndEdgeInformation(config.nodes_data_path, config.edges_data_path); - - util::SimpleLogger().Write() << "loading core information"; - LoadCoreInformation(config.core_data_path); - - util::SimpleLogger().Write() << "loading geometries"; - LoadGeometries(config.geometries_path); - - util::SimpleLogger().Write() << "loading datasource info"; - LoadDatasourceInfo(config.datasource_names_path, config.datasource_indexes_path); - - util::SimpleLogger().Write() << "loading timestamp"; - LoadTimestamp(config.timestamp_path); - - util::SimpleLogger().Write() << "loading profile properties"; - LoadProfileProperties(config.properties_path); - - util::SimpleLogger().Write() << "loading street names"; - LoadStreetNames(config.names_data_path); - - util::SimpleLogger().Write() << "loading lane tags"; - LoadLaneDescriptions(config.turn_lane_description_path); - - util::SimpleLogger().Write() << "loading rtree"; - LoadRTree(); - - util::SimpleLogger().Write() << "loading intersection class data"; - LoadIntersectionClasses(config.intersection_class_path); - - util::SimpleLogger().Write() << "Loading Lane Data Pairs"; - LoadLaneTupleIdPairs(config.turn_lane_data_path); - } - - // search graph access - unsigned GetNumberOfNodes() const override final { return m_query_graph->GetNumberOfNodes(); } - - unsigned GetNumberOfEdges() const override final { return m_query_graph->GetNumberOfEdges(); } - - unsigned GetOutDegree(const NodeID n) const override final - { - return m_query_graph->GetOutDegree(n); - } - - NodeID GetTarget(const EdgeID e) const override final { return m_query_graph->GetTarget(e); } - - EdgeData &GetEdgeData(const EdgeID e) const override final - { - return m_query_graph->GetEdgeData(e); - } - - EdgeID BeginEdges(const NodeID n) const override final { return m_query_graph->BeginEdges(n); } - - EdgeID EndEdges(const NodeID n) const override final { return m_query_graph->EndEdges(n); } - - EdgeRange GetAdjacentEdgeRange(const NodeID node) const override final - { - return m_query_graph->GetAdjacentEdgeRange(node); - } - - // searches for a specific edge - EdgeID FindEdge(const NodeID from, const NodeID to) const override final - { - return m_query_graph->FindEdge(from, to); - } - - EdgeID FindEdgeInEitherDirection(const NodeID from, const NodeID to) const override final - { - return m_query_graph->FindEdgeInEitherDirection(from, to); - } - - EdgeID - FindEdgeIndicateIfReverse(const NodeID from, const NodeID to, bool &result) const override final - { - return m_query_graph->FindEdgeIndicateIfReverse(from, to, result); - } - - EdgeID FindSmallestEdge(const NodeID from, - const NodeID to, - std::function filter) const override final - { - return m_query_graph->FindSmallestEdge(from, to, filter); - } - - // node and edge information access - util::Coordinate GetCoordinateOfNode(const unsigned id) const override final - { - return m_coordinate_list[id]; - } - - OSMNodeID GetOSMNodeIDOfNode(const unsigned id) const override final - { - return m_osmnodeid_list.at(id); - } - - extractor::guidance::TurnInstruction - GetTurnInstructionForEdgeID(const unsigned id) const override final - { - return m_turn_instruction_list.at(id); - } - - extractor::TravelMode GetTravelModeForEdgeID(const unsigned id) const override final - { - return m_travel_mode_list.at(id); - } - - std::vector GetEdgesInBox(const util::Coordinate south_west, - const util::Coordinate north_east) const override final - { - BOOST_ASSERT(m_geospatial_query.get()); - const util::RectangleInt2D bbox{ - south_west.lon, north_east.lon, south_west.lat, north_east.lat}; - return m_geospatial_query->Search(bbox); - } - - std::vector - NearestPhantomNodesInRange(const util::Coordinate input_coordinate, - const float max_distance) const override final - { - BOOST_ASSERT(m_geospatial_query.get()); - - return m_geospatial_query->NearestPhantomNodesInRange(input_coordinate, max_distance); - } - - std::vector - NearestPhantomNodesInRange(const util::Coordinate input_coordinate, - const float max_distance, - const int bearing, - const int bearing_range) const override final - { - BOOST_ASSERT(m_geospatial_query.get()); - - return m_geospatial_query->NearestPhantomNodesInRange( - input_coordinate, max_distance, bearing, bearing_range); - } - - std::vector - NearestPhantomNodes(const util::Coordinate input_coordinate, - const unsigned max_results) const override final - { - BOOST_ASSERT(m_geospatial_query.get()); - - return m_geospatial_query->NearestPhantomNodes(input_coordinate, max_results); - } - - std::vector - NearestPhantomNodes(const util::Coordinate input_coordinate, - const unsigned max_results, - const double max_distance) const override final - { - BOOST_ASSERT(m_geospatial_query.get()); - - return m_geospatial_query->NearestPhantomNodes(input_coordinate, max_results, max_distance); - } - - std::vector - NearestPhantomNodes(const util::Coordinate input_coordinate, - const unsigned max_results, - const int bearing, - const int bearing_range) const override final - { - BOOST_ASSERT(m_geospatial_query.get()); - - return m_geospatial_query->NearestPhantomNodes( - input_coordinate, max_results, bearing, bearing_range); - } - - std::vector - NearestPhantomNodes(const util::Coordinate input_coordinate, - const unsigned max_results, - const double max_distance, - const int bearing, - const int bearing_range) const override final - { - BOOST_ASSERT(m_geospatial_query.get()); - - return m_geospatial_query->NearestPhantomNodes( - input_coordinate, max_results, max_distance, bearing, bearing_range); - } - - std::pair NearestPhantomNodeWithAlternativeFromBigComponent( - const util::Coordinate input_coordinate, const double max_distance) const override final - { - BOOST_ASSERT(m_geospatial_query.get()); - - return m_geospatial_query->NearestPhantomNodeWithAlternativeFromBigComponent( - input_coordinate, max_distance); - } - - std::pair NearestPhantomNodeWithAlternativeFromBigComponent( - const util::Coordinate input_coordinate) const override final - { - BOOST_ASSERT(m_geospatial_query.get()); - - return m_geospatial_query->NearestPhantomNodeWithAlternativeFromBigComponent( - input_coordinate); - } - - std::pair - NearestPhantomNodeWithAlternativeFromBigComponent(const util::Coordinate input_coordinate, - const double max_distance, - const int bearing, - const int bearing_range) const override final - { - BOOST_ASSERT(m_geospatial_query.get()); - - return m_geospatial_query->NearestPhantomNodeWithAlternativeFromBigComponent( - input_coordinate, max_distance, bearing, bearing_range); - } - - std::pair - NearestPhantomNodeWithAlternativeFromBigComponent(const util::Coordinate input_coordinate, - const int bearing, - const int bearing_range) const override final - { - BOOST_ASSERT(m_geospatial_query.get()); - - return m_geospatial_query->NearestPhantomNodeWithAlternativeFromBigComponent( - input_coordinate, bearing, bearing_range); - } - - unsigned GetCheckSum() const override final { return m_check_sum; } - - unsigned GetNameIndexFromEdgeID(const unsigned id) const override final - { - return m_name_ID_list.at(id); - } - - std::string GetNameForID(const unsigned name_id) const override final - { - if (std::numeric_limits::max() == name_id) - { - return ""; - } - auto range = m_name_table.GetRange(name_id); - - std::string result; - result.reserve(range.size()); - if (range.begin() != range.end()) - { - result.resize(range.back() - range.front() + 1); - std::copy(m_names_char_list.begin() + range.front(), - m_names_char_list.begin() + range.back() + 1, - result.begin()); - } - return result; - } - - std::string GetRefForID(const unsigned name_id) const override final - { - // We store the ref after the name, destination and pronunciation of a street. - // We do this to get around the street length limit of 255 which would hit - // if we concatenate these. Order (see extractor_callbacks): - // name (0), destination (1), pronunciation (2), ref (3) - return GetNameForID(name_id + 3); - } - - std::string GetPronunciationForID(const unsigned name_id) const override final - { - // We store the pronunciation after the name and destination of a street. - // We do this to get around the street length limit of 255 which would hit - // if we concatenate these. Order (see extractor_callbacks): - // name (0), destination (1), pronunciation (2) - return GetNameForID(name_id + 2); - } - - std::string GetDestinationsForID(const unsigned name_id) const override final - { - // We store the destination after the name of a street. - // We do this to get around the street length limit of 255 which would hit - // if we concatenate these. Order (see extractor_callbacks): - // name (0), destination (1), pronunciation (2) - return GetNameForID(name_id + 1); - } - - virtual GeometryID GetGeometryIndexForEdgeID(const unsigned id) const override final - { - return m_via_geometry_list.at(id); - } - - virtual std::size_t GetCoreSize() const override final { return m_is_core_node.size(); } - - virtual bool IsCoreNode(const NodeID id) const override final - { - if (m_is_core_node.size() > 0) - { - return m_is_core_node[id]; - } - else - { - return false; - } - } - - virtual std::vector GetUncompressedForwardGeometry(const EdgeID id) const override final - { - /* - * NodeID's for geometries are stored in one place for - * both forward and reverse segments along the same bi- - * directional edge. The m_geometry_indices stores - * refences to where to find the beginning of the bi- - * directional edge in the m_geometry_node_list vector. For - * forward geometries of bi-directional edges, edges 2 to - * n of that edge need to be read. - */ - const unsigned begin = m_geometry_indices.at(id); - const unsigned end = m_geometry_indices.at(id + 1); - - std::vector result_nodes; - - result_nodes.resize(end - begin); - - std::copy(m_geometry_node_list.begin() + begin, - m_geometry_node_list.begin() + end, - result_nodes.begin()); - - return result_nodes; - } - - virtual std::vector GetUncompressedReverseGeometry(const EdgeID id) const override final - { - /* - * NodeID's for geometries are stored in one place for - * both forward and reverse segments along the same bi- - * directional edge. The m_geometry_indices stores - * refences to where to find the beginning of the bi- - * directional edge in the m_geometry_node_list vector. - * */ - const unsigned begin = m_geometry_indices.at(id); - const unsigned end = m_geometry_indices.at(id + 1); - - std::vector result_nodes; - - result_nodes.resize(end - begin); - - std::copy(m_geometry_node_list.rbegin() + (m_geometry_node_list.size() - end), - m_geometry_node_list.rbegin() + (m_geometry_node_list.size() - begin), - result_nodes.begin()); - - return result_nodes; - } - - virtual std::vector - GetUncompressedForwardWeights(const EdgeID id) const override final - { - /* - * EdgeWeights's for geometries are stored in one place for - * both forward and reverse segments along the same bi- - * directional edge. The m_geometry_indices stores - * refences to where to find the beginning of the bi- - * directional edge in the m_geometry_fwd_weight_list vector. - * */ - const unsigned begin = m_geometry_indices.at(id) + 1; - const unsigned end = m_geometry_indices.at(id + 1); - - std::vector result_weights; - result_weights.resize(end - begin); - - std::copy(m_geometry_fwd_weight_list.begin() + begin, - m_geometry_fwd_weight_list.begin() + end, - result_weights.begin()); - - return result_weights; - } - - virtual std::vector - GetUncompressedReverseWeights(const EdgeID id) const override final - { - /* - * EdgeWeights for geometries are stored in one place for - * both forward and reverse segments along the same bi- - * directional edge. The m_geometry_indices stores - * refences to where to find the beginning of the bi- - * directional edge in the m_geometry_rev_weight_list vector. For - * reverse weights of bi-directional edges, edges 1 to - * n-1 of that edge need to be read in reverse. - */ - const unsigned begin = m_geometry_indices.at(id); - const unsigned end = m_geometry_indices.at(id + 1) - 1; - - std::vector result_weights; - result_weights.resize(end - begin); - - std::copy(m_geometry_rev_weight_list.rbegin() + (m_geometry_rev_weight_list.size() - end), - m_geometry_rev_weight_list.rbegin() + (m_geometry_rev_weight_list.size() - begin), - result_weights.begin()); - - return result_weights; - } - - // Returns the data source ids that were used to supply the edge - // weights. - virtual std::vector - GetUncompressedForwardDatasources(const EdgeID id) const override final - { - /* - * Data sources for geometries are stored in one place for - * both forward and reverse segments along the same bi- - * directional edge. The m_geometry_indices stores - * refences to where to find the beginning of the bi- - * directional edge in the m_geometry_list vector. For - * forward datasources of bi-directional edges, edges 2 to - * n of that edge need to be read. - */ - const unsigned begin = m_geometry_indices.at(id) + 1; - const unsigned end = m_geometry_indices.at(id + 1); - - std::vector result_datasources; - result_datasources.resize(end - begin); - - // If there was no datasource info, return an array of 0's. - if (m_datasource_list.empty()) - { - for (unsigned i = 0; i < end - begin; ++i) - { - result_datasources.push_back(0); - } - } - else - { - std::copy(m_datasource_list.begin() + begin, - m_datasource_list.begin() + end, - result_datasources.begin()); - } - - return result_datasources; - } - - // Returns the data source ids that were used to supply the edge - // weights. - virtual std::vector - GetUncompressedReverseDatasources(const EdgeID id) const override final - { - /* - * Datasources for geometries are stored in one place for - * both forward and reverse segments along the same bi- - * directional edge. The m_geometry_indices stores - * refences to where to find the beginning of the bi- - * directional edge in the m_geometry_list vector. For - * reverse datasources of bi-directional edges, edges 1 to - * n-1 of that edge need to be read in reverse. - */ - const unsigned begin = m_geometry_indices.at(id); - const unsigned end = m_geometry_indices.at(id + 1) - 1; - - std::vector result_datasources; - result_datasources.resize(end - begin); - - // If there was no datasource info, return an array of 0's. - if (m_datasource_list.empty()) - { - for (unsigned i = 0; i < end - begin; ++i) - { - result_datasources.push_back(0); - } - } - else - { - std::copy(m_datasource_list.rbegin() + (m_datasource_list.size() - end), - m_datasource_list.rbegin() + (m_datasource_list.size() - begin), - result_datasources.begin()); - } - - return result_datasources; - } - - virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const override final - { - BOOST_ASSERT(m_datasource_names.size() >= 1); - BOOST_ASSERT(m_datasource_names.size() > datasource_name_id); - return m_datasource_names[datasource_name_id]; - } - - std::string GetTimestamp() const override final { return m_timestamp; } - - bool GetContinueStraightDefault() const override final - { - return m_profile_properties.continue_straight_at_waypoint; - } - - double GetMapMatchingMaxSpeed() const override final - { - return m_profile_properties.max_speed_for_map_matching; - } - - BearingClassID GetBearingClassID(const NodeID nid) const override final - { - return m_bearing_class_id_table.at(nid); - } - - util::guidance::BearingClass - GetBearingClass(const BearingClassID bearing_class_id) const override final - { - BOOST_ASSERT(bearing_class_id != INVALID_BEARING_CLASSID); - auto range = m_bearing_ranges_table.GetRange(bearing_class_id); - - util::guidance::BearingClass result; - - for (auto itr = m_bearing_values_table.begin() + range.front(); - itr != m_bearing_values_table.begin() + range.back() + 1; - ++itr) - result.add(*itr); - - return result; - } - - EntryClassID GetEntryClassID(const EdgeID eid) const override final - { - return m_entry_class_id_list.at(eid); - } - - util::guidance::TurnBearing PreTurnBearing(const EdgeID eid) const override final - { - return m_pre_turn_bearing.at(eid); - } - util::guidance::TurnBearing PostTurnBearing(const EdgeID eid) const override final - { - return m_post_turn_bearing.at(eid); - } - - util::guidance::EntryClass GetEntryClass(const EntryClassID entry_class_id) const override final - { - return m_entry_class_table.at(entry_class_id); - } - - bool hasLaneData(const EdgeID id) const override final - { - return m_lane_data_id[id] != INVALID_LANE_DATAID; - } - - util::guidance::LaneTupleIdPair GetLaneData(const EdgeID id) const override final - { - BOOST_ASSERT(hasLaneData(id)); - return m_lane_tuple_id_pairs[m_lane_data_id[id]]; - } - - extractor::guidance::TurnLaneDescription - GetTurnDescription(const LaneDescriptionID lane_description_id) const override final - { - if (lane_description_id == INVALID_LANE_DESCRIPTIONID) - return {}; - else - return extractor::guidance::TurnLaneDescription( - m_lane_description_masks.begin() + m_lane_description_offsets[lane_description_id], - m_lane_description_masks.begin() + - m_lane_description_offsets[lane_description_id + 1]); - } -}; -} -} -} - -#endif // INTERNAL_DATAFACADE_HPP diff --git a/include/engine/datafacade/internal_memory_datafacade.hpp b/include/engine/datafacade/internal_memory_datafacade.hpp new file mode 100644 index 000000000..29e4c3216 --- /dev/null +++ b/include/engine/datafacade/internal_memory_datafacade.hpp @@ -0,0 +1,56 @@ +#ifndef INTERNAL_DATAFACADE_HPP +#define INTERNAL_DATAFACADE_HPP + +// implements all data storage when shared memory is _NOT_ used + +#include "engine/datafacade/memory_datafacade_base.hpp" +#include "storage/storage.hpp" + +namespace osrm +{ +namespace engine +{ +namespace datafacade +{ + +/** + * This datafacade uses a process-local memory block to load + * data into. The structure and layout is the same as when using + * shared memory, so this class, and the SharedDatafacade both + * share a common base. + * This class holds a unique_ptr to the memory block, so it + * is auto-freed upon destruction. + */ +class InternalDataFacade final : public MemoryDataFacadeBase +{ + + private: + std::unique_ptr internal_memory; + std::unique_ptr internal_layout; + + public: + explicit InternalDataFacade(const storage::StorageConfig &config) + { + storage::Storage storage(config); + + // Calculate the layout/size of the memory block + internal_layout = std::make_unique(); + storage.LoadLayout(internal_layout.get()); + + // Allocate the memory block, then load data from files into it + internal_memory.reset(new char[internal_layout->GetSizeOfLayout()]); + storage.LoadData(internal_layout.get(), internal_memory.get()); + + // Set the common base-class pointers + data_layout = internal_layout.get(); + memory_block = internal_memory.get(); + + // Adjust all the private m_* members to point to the right places + Init(); + } +}; +} +} +} + +#endif // INTERNAL_DATAFACADE_HPP diff --git a/include/engine/datafacade/shared_datafacade.hpp b/include/engine/datafacade/memory_datafacade_base.hpp similarity index 74% rename from include/engine/datafacade/shared_datafacade.hpp rename to include/engine/datafacade/memory_datafacade_base.hpp index ea3b56f23..c54d966fa 100644 --- a/include/engine/datafacade/shared_datafacade.hpp +++ b/include/engine/datafacade/memory_datafacade_base.hpp @@ -3,9 +3,6 @@ // implements all data storage when shared memory _IS_ used -#include "storage/shared_barriers.hpp" -#include "storage/shared_datatype.hpp" -#include "storage/shared_memory.hpp" #include "engine/datafacade/datafacade_base.hpp" #include "extractor/compressed_edge_container.hpp" @@ -47,10 +44,10 @@ namespace engine namespace datafacade { -class SharedDataFacade final : public BaseDataFacade +class MemoryDataFacadeBase : public BaseDataFacade { - private: + protected: using super = BaseDataFacade; using QueryGraph = util::StaticGraph; using GraphNode = QueryGraph::NodeArrayEntry; @@ -63,13 +60,8 @@ class SharedDataFacade final : public BaseDataFacade using SharedGeospatialQuery = GeospatialQuery; using RTreeNode = SharedRTree::TreeNode; - storage::SharedDataLayout *data_layout; - char *shared_memory; - - std::shared_ptr shared_barriers; - storage::SharedDataType layout_region; - storage::SharedDataType data_region; - unsigned shared_timestamp; + storage::DataLayout *data_layout; + char *memory_block; unsigned m_check_sum; std::unique_ptr m_query_graph; @@ -121,35 +113,35 @@ class SharedDataFacade final : public BaseDataFacade std::shared_ptr> m_bearing_ranges_table; util::ShM::vector m_bearing_values_table; - void LoadChecksum() + void InitChecksum() { - m_check_sum = *data_layout->GetBlockPtr(shared_memory, - storage::SharedDataLayout::HSGR_CHECKSUM); + m_check_sum = + *data_layout->GetBlockPtr(memory_block, storage::DataLayout::HSGR_CHECKSUM); util::SimpleLogger().Write() << "set checksum: " << m_check_sum; } - void LoadProfileProperties() + void InitProfileProperties() { m_profile_properties = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::PROPERTIES); + memory_block, storage::DataLayout::PROPERTIES); } - void LoadTimestamp() + void InitTimestamp() { auto timestamp_ptr = - data_layout->GetBlockPtr(shared_memory, storage::SharedDataLayout::TIMESTAMP); - m_timestamp.resize(data_layout->GetBlockSize(storage::SharedDataLayout::TIMESTAMP)); + data_layout->GetBlockPtr(memory_block, storage::DataLayout::TIMESTAMP); + m_timestamp.resize(data_layout->GetBlockSize(storage::DataLayout::TIMESTAMP)); std::copy(timestamp_ptr, - timestamp_ptr + data_layout->GetBlockSize(storage::SharedDataLayout::TIMESTAMP), + timestamp_ptr + data_layout->GetBlockSize(storage::DataLayout::TIMESTAMP), m_timestamp.begin()); } - void LoadRTree() + void InitRTree() { BOOST_ASSERT_MSG(!m_coordinate_list.empty(), "coordinates must be loaded before r-tree"); - const auto file_index_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::FILE_INDEX_PATH); + const auto file_index_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::FILE_INDEX_PATH); file_index_path = boost::filesystem::path(file_index_ptr); if (!boost::filesystem::exists(file_index_path)) { @@ -158,39 +150,38 @@ class SharedDataFacade final : public BaseDataFacade "Is any data loaded into shared memory?"); } - auto tree_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::R_SEARCH_TREE); + auto tree_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::R_SEARCH_TREE); m_static_rtree.reset( new SharedRTree(tree_ptr, - data_layout->num_entries[storage::SharedDataLayout::R_SEARCH_TREE], + data_layout->num_entries[storage::DataLayout::R_SEARCH_TREE], file_index_path, m_coordinate_list)); m_geospatial_query.reset( new SharedGeospatialQuery(*m_static_rtree, m_coordinate_list, *this)); } - void LoadGraph() + void InitGraph() { - auto graph_nodes_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::GRAPH_NODE_LIST); + auto graph_nodes_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::GRAPH_NODE_LIST); - auto graph_edges_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::GRAPH_EDGE_LIST); + auto graph_edges_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::GRAPH_EDGE_LIST); util::ShM::vector node_list( - graph_nodes_ptr, data_layout->num_entries[storage::SharedDataLayout::GRAPH_NODE_LIST]); + graph_nodes_ptr, data_layout->num_entries[storage::DataLayout::GRAPH_NODE_LIST]); util::ShM::vector edge_list( - graph_edges_ptr, data_layout->num_entries[storage::SharedDataLayout::GRAPH_EDGE_LIST]); + graph_edges_ptr, data_layout->num_entries[storage::DataLayout::GRAPH_EDGE_LIST]); m_query_graph.reset(new QueryGraph(node_list, edge_list)); } - void LoadNodeAndEdgeInformation() + void InitNodeAndEdgeInformation() { const auto coordinate_list_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::COORDINATE_LIST); - m_coordinate_list.reset( - coordinate_list_ptr, - data_layout->num_entries[storage::SharedDataLayout::COORDINATE_LIST]); + memory_block, storage::DataLayout::COORDINATE_LIST); + m_coordinate_list.reset(coordinate_list_ptr, + data_layout->num_entries[storage::DataLayout::COORDINATE_LIST]); for (unsigned i = 0; i < m_coordinate_list.size(); ++i) { @@ -198,285 +189,226 @@ class SharedDataFacade final : public BaseDataFacade } const auto osmnodeid_list_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::OSM_NODE_ID_LIST); - m_osmnodeid_list.reset( - osmnodeid_list_ptr, - data_layout->num_entries[storage::SharedDataLayout::OSM_NODE_ID_LIST]); + memory_block, storage::DataLayout::OSM_NODE_ID_LIST); + m_osmnodeid_list.reset(osmnodeid_list_ptr, + data_layout->num_entries[storage::DataLayout::OSM_NODE_ID_LIST]); // We (ab)use the number of coordinates here because we know we have the same amount of ids m_osmnodeid_list.set_number_of_entries( - data_layout->num_entries[storage::SharedDataLayout::COORDINATE_LIST]); + data_layout->num_entries[storage::DataLayout::COORDINATE_LIST]); const auto travel_mode_list_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::TRAVEL_MODE); + memory_block, storage::DataLayout::TRAVEL_MODE); util::ShM::vector travel_mode_list( - travel_mode_list_ptr, data_layout->num_entries[storage::SharedDataLayout::TRAVEL_MODE]); + travel_mode_list_ptr, data_layout->num_entries[storage::DataLayout::TRAVEL_MODE]); m_travel_mode_list = std::move(travel_mode_list); - const auto lane_data_id_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::LANE_DATA_ID); + const auto lane_data_id_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::LANE_DATA_ID); util::ShM::vector lane_data_id( - lane_data_id_ptr, data_layout->num_entries[storage::SharedDataLayout::LANE_DATA_ID]); + lane_data_id_ptr, data_layout->num_entries[storage::DataLayout::LANE_DATA_ID]); m_lane_data_id = std::move(lane_data_id); const auto lane_tupel_id_pair_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::TURN_LANE_DATA); + memory_block, storage::DataLayout::TURN_LANE_DATA); util::ShM::vector lane_tupel_id_pair( - lane_tupel_id_pair_ptr, - data_layout->num_entries[storage::SharedDataLayout::TURN_LANE_DATA]); + lane_tupel_id_pair_ptr, data_layout->num_entries[storage::DataLayout::TURN_LANE_DATA]); m_lane_tupel_id_pairs = std::move(lane_tupel_id_pair); const auto turn_instruction_list_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::TURN_INSTRUCTION); + memory_block, storage::DataLayout::TURN_INSTRUCTION); util::ShM::vector turn_instruction_list( turn_instruction_list_ptr, - data_layout->num_entries[storage::SharedDataLayout::TURN_INSTRUCTION]); + data_layout->num_entries[storage::DataLayout::TURN_INSTRUCTION]); m_turn_instruction_list = std::move(turn_instruction_list); - const auto name_id_list_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::NAME_ID_LIST); + const auto name_id_list_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::NAME_ID_LIST); util::ShM::vector name_id_list( - name_id_list_ptr, data_layout->num_entries[storage::SharedDataLayout::NAME_ID_LIST]); + name_id_list_ptr, data_layout->num_entries[storage::DataLayout::NAME_ID_LIST]); m_name_ID_list = std::move(name_id_list); const auto entry_class_id_list_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::ENTRY_CLASSID); + memory_block, storage::DataLayout::ENTRY_CLASSID); typename util::ShM::vector entry_class_id_list( - entry_class_id_list_ptr, - data_layout->num_entries[storage::SharedDataLayout::ENTRY_CLASSID]); + entry_class_id_list_ptr, data_layout->num_entries[storage::DataLayout::ENTRY_CLASSID]); m_entry_class_id_list = std::move(entry_class_id_list); const auto pre_turn_bearing_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::PRE_TURN_BEARING); + memory_block, storage::DataLayout::PRE_TURN_BEARING); typename util::ShM::vector pre_turn_bearing( - pre_turn_bearing_ptr, - data_layout->num_entries[storage::SharedDataLayout::PRE_TURN_BEARING]); + pre_turn_bearing_ptr, data_layout->num_entries[storage::DataLayout::PRE_TURN_BEARING]); m_pre_turn_bearing = std::move(pre_turn_bearing); const auto post_turn_bearing_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::POST_TURN_BEARING); + memory_block, storage::DataLayout::POST_TURN_BEARING); typename util::ShM::vector post_turn_bearing( post_turn_bearing_ptr, - data_layout->num_entries[storage::SharedDataLayout::POST_TURN_BEARING]); + data_layout->num_entries[storage::DataLayout::POST_TURN_BEARING]); m_post_turn_bearing = std::move(post_turn_bearing); } - void LoadViaNodeList() + void InitViaNodeList() { - auto via_geometry_list_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::VIA_NODE_LIST); + auto via_geometry_list_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::VIA_NODE_LIST); util::ShM::vector via_geometry_list( - via_geometry_list_ptr, - data_layout->num_entries[storage::SharedDataLayout::VIA_NODE_LIST]); + via_geometry_list_ptr, data_layout->num_entries[storage::DataLayout::VIA_NODE_LIST]); m_via_geometry_list = std::move(via_geometry_list); } - void LoadNames() + void InitNames() { - auto offsets_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::NAME_OFFSETS); - auto blocks_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::NAME_BLOCKS); + auto offsets_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::NAME_OFFSETS); + auto blocks_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::NAME_BLOCKS); util::ShM::vector name_offsets( - offsets_ptr, data_layout->num_entries[storage::SharedDataLayout::NAME_OFFSETS]); + offsets_ptr, data_layout->num_entries[storage::DataLayout::NAME_OFFSETS]); util::ShM::vector name_blocks( - blocks_ptr, data_layout->num_entries[storage::SharedDataLayout::NAME_BLOCKS]); + blocks_ptr, data_layout->num_entries[storage::DataLayout::NAME_BLOCKS]); - auto names_list_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::NAME_CHAR_LIST); + auto names_list_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::NAME_CHAR_LIST); util::ShM::vector names_char_list( - names_list_ptr, data_layout->num_entries[storage::SharedDataLayout::NAME_CHAR_LIST]); + names_list_ptr, data_layout->num_entries[storage::DataLayout::NAME_CHAR_LIST]); m_name_table = std::make_unique>( name_offsets, name_blocks, static_cast(names_char_list.size())); m_names_char_list = std::move(names_char_list); } - void LoadTurnLaneDescriptions() + void InitTurnLaneDescriptions() { auto offsets_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::LANE_DESCRIPTION_OFFSETS); + memory_block, storage::DataLayout::LANE_DESCRIPTION_OFFSETS); util::ShM::vector offsets( - offsets_ptr, - data_layout->num_entries[storage::SharedDataLayout::LANE_DESCRIPTION_OFFSETS]); + offsets_ptr, data_layout->num_entries[storage::DataLayout::LANE_DESCRIPTION_OFFSETS]); m_lane_description_offsets = std::move(offsets); auto masks_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::LANE_DESCRIPTION_MASKS); + memory_block, storage::DataLayout::LANE_DESCRIPTION_MASKS); util::ShM::vector masks( - masks_ptr, data_layout->num_entries[storage::SharedDataLayout::LANE_DESCRIPTION_MASKS]); + masks_ptr, data_layout->num_entries[storage::DataLayout::LANE_DESCRIPTION_MASKS]); m_lane_description_masks = std::move(masks); } - void LoadCoreInformation() + void InitCoreInformation() { - auto core_marker_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::CORE_MARKER); + auto core_marker_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::CORE_MARKER); util::ShM::vector is_core_node( - core_marker_ptr, data_layout->num_entries[storage::SharedDataLayout::CORE_MARKER]); + core_marker_ptr, data_layout->num_entries[storage::DataLayout::CORE_MARKER]); m_is_core_node = std::move(is_core_node); } - void LoadGeometries() + void InitGeometries() { - auto geometries_index_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::GEOMETRIES_INDEX); + auto geometries_index_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::GEOMETRIES_INDEX); util::ShM::vector geometry_begin_indices( - geometries_index_ptr, - data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_INDEX]); + geometries_index_ptr, data_layout->num_entries[storage::DataLayout::GEOMETRIES_INDEX]); m_geometry_indices = std::move(geometry_begin_indices); auto geometries_node_list_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::GEOMETRIES_NODE_LIST); + memory_block, storage::DataLayout::GEOMETRIES_NODE_LIST); util::ShM::vector geometry_node_list( geometries_node_list_ptr, - data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_NODE_LIST]); + data_layout->num_entries[storage::DataLayout::GEOMETRIES_NODE_LIST]); m_geometry_node_list = std::move(geometry_node_list); auto geometries_fwd_weight_list_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST); + memory_block, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST); util::ShM::vector geometry_fwd_weight_list( geometries_fwd_weight_list_ptr, - data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST]); + data_layout->num_entries[storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]); m_geometry_fwd_weight_list = std::move(geometry_fwd_weight_list); auto geometries_rev_weight_list_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST); + memory_block, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST); util::ShM::vector geometry_rev_weight_list( geometries_rev_weight_list_ptr, - data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST]); + data_layout->num_entries[storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST]); m_geometry_rev_weight_list = std::move(geometry_rev_weight_list); - auto datasources_list_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::DATASOURCES_LIST); + auto datasources_list_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::DATASOURCES_LIST); util::ShM::vector datasources_list( - datasources_list_ptr, - data_layout->num_entries[storage::SharedDataLayout::DATASOURCES_LIST]); + datasources_list_ptr, data_layout->num_entries[storage::DataLayout::DATASOURCES_LIST]); m_datasource_list = std::move(datasources_list); - auto datasource_name_data_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_DATA); + auto datasource_name_data_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::DATASOURCE_NAME_DATA); util::ShM::vector datasource_name_data( datasource_name_data_ptr, - data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_DATA]); + data_layout->num_entries[storage::DataLayout::DATASOURCE_NAME_DATA]); m_datasource_name_data = std::move(datasource_name_data); auto datasource_name_offsets_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_OFFSETS); + memory_block, storage::DataLayout::DATASOURCE_NAME_OFFSETS); util::ShM::vector datasource_name_offsets( datasource_name_offsets_ptr, - data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_OFFSETS]); + data_layout->num_entries[storage::DataLayout::DATASOURCE_NAME_OFFSETS]); m_datasource_name_offsets = std::move(datasource_name_offsets); auto datasource_name_lengths_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_LENGTHS); + memory_block, storage::DataLayout::DATASOURCE_NAME_LENGTHS); util::ShM::vector datasource_name_lengths( datasource_name_lengths_ptr, - data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_LENGTHS]); + data_layout->num_entries[storage::DataLayout::DATASOURCE_NAME_LENGTHS]); m_datasource_name_lengths = std::move(datasource_name_lengths); } - void LoadIntersectionClasses() + void InitIntersectionClasses() { auto bearing_class_id_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::BEARING_CLASSID); + memory_block, storage::DataLayout::BEARING_CLASSID); typename util::ShM::vector bearing_class_id_table( - bearing_class_id_ptr, - data_layout->num_entries[storage::SharedDataLayout::BEARING_CLASSID]); + bearing_class_id_ptr, data_layout->num_entries[storage::DataLayout::BEARING_CLASSID]); m_bearing_class_id_table = std::move(bearing_class_id_table); auto bearing_class_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::BEARING_VALUES); + memory_block, storage::DataLayout::BEARING_VALUES); typename util::ShM::vector bearing_class_table( - bearing_class_ptr, data_layout->num_entries[storage::SharedDataLayout::BEARING_VALUES]); + bearing_class_ptr, data_layout->num_entries[storage::DataLayout::BEARING_VALUES]); m_bearing_values_table = std::move(bearing_class_table); - auto offsets_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::BEARING_OFFSETS); - auto blocks_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::BEARING_BLOCKS); + auto offsets_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::BEARING_OFFSETS); + auto blocks_ptr = + data_layout->GetBlockPtr(memory_block, storage::DataLayout::BEARING_BLOCKS); util::ShM::vector bearing_offsets( - offsets_ptr, data_layout->num_entries[storage::SharedDataLayout::BEARING_OFFSETS]); + offsets_ptr, data_layout->num_entries[storage::DataLayout::BEARING_OFFSETS]); util::ShM::vector bearing_blocks( - blocks_ptr, data_layout->num_entries[storage::SharedDataLayout::BEARING_BLOCKS]); + blocks_ptr, data_layout->num_entries[storage::DataLayout::BEARING_BLOCKS]); m_bearing_ranges_table = std::make_unique>( bearing_offsets, bearing_blocks, static_cast(m_bearing_values_table.size())); auto entry_class_ptr = data_layout->GetBlockPtr( - shared_memory, storage::SharedDataLayout::ENTRY_CLASS); + memory_block, storage::DataLayout::ENTRY_CLASS); typename util::ShM::vector entry_class_table( - entry_class_ptr, data_layout->num_entries[storage::SharedDataLayout::ENTRY_CLASS]); + entry_class_ptr, data_layout->num_entries[storage::DataLayout::ENTRY_CLASS]); m_entry_class_table = std::move(entry_class_table); } public: - // this function handle the deallocation of the shared memory it we can prove it will not be - // used anymore - virtual ~SharedDataFacade() + void Init() { - boost::interprocess::scoped_lock exclusive_lock( - data_region == storage::DATA_1 ? shared_barriers->regions_1_mutex - : shared_barriers->regions_2_mutex, - boost::interprocess::defer_lock); - - // if this returns false this is still in use - if (exclusive_lock.try_lock()) - { - // Now check if this is still the newest dataset - const boost::interprocess::sharable_lock - lock(shared_barriers->current_regions_mutex); - - auto shared_regions = storage::makeSharedMemory(storage::CURRENT_REGIONS); - const auto current_timestamp = - static_cast(shared_regions->Ptr()); - - if (current_timestamp->timestamp == shared_timestamp) - { - util::SimpleLogger().Write(logDEBUG) << "Retaining data with shared timestamp " - << shared_timestamp; - } - else - { - storage::SharedMemory::Remove(data_region); - storage::SharedMemory::Remove(layout_region); - } - } - } - - SharedDataFacade(const std::shared_ptr &shared_barriers_, - storage::SharedDataType layout_region_, - storage::SharedDataType data_region_, - unsigned shared_timestamp_) - : shared_barriers(shared_barriers_), layout_region(layout_region_), - data_region(data_region_), shared_timestamp(shared_timestamp_) - { - util::SimpleLogger().Write(logDEBUG) << "Loading new data with shared timestamp " - << shared_timestamp; - - BOOST_ASSERT(storage::SharedMemory::RegionExists(layout_region)); - m_layout_memory = storage::makeSharedMemory(layout_region); - - data_layout = static_cast(m_layout_memory->Ptr()); - - BOOST_ASSERT(storage::SharedMemory::RegionExists(data_region)); - m_large_memory = storage::makeSharedMemory(data_region); - shared_memory = (char *)(m_large_memory->Ptr()); - - LoadGraph(); - LoadChecksum(); - LoadNodeAndEdgeInformation(); - LoadGeometries(); - LoadTimestamp(); - LoadViaNodeList(); - LoadNames(); - LoadTurnLaneDescriptions(); - LoadCoreInformation(); - LoadProfileProperties(); - LoadRTree(); - LoadIntersectionClasses(); + InitGraph(); + InitChecksum(); + InitNodeAndEdgeInformation(); + InitGeometries(); + InitTimestamp(); + InitViaNodeList(); + InitNames(); + InitTurnLaneDescriptions(); + InitCoreInformation(); + InitProfileProperties(); + InitRTree(); + InitIntersectionClasses(); } // search graph access diff --git a/include/engine/datafacade/shared_memory_datafacade.hpp b/include/engine/datafacade/shared_memory_datafacade.hpp new file mode 100644 index 000000000..16c06e1fe --- /dev/null +++ b/include/engine/datafacade/shared_memory_datafacade.hpp @@ -0,0 +1,88 @@ +#ifndef SHARED_MEMORY_DATAFACADE_HPP +#define SHARED_MEMORY_DATAFACADE_HPP + +// implements all data storage when shared memory _IS_ used + +#include "storage/shared_barriers.hpp" +#include "storage/shared_datatype.hpp" +#include "storage/shared_memory.hpp" +#include "engine/datafacade/memory_datafacade_base.hpp" + +namespace osrm +{ +namespace engine +{ +namespace datafacade +{ + +class SharedDataFacade : public MemoryDataFacadeBase +{ + + protected: + std::shared_ptr shared_barriers; + storage::SharedDataType layout_region; + storage::SharedDataType data_region; + unsigned shared_timestamp; + + SharedDataFacade() {} + + public: + // this function handle the deallocation of the shared memory it we can prove it will not be + // used anymore + virtual ~SharedDataFacade() + { + boost::interprocess::scoped_lock exclusive_lock( + data_region == storage::DATA_1 ? shared_barriers->regions_1_mutex + : shared_barriers->regions_2_mutex, + boost::interprocess::defer_lock); + + // if this returns false this is still in use + if (exclusive_lock.try_lock()) + { + // Now check if this is still the newest dataset + const boost::interprocess::sharable_lock + lock(shared_barriers->current_regions_mutex); + + auto shared_regions = storage::makeSharedMemory(storage::CURRENT_REGIONS); + const auto current_timestamp = + static_cast(shared_regions->Ptr()); + + if (current_timestamp->timestamp == shared_timestamp) + { + util::SimpleLogger().Write(logDEBUG) << "Retaining data with shared timestamp " + << shared_timestamp; + } + else + { + storage::SharedMemory::Remove(data_region); + storage::SharedMemory::Remove(layout_region); + } + } + } + + SharedDataFacade(const std::shared_ptr &shared_barriers_, + storage::SharedDataType layout_region_, + storage::SharedDataType data_region_, + unsigned shared_timestamp_) + : shared_barriers(shared_barriers_), layout_region(layout_region_), + data_region(data_region_), shared_timestamp(shared_timestamp_) + { + util::SimpleLogger().Write(logDEBUG) << "Loading new data with shared timestamp " + << shared_timestamp; + + BOOST_ASSERT(storage::SharedMemory::RegionExists(layout_region)); + m_layout_memory = storage::makeSharedMemory(layout_region); + + data_layout = static_cast(m_layout_memory->Ptr()); + + BOOST_ASSERT(storage::SharedMemory::RegionExists(data_region)); + m_large_memory = storage::makeSharedMemory(data_region); + memory_block = (char *)(m_large_memory->Ptr()); + Init(); + } +}; +} +} +} + +#endif // SHARED_MEMORY_DATAFACADE_HPP diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp index 65ee9bd85..49a70e07a 100644 --- a/include/storage/shared_datatype.hpp +++ b/include/storage/shared_datatype.hpp @@ -7,6 +7,7 @@ #include #include +#include namespace osrm { @@ -54,7 +55,7 @@ const constexpr char *block_id_to_name[] = {"NAME_OFFSETS", "LANE_DESCRIPTION_OFFSETS", "LANE_DESCRIPTION_MASKS"}; -struct SharedDataLayout +struct DataLayout { enum BlockID { @@ -101,7 +102,7 @@ struct SharedDataLayout std::array num_entries; std::array entry_size; - SharedDataLayout() : num_entries(), entry_size() {} + DataLayout() : num_entries(), entry_size() {} template inline void SetBlockSize(BlockID bid, uint64_t entries) { @@ -171,6 +172,20 @@ struct SharedDataLayout return ptr; } + friend std::ostream &operator<<(std::ostream &os, const DataLayout &layout) + { + os << "Memory layout: " << std::endl; + os << " Total size: " << layout.GetSizeOfLayout() << std::endl; + + for (unsigned bid = 0; bid < BlockID::NUM_BLOCKS; bid++) + { + os << " " << block_id_to_name[bid] + << " offset: " << layout.GetBlockOffset(BlockID(bid)) + << " size: " << layout.GetBlockSize(BlockID(bid)) << std::endl; + } + + return os; + } }; enum SharedDataType @@ -214,7 +229,7 @@ inline std::string regionToString(const SharedDataType region) } } -static_assert(sizeof(block_id_to_name) / sizeof(*block_id_to_name) == SharedDataLayout::NUM_BLOCKS, +static_assert(sizeof(block_id_to_name) / sizeof(*block_id_to_name) == DataLayout::NUM_BLOCKS, "Number of blocks needs to match the number of Block names."); } } diff --git a/include/storage/storage.hpp b/include/storage/storage.hpp index e914e5872..4642c72cd 100644 --- a/include/storage/storage.hpp +++ b/include/storage/storage.hpp @@ -29,6 +29,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define STORAGE_HPP #include "storage/storage_config.hpp" +#include "storage/shared_datatype.hpp" #include @@ -52,6 +53,9 @@ class Storage ReturnCode Run(int max_wait); + void LoadLayout(DataLayout *layout); + void LoadData(DataLayout *layout_ptr, char *memory_ptr); + private: StorageConfig config; }; diff --git a/src/engine/engine.cpp b/src/engine/engine.cpp index 8650208ef..44372a446 100644 --- a/src/engine/engine.cpp +++ b/src/engine/engine.cpp @@ -3,8 +3,8 @@ #include "engine/engine_config.hpp" #include "engine/status.hpp" -#include "engine/datafacade/internal_datafacade.hpp" -#include "engine/datafacade/shared_datafacade.hpp" +#include "engine/datafacade/internal_memory_datafacade.hpp" +#include "engine/datafacade/shared_memory_datafacade.hpp" #include "storage/shared_barriers.hpp" #include "util/simple_logger.hpp" diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 28881e1be..7f712d6f3 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -184,257 +184,10 @@ Storage::ReturnCode Storage::Run(int max_wait) } // Allocate a memory layout in shared memory - auto layout_memory = makeSharedMemory(layout_region, sizeof(SharedDataLayout), true); - auto shared_layout_ptr = new (layout_memory->Ptr()) SharedDataLayout(); - auto absolute_file_index_path = boost::filesystem::absolute(config.file_index_path); + auto layout_memory = makeSharedMemory(layout_region, sizeof(DataLayout), true); + auto shared_layout_ptr = new (layout_memory->Ptr()) DataLayout(); - shared_layout_ptr->SetBlockSize(SharedDataLayout::FILE_INDEX_PATH, - absolute_file_index_path.string().length() + 1); - - // collect number of elements to store in shared memory object - util::SimpleLogger().Write() << "load names from: " << config.names_data_path; - // number of entries in name index - boost::filesystem::ifstream name_stream(config.names_data_path, std::ios::binary); - if (!name_stream) - { - throw util::exception("Could not open " + config.names_data_path.string() + - " for reading."); - } - unsigned name_blocks = 0; - name_stream.read((char *)&name_blocks, sizeof(unsigned)); - shared_layout_ptr->SetBlockSize(SharedDataLayout::NAME_OFFSETS, name_blocks); - shared_layout_ptr->SetBlockSize::BlockT>( - SharedDataLayout::NAME_BLOCKS, name_blocks); - BOOST_ASSERT_MSG(0 != name_blocks, "name file broken"); - - unsigned number_of_chars = 0; - name_stream.read((char *)&number_of_chars, sizeof(unsigned)); - shared_layout_ptr->SetBlockSize(SharedDataLayout::NAME_CHAR_LIST, number_of_chars); - - std::vector lane_description_offsets; - std::vector lane_description_masks; - if (!util::deserializeAdjacencyArray(config.turn_lane_description_path.string(), - lane_description_offsets, - lane_description_masks)) - throw util::exception("Failed to read lane descriptions from: " + - config.turn_lane_description_path.string()); - shared_layout_ptr->SetBlockSize(SharedDataLayout::LANE_DESCRIPTION_OFFSETS, - lane_description_offsets.size()); - shared_layout_ptr->SetBlockSize( - SharedDataLayout::LANE_DESCRIPTION_MASKS, lane_description_masks.size()); - - // Loading information for original edges - boost::filesystem::ifstream edges_input_stream(config.edges_data_path, std::ios::binary); - if (!edges_input_stream) - { - throw util::exception("Could not open " + config.edges_data_path.string() + - " for reading."); - } - const auto number_of_original_edges = io::readElementCount(edges_input_stream); - - // note: settings this all to the same size is correct, we extract them from the same struct - shared_layout_ptr->SetBlockSize(SharedDataLayout::VIA_NODE_LIST, - number_of_original_edges); - shared_layout_ptr->SetBlockSize(SharedDataLayout::NAME_ID_LIST, - number_of_original_edges); - shared_layout_ptr->SetBlockSize(SharedDataLayout::TRAVEL_MODE, - number_of_original_edges); - shared_layout_ptr->SetBlockSize(SharedDataLayout::PRE_TURN_BEARING, - number_of_original_edges); - shared_layout_ptr->SetBlockSize( - SharedDataLayout::POST_TURN_BEARING, number_of_original_edges); - shared_layout_ptr->SetBlockSize( - SharedDataLayout::TURN_INSTRUCTION, number_of_original_edges); - shared_layout_ptr->SetBlockSize(SharedDataLayout::LANE_DATA_ID, - number_of_original_edges); - shared_layout_ptr->SetBlockSize(SharedDataLayout::ENTRY_CLASSID, - number_of_original_edges); - - boost::filesystem::ifstream hsgr_input_stream(config.hsgr_data_path, std::ios::binary); - if (!hsgr_input_stream) - { - throw util::exception("Could not open " + config.hsgr_data_path.string() + " for reading."); - } - - const auto hsgr_header = io::readHSGRHeader(hsgr_input_stream); - shared_layout_ptr->SetBlockSize(SharedDataLayout::HSGR_CHECKSUM, 1); - shared_layout_ptr->SetBlockSize(SharedDataLayout::GRAPH_NODE_LIST, - hsgr_header.number_of_nodes); - shared_layout_ptr->SetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST, - hsgr_header.number_of_edges); - - // load rsearch tree size - boost::filesystem::ifstream tree_node_file(config.ram_index_path, std::ios::binary); - - const auto tree_size = io::readElementCount(tree_node_file); - shared_layout_ptr->SetBlockSize(SharedDataLayout::R_SEARCH_TREE, tree_size); - - // allocate space in shared memory for profile properties - const auto properties_size = io::readPropertiesCount(); - shared_layout_ptr->SetBlockSize(SharedDataLayout::PROPERTIES, - properties_size); - - // read timestampsize - boost::filesystem::ifstream timestamp_stream(config.timestamp_path); - if (!timestamp_stream) - { - throw util::exception("Could not open " + config.timestamp_path.string() + " for reading."); - } - const auto timestamp_size = io::readNumberOfBytes(timestamp_stream); - shared_layout_ptr->SetBlockSize(SharedDataLayout::TIMESTAMP, timestamp_size); - - // load core marker size - boost::filesystem::ifstream core_marker_file(config.core_data_path, std::ios::binary); - if (!core_marker_file) - { - throw util::exception("Could not open " + config.core_data_path.string() + " for reading."); - } - - uint32_t number_of_core_markers = 0; - core_marker_file.read((char *)&number_of_core_markers, sizeof(uint32_t)); - shared_layout_ptr->SetBlockSize(SharedDataLayout::CORE_MARKER, - number_of_core_markers); - - // load coordinate size - boost::filesystem::ifstream nodes_input_stream(config.nodes_data_path, std::ios::binary); - if (!nodes_input_stream) - { - throw util::exception("Could not open " + config.core_data_path.string() + " for reading."); - } - const auto coordinate_list_size = io::readElementCount(nodes_input_stream); - shared_layout_ptr->SetBlockSize(SharedDataLayout::COORDINATE_LIST, - coordinate_list_size); - // we'll read a list of OSM node IDs from the same data, so set the block size for the same - // number of items: - shared_layout_ptr->SetBlockSize( - SharedDataLayout::OSM_NODE_ID_LIST, - util::PackedVector::elements_to_blocks(coordinate_list_size)); - - // load geometries sizes - boost::filesystem::ifstream geometry_input_stream(config.geometries_path, std::ios::binary); - if (!geometry_input_stream) - { - throw util::exception("Could not open " + config.geometries_path.string() + - " for reading."); - } - unsigned number_of_geometries_indices = 0; - unsigned number_of_compressed_geometries = 0; - - geometry_input_stream.read((char *)&number_of_geometries_indices, sizeof(unsigned)); - shared_layout_ptr->SetBlockSize(SharedDataLayout::GEOMETRIES_INDEX, - number_of_geometries_indices); - boost::iostreams::seek( - geometry_input_stream, number_of_geometries_indices * sizeof(unsigned), BOOST_IOS::cur); - geometry_input_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned)); - shared_layout_ptr->SetBlockSize(SharedDataLayout::GEOMETRIES_NODE_LIST, - number_of_compressed_geometries); - shared_layout_ptr->SetBlockSize(SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST, - number_of_compressed_geometries); - shared_layout_ptr->SetBlockSize(SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST, - number_of_compressed_geometries); - - // load datasource sizes. This file is optional, and it's non-fatal if it doesn't - // exist. - boost::filesystem::ifstream geometry_datasource_input_stream(config.datasource_indexes_path, - std::ios::binary); - if (!geometry_datasource_input_stream) - { - throw util::exception("Could not open " + config.datasource_indexes_path.string() + - " for reading."); - } - const auto number_of_compressed_datasources = - io::readElementCount(geometry_datasource_input_stream); - shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCES_LIST, - number_of_compressed_datasources); - - // Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't - // exist - - boost::filesystem::ifstream datasource_names_input_stream(config.datasource_names_path, - std::ios::binary); - if (!datasource_names_input_stream) - { - throw util::exception("Could not open " + config.datasource_names_path.string() + - " for reading."); - } - const io::DatasourceNamesData datasource_names_data = - io::readDatasourceNames(datasource_names_input_stream); - - shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCE_NAME_DATA, - datasource_names_data.names.size()); - shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCE_NAME_OFFSETS, - datasource_names_data.offsets.size()); - shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCE_NAME_LENGTHS, - datasource_names_data.lengths.size()); - boost::filesystem::ifstream intersection_stream(config.intersection_class_path, - std::ios::binary); - - if (!static_cast(intersection_stream)) - throw util::exception("Could not open " + config.intersection_class_path.string() + - " for reading."); - - if (!util::readAndCheckFingerprint(intersection_stream)) - throw util::exception("Fingerprint of " + config.intersection_class_path.string() + - " does not match or could not read from file"); - - std::vector bearing_class_id_table; - if (!util::deserializeVector(intersection_stream, bearing_class_id_table)) - throw util::exception("Failed to bearing class ids read from " + - config.names_data_path.string()); - - shared_layout_ptr->SetBlockSize(SharedDataLayout::BEARING_CLASSID, - bearing_class_id_table.size()); - unsigned bearing_blocks = 0; - intersection_stream.read((char *)&bearing_blocks, sizeof(unsigned)); - unsigned sum_lengths = 0; - intersection_stream.read((char *)&sum_lengths, sizeof(unsigned)); - - shared_layout_ptr->SetBlockSize(SharedDataLayout::BEARING_OFFSETS, bearing_blocks); - shared_layout_ptr->SetBlockSize::BlockT>( - SharedDataLayout::BEARING_BLOCKS, bearing_blocks); - - std::vector bearing_offsets_data(bearing_blocks); - std::vector::BlockT> bearing_blocks_data(bearing_blocks); - - if (bearing_blocks) - { - intersection_stream.read(reinterpret_cast(&bearing_offsets_data[0]), - bearing_blocks * sizeof(bearing_offsets_data[0])); - } - - if (bearing_blocks) - { - intersection_stream.read(reinterpret_cast(&bearing_blocks_data[0]), - bearing_blocks * sizeof(bearing_blocks_data[0])); - } - - std::uint64_t num_bearings; - intersection_stream.read(reinterpret_cast(&num_bearings), sizeof(num_bearings)); - - std::vector bearing_class_table(num_bearings); - intersection_stream.read(reinterpret_cast(&bearing_class_table[0]), - sizeof(bearing_class_table[0]) * num_bearings); - shared_layout_ptr->SetBlockSize(SharedDataLayout::BEARING_VALUES, - num_bearings); - - // Loading turn lane data - boost::filesystem::ifstream lane_data_stream(config.turn_lane_data_path, std::ios::binary); - std::uint64_t lane_tupel_count = 0; - lane_data_stream.read(reinterpret_cast(&lane_tupel_count), sizeof(lane_tupel_count)); - shared_layout_ptr->SetBlockSize( - SharedDataLayout::TURN_LANE_DATA, lane_tupel_count); - - if (!static_cast(intersection_stream)) - throw util::exception("Failed to read bearing values from " + - config.intersection_class_path.string()); - - std::vector entry_class_table; - if (!util::deserializeVector(intersection_stream, entry_class_table)) - throw util::exception("Failed to read entry classes from " + - config.intersection_class_path.string()); - - shared_layout_ptr->SetBlockSize(SharedDataLayout::ENTRY_CLASS, - entry_class_table.size()); + LoadLayout(shared_layout_ptr); // allocate shared memory block util::SimpleLogger().Write() << "allocating shared memory of " @@ -442,345 +195,14 @@ Storage::ReturnCode Storage::Run(int max_wait) auto shared_memory = makeSharedMemory(data_region, shared_layout_ptr->GetSizeOfLayout(), true); char *shared_memory_ptr = static_cast(shared_memory->Ptr()); - // read actual data into shared memory object // - - // hsgr checksum - unsigned *checksum_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::HSGR_CHECKSUM); - *checksum_ptr = hsgr_header.checksum; - - // ram index file name - char *file_index_path_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::FILE_INDEX_PATH); - // make sure we have 0 ending - std::fill(file_index_path_ptr, - file_index_path_ptr + - shared_layout_ptr->GetBlockSize(SharedDataLayout::FILE_INDEX_PATH), - 0); - std::copy(absolute_file_index_path.string().begin(), - absolute_file_index_path.string().end(), - file_index_path_ptr); - - // Loading street names - unsigned *name_offsets_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::NAME_OFFSETS); - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_OFFSETS) > 0) - { - name_stream.read((char *)name_offsets_ptr, - shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_OFFSETS)); - } - - unsigned *name_blocks_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::NAME_BLOCKS); - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_BLOCKS) > 0) - { - name_stream.read((char *)name_blocks_ptr, - shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_BLOCKS)); - } - - char *name_char_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::NAME_CHAR_LIST); - unsigned temp_length = 0; - name_stream.read((char *)&temp_length, sizeof(unsigned)); - - BOOST_ASSERT_MSG(shared_layout_ptr->AlignBlockSize(temp_length) == - shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST), - "Name file corrupted!"); - - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST) > 0) - { - name_stream.read(name_char_ptr, - shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST)); - } - name_stream.close(); - - // make sure do write canary... - auto *turn_lane_data_ptr = - shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::TURN_LANE_DATA); - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::TURN_LANE_DATA) > 0) - { - lane_data_stream.read(reinterpret_cast(turn_lane_data_ptr), - shared_layout_ptr->GetBlockSize(SharedDataLayout::TURN_LANE_DATA)); - } - lane_data_stream.close(); - - auto *turn_lane_offset_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::LANE_DESCRIPTION_OFFSETS); - if (!lane_description_offsets.empty()) - { - BOOST_ASSERT(shared_layout_ptr->GetBlockSize(SharedDataLayout::LANE_DESCRIPTION_OFFSETS) >= - sizeof(lane_description_offsets[0]) * lane_description_offsets.size()); - std::copy( - lane_description_offsets.begin(), lane_description_offsets.end(), turn_lane_offset_ptr); - std::vector tmp; - lane_description_offsets.swap(tmp); - } - - auto *turn_lane_mask_ptr = - shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::LANE_DESCRIPTION_MASKS); - if (!lane_description_masks.empty()) - { - BOOST_ASSERT(shared_layout_ptr->GetBlockSize(SharedDataLayout::LANE_DESCRIPTION_MASKS) >= - sizeof(lane_description_masks[0]) * lane_description_masks.size()); - std::copy(lane_description_masks.begin(), lane_description_masks.end(), turn_lane_mask_ptr); - std::vector tmp; - lane_description_masks.swap(tmp); - } - - // load original edge information - GeometryID *via_geometry_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::VIA_NODE_LIST); - - unsigned *name_id_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::NAME_ID_LIST); - - extractor::TravelMode *travel_mode_ptr = - shared_layout_ptr->GetBlockPtr(shared_memory_ptr, - SharedDataLayout::TRAVEL_MODE); - util::guidance::TurnBearing *pre_turn_bearing_ptr = - shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::PRE_TURN_BEARING); - util::guidance::TurnBearing *post_turn_bearing_ptr = - shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::POST_TURN_BEARING); - - LaneDataID *lane_data_id_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::LANE_DATA_ID); - - extractor::guidance::TurnInstruction *turn_instructions_ptr = - shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::TURN_INSTRUCTION); - - EntryClassID *entry_class_id_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::ENTRY_CLASSID); - - io::readEdges(edges_input_stream, - via_geometry_ptr, - name_id_ptr, - turn_instructions_ptr, - lane_data_id_ptr, - travel_mode_ptr, - entry_class_id_ptr, - pre_turn_bearing_ptr, - post_turn_bearing_ptr, - number_of_original_edges); - edges_input_stream.close(); - - // load compressed geometry - unsigned temporary_value; - unsigned *geometries_index_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::GEOMETRIES_INDEX); - geometry_input_stream.seekg(0, geometry_input_stream.beg); - geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned)); - BOOST_ASSERT(temporary_value == - shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_INDEX]); - - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_INDEX) > 0) - { - geometry_input_stream.read( - (char *)geometries_index_ptr, - shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_INDEX)); - } - NodeID *geometries_node_id_list_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::GEOMETRIES_NODE_LIST); - - geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned)); - BOOST_ASSERT(temporary_value == - shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_NODE_LIST]); - - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_NODE_LIST) > 0) - { - geometry_input_stream.read( - (char *)geometries_node_id_list_ptr, - shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_NODE_LIST)); - } - EdgeWeight *geometries_fwd_weight_list_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST); - - BOOST_ASSERT(temporary_value == - shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST]); - - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST) > 0) - { - geometry_input_stream.read( - (char *)geometries_fwd_weight_list_ptr, - shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST)); - } - EdgeWeight *geometries_rev_weight_list_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST); - - BOOST_ASSERT(temporary_value == - shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST]); - - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST) > 0) - { - geometry_input_stream.read( - (char *)geometries_rev_weight_list_ptr, - shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST)); - } - - // load datasource information (if it exists) - uint8_t *datasources_list_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::DATASOURCES_LIST); - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCES_LIST) > 0) - { - io::readDatasourceIndexes(geometry_datasource_input_stream, - datasources_list_ptr, - number_of_compressed_datasources); - } - - // load datasource name information (if it exists) - - char *datasource_name_data_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_DATA); - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_DATA) > 0) - { - std::copy(datasource_names_data.names.begin(), - datasource_names_data.names.end(), - datasource_name_data_ptr); - } - - auto datasource_name_offsets_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_OFFSETS); - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_OFFSETS) > 0) - { - std::copy(datasource_names_data.offsets.begin(), - datasource_names_data.offsets.end(), - datasource_name_offsets_ptr); - } - - auto datasource_name_lengths_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_LENGTHS); - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_LENGTHS) > 0) - { - std::copy(datasource_names_data.lengths.begin(), - datasource_names_data.lengths.end(), - datasource_name_lengths_ptr); - } - - // Loading list of coordinates - util::Coordinate *coordinates_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::COORDINATE_LIST); - std::uint64_t *osmnodeid_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::OSM_NODE_ID_LIST); - util::PackedVector osmnodeid_list; - osmnodeid_list.reset(osmnodeid_ptr, - shared_layout_ptr->num_entries[SharedDataLayout::OSM_NODE_ID_LIST]); - io::readNodes(nodes_input_stream, coordinates_ptr, osmnodeid_list, coordinate_list_size); - nodes_input_stream.close(); - - // store timestamp - char *timestamp_ptr = - shared_layout_ptr->GetBlockPtr(shared_memory_ptr, SharedDataLayout::TIMESTAMP); - io::readTimestamp(timestamp_stream, timestamp_ptr, timestamp_size); - - // store search tree portion of rtree - RTreeNode *rtree_ptrtest = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::R_SEARCH_TREE); - io::readRamIndex(tree_node_file, rtree_ptrtest, tree_size); - - // load core markers - std::vector unpacked_core_markers(number_of_core_markers); - core_marker_file.read((char *)unpacked_core_markers.data(), - sizeof(char) * number_of_core_markers); - - unsigned *core_marker_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::CORE_MARKER); - - for (auto i = 0u; i < number_of_core_markers; ++i) - { - BOOST_ASSERT(unpacked_core_markers[i] == 0 || unpacked_core_markers[i] == 1); - - if (unpacked_core_markers[i] == 1) - { - const unsigned bucket = i / 32; - const unsigned offset = i % 32; - const unsigned value = [&] { - unsigned return_value = 0; - if (0 != offset) - { - return_value = core_marker_ptr[bucket]; - } - return return_value; - }(); - - core_marker_ptr[bucket] = (value | (1u << offset)); - } - } - - // load the nodes of the search graph - QueryGraph::NodeArrayEntry *graph_node_list_ptr = - shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::GRAPH_NODE_LIST); - - // load the edges of the search graph - QueryGraph::EdgeArrayEntry *graph_edge_list_ptr = - shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::GRAPH_EDGE_LIST); - - io::readHSGR(hsgr_input_stream, - graph_node_list_ptr, - hsgr_header.number_of_nodes, - graph_edge_list_ptr, - hsgr_header.number_of_edges); - hsgr_input_stream.close(); - - // load profile properties - extractor::ProfileProperties *profile_properties_ptr = - shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::PROPERTIES); - boost::filesystem::ifstream profile_properties_stream(config.properties_path); - if (!profile_properties_stream) - { - util::exception("Could not open " + config.properties_path.string() + " for reading!"); - } - io::readProperties( - profile_properties_stream, profile_properties_ptr, sizeof(extractor::ProfileProperties)); - - // load intersection classes - if (!bearing_class_id_table.empty()) - { - auto bearing_id_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::BEARING_CLASSID); - std::copy(bearing_class_id_table.begin(), bearing_class_id_table.end(), bearing_id_ptr); - } - - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::BEARING_OFFSETS) > 0) - { - auto *bearing_offsets_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::BEARING_OFFSETS); - std::copy(bearing_offsets_data.begin(), bearing_offsets_data.end(), bearing_offsets_ptr); - } - - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::BEARING_BLOCKS) > 0) - { - auto *bearing_blocks_ptr = - shared_layout_ptr->GetBlockPtr::BlockT, true>( - shared_memory_ptr, SharedDataLayout::BEARING_BLOCKS); - std::copy(bearing_blocks_data.begin(), bearing_blocks_data.end(), bearing_blocks_ptr); - } - - if (!bearing_class_table.empty()) - { - auto bearing_class_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::BEARING_VALUES); - std::copy(bearing_class_table.begin(), bearing_class_table.end(), bearing_class_ptr); - } - - if (!entry_class_table.empty()) - { - auto entry_class_ptr = shared_layout_ptr->GetBlockPtr( - shared_memory_ptr, SharedDataLayout::ENTRY_CLASS); - std::copy(entry_class_table.begin(), entry_class_table.end(), entry_class_ptr); - } + LoadData(shared_layout_ptr, shared_memory_ptr); auto data_type_memory = makeSharedMemory(CURRENT_REGIONS, sizeof(SharedDataTimestamp), true); SharedDataTimestamp *data_timestamp_ptr = static_cast(data_type_memory->Ptr()); { + boost::interprocess::scoped_lock current_regions_exclusive_lock; @@ -822,5 +244,763 @@ Storage::ReturnCode Storage::Run(int max_wait) return ReturnCode::Ok; } + +/** + * This function examines all our data files and figures out how much + * memory needs to be allocated, and the position of each data structure + * in that big block. It updates the fields in the DataLayout parameter. + */ +void Storage::LoadLayout(DataLayout *layout_ptr) +{ + auto absolute_file_index_path = boost::filesystem::absolute(config.file_index_path); + + layout_ptr->SetBlockSize(DataLayout::FILE_INDEX_PATH, + absolute_file_index_path.string().length() + 1); + + // collect number of elements to store in shared memory object + util::SimpleLogger().Write() << "load names from: " << config.names_data_path; + // number of entries in name index + boost::filesystem::ifstream name_stream(config.names_data_path, std::ios::binary); + if (!name_stream) + { + throw util::exception("Could not open " + config.names_data_path.string() + + " for reading."); + } + unsigned name_blocks = 0; + name_stream.read((char *)&name_blocks, sizeof(unsigned)); + layout_ptr->SetBlockSize(DataLayout::NAME_OFFSETS, name_blocks); + layout_ptr->SetBlockSize::BlockT>(DataLayout::NAME_BLOCKS, + name_blocks); + BOOST_ASSERT_MSG(0 != name_blocks, "name file broken"); + + unsigned number_of_chars = 0; + name_stream.read((char *)&number_of_chars, sizeof(unsigned)); + layout_ptr->SetBlockSize(DataLayout::NAME_CHAR_LIST, number_of_chars); + + std::vector lane_description_offsets; + std::vector lane_description_masks; + if (!util::deserializeAdjacencyArray(config.turn_lane_description_path.string(), + lane_description_offsets, + lane_description_masks)) + throw util::exception("Failed to read lane descriptions from: " + + config.turn_lane_description_path.string()); + layout_ptr->SetBlockSize(DataLayout::LANE_DESCRIPTION_OFFSETS, + lane_description_offsets.size()); + layout_ptr->SetBlockSize( + DataLayout::LANE_DESCRIPTION_MASKS, lane_description_masks.size()); + + // Loading information for original edges + boost::filesystem::ifstream edges_input_stream(config.edges_data_path, std::ios::binary); + if (!edges_input_stream) + { + throw util::exception("Could not open " + config.edges_data_path.string() + + " for reading."); + } + const auto number_of_original_edges = io::readElementCount(edges_input_stream); + + // note: settings this all to the same size is correct, we extract them from the same struct + layout_ptr->SetBlockSize(DataLayout::VIA_NODE_LIST, number_of_original_edges); + layout_ptr->SetBlockSize(DataLayout::NAME_ID_LIST, number_of_original_edges); + layout_ptr->SetBlockSize(DataLayout::TRAVEL_MODE, + number_of_original_edges); + layout_ptr->SetBlockSize(DataLayout::PRE_TURN_BEARING, + number_of_original_edges); + layout_ptr->SetBlockSize(DataLayout::POST_TURN_BEARING, + number_of_original_edges); + layout_ptr->SetBlockSize(DataLayout::TURN_INSTRUCTION, + number_of_original_edges); + layout_ptr->SetBlockSize(DataLayout::LANE_DATA_ID, number_of_original_edges); + layout_ptr->SetBlockSize(DataLayout::ENTRY_CLASSID, number_of_original_edges); + + boost::filesystem::ifstream hsgr_input_stream(config.hsgr_data_path, std::ios::binary); + if (!hsgr_input_stream) + { + throw util::exception("Could not open " + config.hsgr_data_path.string() + " for reading."); + } + + const auto hsgr_header = io::readHSGRHeader(hsgr_input_stream); + layout_ptr->SetBlockSize(DataLayout::HSGR_CHECKSUM, 1); + layout_ptr->SetBlockSize(DataLayout::GRAPH_NODE_LIST, + hsgr_header.number_of_nodes); + layout_ptr->SetBlockSize(DataLayout::GRAPH_EDGE_LIST, + hsgr_header.number_of_edges); + + // load rsearch tree size + boost::filesystem::ifstream tree_node_file(config.ram_index_path, std::ios::binary); + + const auto tree_size = io::readElementCount(tree_node_file); + layout_ptr->SetBlockSize(DataLayout::R_SEARCH_TREE, tree_size); + + // allocate space in shared memory for profile properties + const auto properties_size = io::readPropertiesCount(); + layout_ptr->SetBlockSize(DataLayout::PROPERTIES, properties_size); + + // read timestampsize + boost::filesystem::ifstream timestamp_stream(config.timestamp_path); + if (!timestamp_stream) + { + throw util::exception("Could not open " + config.timestamp_path.string() + " for reading."); + } + const auto timestamp_size = io::readNumberOfBytes(timestamp_stream); + layout_ptr->SetBlockSize(DataLayout::TIMESTAMP, timestamp_size); + + // load core marker size + boost::filesystem::ifstream core_marker_file(config.core_data_path, std::ios::binary); + if (!core_marker_file) + { + throw util::exception("Could not open " + config.core_data_path.string() + " for reading."); + } + + uint32_t number_of_core_markers = 0; + core_marker_file.read((char *)&number_of_core_markers, sizeof(uint32_t)); + layout_ptr->SetBlockSize(DataLayout::CORE_MARKER, number_of_core_markers); + + // load coordinate size + boost::filesystem::ifstream nodes_input_stream(config.nodes_data_path, std::ios::binary); + if (!nodes_input_stream) + { + throw util::exception("Could not open " + config.core_data_path.string() + " for reading."); + } + const auto coordinate_list_size = io::readElementCount(nodes_input_stream); + layout_ptr->SetBlockSize(DataLayout::COORDINATE_LIST, coordinate_list_size); + // we'll read a list of OSM node IDs from the same data, so set the block size for the same + // number of items: + layout_ptr->SetBlockSize( + DataLayout::OSM_NODE_ID_LIST, + util::PackedVector::elements_to_blocks(coordinate_list_size)); + + // load geometries sizes + boost::filesystem::ifstream geometry_input_stream(config.geometries_path, std::ios::binary); + if (!geometry_input_stream) + { + throw util::exception("Could not open " + config.geometries_path.string() + + " for reading."); + } + unsigned number_of_geometries_indices = 0; + unsigned number_of_compressed_geometries = 0; + + geometry_input_stream.read((char *)&number_of_geometries_indices, sizeof(unsigned)); + layout_ptr->SetBlockSize(DataLayout::GEOMETRIES_INDEX, number_of_geometries_indices); + boost::iostreams::seek( + geometry_input_stream, number_of_geometries_indices * sizeof(unsigned), BOOST_IOS::cur); + geometry_input_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned)); + layout_ptr->SetBlockSize(DataLayout::GEOMETRIES_NODE_LIST, + number_of_compressed_geometries); + layout_ptr->SetBlockSize(DataLayout::GEOMETRIES_FWD_WEIGHT_LIST, + number_of_compressed_geometries); + layout_ptr->SetBlockSize(DataLayout::GEOMETRIES_REV_WEIGHT_LIST, + number_of_compressed_geometries); + + // load datasource sizes. This file is optional, and it's non-fatal if it doesn't + // exist. + boost::filesystem::ifstream geometry_datasource_input_stream(config.datasource_indexes_path, + std::ios::binary); + if (!geometry_datasource_input_stream) + { + throw util::exception("Could not open " + config.datasource_indexes_path.string() + + " for reading."); + } + const auto number_of_compressed_datasources = + io::readElementCount(geometry_datasource_input_stream); + layout_ptr->SetBlockSize(DataLayout::DATASOURCES_LIST, + number_of_compressed_datasources); + + // Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't + // exist + + boost::filesystem::ifstream datasource_names_input_stream(config.datasource_names_path, + std::ios::binary); + if (!datasource_names_input_stream) + { + throw util::exception("Could not open " + config.datasource_names_path.string() + + " for reading."); + } + + const io::DatasourceNamesData datasource_names_data = + io::readDatasourceNames(datasource_names_input_stream); + + layout_ptr->SetBlockSize(DataLayout::DATASOURCE_NAME_DATA, + datasource_names_data.names.size()); + layout_ptr->SetBlockSize(DataLayout::DATASOURCE_NAME_OFFSETS, + datasource_names_data.offsets.size()); + layout_ptr->SetBlockSize(DataLayout::DATASOURCE_NAME_LENGTHS, + datasource_names_data.lengths.size()); + + boost::filesystem::ifstream intersection_stream(config.intersection_class_path, + std::ios::binary); + + if (!static_cast(intersection_stream)) + throw util::exception("Could not open " + config.intersection_class_path.string() + + " for reading."); + + if (!util::readAndCheckFingerprint(intersection_stream)) + throw util::exception("Fingerprint of " + config.intersection_class_path.string() + + " does not match or could not read from file"); + + std::vector bearing_class_id_table; + if (!util::deserializeVector(intersection_stream, bearing_class_id_table)) + throw util::exception("Failed to bearing class ids read from " + + config.names_data_path.string()); + + layout_ptr->SetBlockSize(DataLayout::BEARING_CLASSID, + bearing_class_id_table.size()); + unsigned bearing_blocks = 0; + intersection_stream.read((char *)&bearing_blocks, sizeof(unsigned)); + unsigned sum_lengths = 0; + intersection_stream.read((char *)&sum_lengths, sizeof(unsigned)); + + layout_ptr->SetBlockSize(DataLayout::BEARING_OFFSETS, bearing_blocks); + layout_ptr->SetBlockSize::BlockT>( + DataLayout::BEARING_BLOCKS, bearing_blocks); + + std::vector bearing_offsets_data(bearing_blocks); + std::vector::BlockT> bearing_blocks_data(bearing_blocks); + + if (bearing_blocks) + { + intersection_stream.read(reinterpret_cast(&bearing_offsets_data[0]), + bearing_blocks * sizeof(bearing_offsets_data[0])); + } + + if (bearing_blocks) + { + intersection_stream.read(reinterpret_cast(&bearing_blocks_data[0]), + bearing_blocks * sizeof(bearing_blocks_data[0])); + } + + std::uint64_t num_bearings; + intersection_stream.read(reinterpret_cast(&num_bearings), sizeof(num_bearings)); + + std::vector bearing_class_table(num_bearings); + intersection_stream.read(reinterpret_cast(&bearing_class_table[0]), + sizeof(bearing_class_table[0]) * num_bearings); + layout_ptr->SetBlockSize(DataLayout::BEARING_VALUES, num_bearings); + + // Loading turn lane data + boost::filesystem::ifstream lane_data_stream(config.turn_lane_data_path, std::ios::binary); + std::uint64_t lane_tupel_count = 0; + lane_data_stream.read(reinterpret_cast(&lane_tupel_count), sizeof(lane_tupel_count)); + layout_ptr->SetBlockSize(DataLayout::TURN_LANE_DATA, + lane_tupel_count); + + if (!static_cast(intersection_stream)) + throw util::exception("Failed to read bearing values from " + + config.intersection_class_path.string()); + + std::vector entry_class_table; + if (!util::deserializeVector(intersection_stream, entry_class_table)) + throw util::exception("Failed to read entry classes from " + + config.intersection_class_path.string()); + + layout_ptr->SetBlockSize(DataLayout::ENTRY_CLASS, + entry_class_table.size()); +} + +void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr) +{ + + // read actual data into shared memory object // + + // hsgr checksum + { + boost::filesystem::ifstream hsgr_input_stream(config.hsgr_data_path, std::ios::binary); + auto hsgr_header = io::readHSGRHeader(hsgr_input_stream); + unsigned *checksum_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::HSGR_CHECKSUM); + *checksum_ptr = hsgr_header.checksum; + + // load the nodes of the search graph + QueryGraph::NodeArrayEntry *graph_node_list_ptr = + layout_ptr->GetBlockPtr(memory_ptr, + DataLayout::GRAPH_NODE_LIST); + + // load the edges of the search graph + QueryGraph::EdgeArrayEntry *graph_edge_list_ptr = + layout_ptr->GetBlockPtr(memory_ptr, + DataLayout::GRAPH_EDGE_LIST); + + io::readHSGR(hsgr_input_stream, + graph_node_list_ptr, + hsgr_header.number_of_nodes, + graph_edge_list_ptr, + hsgr_header.number_of_edges); + hsgr_input_stream.close(); + } + + // ram index file name + { + char *file_index_path_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::FILE_INDEX_PATH); + // make sure we have 0 ending + std::fill(file_index_path_ptr, + file_index_path_ptr + layout_ptr->GetBlockSize(DataLayout::FILE_INDEX_PATH), + 0); + auto absolute_file_index_path = boost::filesystem::absolute(config.file_index_path); + std::copy(absolute_file_index_path.string().begin(), + absolute_file_index_path.string().end(), + file_index_path_ptr); + } + + // Name data + { + /* To be replaced by io loading code */ + boost::filesystem::ifstream name_stream(config.names_data_path, std::ios::binary); + unsigned name_blocks = 0; + name_stream.read((char *)&name_blocks, sizeof(unsigned)); + unsigned number_of_chars = 0; + name_stream.read((char *)&number_of_chars, sizeof(unsigned)); + /* END to be replaced */ + + // Loading street names + unsigned *name_offsets_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::NAME_OFFSETS); + if (layout_ptr->GetBlockSize(DataLayout::NAME_OFFSETS) > 0) + { + name_stream.read((char *)name_offsets_ptr, + layout_ptr->GetBlockSize(DataLayout::NAME_OFFSETS)); + } + + unsigned *name_blocks_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::NAME_BLOCKS); + if (layout_ptr->GetBlockSize(DataLayout::NAME_BLOCKS) > 0) + { + name_stream.read((char *)name_blocks_ptr, + layout_ptr->GetBlockSize(DataLayout::NAME_BLOCKS)); + } + + char *name_char_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::NAME_CHAR_LIST); + unsigned temp_length = 0; + name_stream.read((char *)&temp_length, sizeof(unsigned)); + + BOOST_ASSERT_MSG(layout_ptr->AlignBlockSize(temp_length) == + layout_ptr->GetBlockSize(DataLayout::NAME_CHAR_LIST), + "Name file corrupted!"); + + if (layout_ptr->GetBlockSize(DataLayout::NAME_CHAR_LIST) > 0) + { + name_stream.read(name_char_ptr, layout_ptr->GetBlockSize(DataLayout::NAME_CHAR_LIST)); + } + } + + // Turn lane data + { + /* TODO: remove this block, replace with io.hpp ref */ + boost::filesystem::ifstream lane_data_stream(config.turn_lane_data_path, std::ios::binary); + std::uint64_t lane_tupel_count = 0; + lane_data_stream.read(reinterpret_cast(&lane_tupel_count), + sizeof(lane_tupel_count)); + /* END TODO */ + + // make sure do write canary... + auto *turn_lane_data_ptr = layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::TURN_LANE_DATA); + if (layout_ptr->GetBlockSize(DataLayout::TURN_LANE_DATA) > 0) + { + lane_data_stream.read(reinterpret_cast(turn_lane_data_ptr), + layout_ptr->GetBlockSize(DataLayout::TURN_LANE_DATA)); + } + lane_data_stream.close(); + } + + // Turn lane descriptions + { + /* TODO: replace with io.hpp version */ + std::vector lane_description_offsets; + std::vector lane_description_masks; + if (!util::deserializeAdjacencyArray(config.turn_lane_description_path.string(), + lane_description_offsets, + lane_description_masks)) + throw util::exception("Failed to read lane descriptions from: " + + config.turn_lane_description_path.string()); + /* END TODO */ + auto *turn_lane_offset_ptr = layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::LANE_DESCRIPTION_OFFSETS); + if (!lane_description_offsets.empty()) + { + BOOST_ASSERT(layout_ptr->GetBlockSize(DataLayout::LANE_DESCRIPTION_OFFSETS) >= + sizeof(lane_description_offsets[0]) * lane_description_offsets.size()); + std::copy(lane_description_offsets.begin(), + lane_description_offsets.end(), + turn_lane_offset_ptr); + std::vector tmp; + lane_description_offsets.swap(tmp); + } + + auto *turn_lane_mask_ptr = + layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::LANE_DESCRIPTION_MASKS); + if (!lane_description_masks.empty()) + { + BOOST_ASSERT(layout_ptr->GetBlockSize(DataLayout::LANE_DESCRIPTION_MASKS) >= + sizeof(lane_description_masks[0]) * lane_description_masks.size()); + std::copy( + lane_description_masks.begin(), lane_description_masks.end(), turn_lane_mask_ptr); + std::vector tmp; + lane_description_masks.swap(tmp); + } + } + + // Load original edge data + { + /* TODO: replace with io.hpp */ + boost::filesystem::ifstream edges_input_stream(config.edges_data_path, std::ios::binary); + if (!edges_input_stream) + { + throw util::exception("Could not open " + config.edges_data_path.string() + + " for reading."); + } + + const auto number_of_original_edges = io::readElementCount(edges_input_stream); + /* END TODO */ + + GeometryID *via_geometry_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::VIA_NODE_LIST); + + unsigned *name_id_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::NAME_ID_LIST); + + extractor::TravelMode *travel_mode_ptr = + layout_ptr->GetBlockPtr(memory_ptr, + DataLayout::TRAVEL_MODE); + util::guidance::TurnBearing *pre_turn_bearing_ptr = + layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::PRE_TURN_BEARING); + util::guidance::TurnBearing *post_turn_bearing_ptr = + layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::POST_TURN_BEARING); + + LaneDataID *lane_data_id_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::LANE_DATA_ID); + + extractor::guidance::TurnInstruction *turn_instructions_ptr = + layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::TURN_INSTRUCTION); + + EntryClassID *entry_class_id_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::ENTRY_CLASSID); + + io::readEdges(edges_input_stream, + via_geometry_ptr, + name_id_ptr, + turn_instructions_ptr, + lane_data_id_ptr, + travel_mode_ptr, + entry_class_id_ptr, + pre_turn_bearing_ptr, + post_turn_bearing_ptr, + number_of_original_edges); + } + + // load compressed geometry + { + /* TODO: replace with io.hpp call */ + boost::filesystem::ifstream geometry_input_stream(config.geometries_path, std::ios::binary); + /* END TODO */ + + unsigned temporary_value; + unsigned *geometries_index_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::GEOMETRIES_INDEX); + geometry_input_stream.seekg(0, geometry_input_stream.beg); + geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned)); + BOOST_ASSERT(temporary_value == layout_ptr->num_entries[DataLayout::GEOMETRIES_INDEX]); + + if (layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_INDEX) > 0) + { + geometry_input_stream.read((char *)geometries_index_ptr, + layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_INDEX)); + } + NodeID *geometries_node_id_list_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::GEOMETRIES_NODE_LIST); + + geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned)); + BOOST_ASSERT(temporary_value == layout_ptr->num_entries[DataLayout::GEOMETRIES_NODE_LIST]); + + if (layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_NODE_LIST) > 0) + { + geometry_input_stream.read((char *)geometries_node_id_list_ptr, + layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_NODE_LIST)); + } + EdgeWeight *geometries_fwd_weight_list_ptr = layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::GEOMETRIES_FWD_WEIGHT_LIST); + + BOOST_ASSERT(temporary_value == + layout_ptr->num_entries[DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]); + + if (layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_FWD_WEIGHT_LIST) > 0) + { + geometry_input_stream.read( + (char *)geometries_fwd_weight_list_ptr, + layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_FWD_WEIGHT_LIST)); + } + EdgeWeight *geometries_rev_weight_list_ptr = layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::GEOMETRIES_REV_WEIGHT_LIST); + + BOOST_ASSERT(temporary_value == + layout_ptr->num_entries[DataLayout::GEOMETRIES_REV_WEIGHT_LIST]); + + if (layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_REV_WEIGHT_LIST) > 0) + { + geometry_input_stream.read( + (char *)geometries_rev_weight_list_ptr, + layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_REV_WEIGHT_LIST)); + } + } + + { + boost::filesystem::ifstream geometry_datasource_input_stream(config.datasource_indexes_path, + std::ios::binary); + if (!geometry_datasource_input_stream) + { + throw util::exception("Could not open " + config.datasource_indexes_path.string() + + " for reading."); + } + auto number_of_compressed_datasources = + io::readElementCount(geometry_datasource_input_stream); + + // load datasource information (if it exists) + auto datasources_list_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::DATASOURCES_LIST); + if (number_of_compressed_datasources > 0) + { + io::readDatasourceIndexes(geometry_datasource_input_stream, + datasources_list_ptr, + number_of_compressed_datasources); + } + } + + { + /* Load names */ + boost::filesystem::ifstream datasource_names_input_stream(config.datasource_names_path, + std::ios::binary); + if (!datasource_names_input_stream) + { + throw util::exception("Could not open " + config.datasource_names_path.string() + + " for reading."); + } + + const io::DatasourceNamesData datasource_names_data = + io::readDatasourceNames(datasource_names_input_stream); + + /* END TODO */ + + // load datasource name information (if it exists) + auto datasource_name_data_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::DATASOURCE_NAME_DATA); + if (layout_ptr->GetBlockSize(DataLayout::DATASOURCE_NAME_DATA) > 0) + { + std::copy(datasource_names_data.names.begin(), + datasource_names_data.names.end(), + datasource_name_data_ptr); + } + + auto datasource_name_offsets_ptr = layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::DATASOURCE_NAME_OFFSETS); + if (layout_ptr->GetBlockSize(DataLayout::DATASOURCE_NAME_OFFSETS) > 0) + { + std::copy(datasource_names_data.offsets.begin(), + datasource_names_data.offsets.end(), + datasource_name_offsets_ptr); + } + + auto datasource_name_lengths_ptr = layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::DATASOURCE_NAME_LENGTHS); + if (layout_ptr->GetBlockSize(DataLayout::DATASOURCE_NAME_LENGTHS) > 0) + { + std::copy(datasource_names_data.lengths.begin(), + datasource_names_data.lengths.end(), + datasource_name_lengths_ptr); + } + } + + // Loading list of coordinates + { + boost::filesystem::ifstream nodes_input_stream(config.nodes_data_path, std::ios::binary); + io::readElementCount(nodes_input_stream); + util::Coordinate *coordinates_ptr = layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::COORDINATE_LIST); + std::uint64_t *osmnodeid_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::OSM_NODE_ID_LIST); + util::PackedVector osmnodeid_list; + + osmnodeid_list.reset(osmnodeid_ptr, layout_ptr->num_entries[DataLayout::OSM_NODE_ID_LIST]); + + io::readNodes(nodes_input_stream, + coordinates_ptr, + osmnodeid_list, + layout_ptr->num_entries[DataLayout::COORDINATE_LIST]); + } + + // store timestamp + { + /* TODO: replace this */ + boost::filesystem::ifstream timestamp_stream(config.timestamp_path); + const auto timestamp_size = io::readNumberOfBytes(timestamp_stream); + /* END TODO */ + + char *timestamp_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::TIMESTAMP); + io::readTimestamp(timestamp_stream, timestamp_ptr, timestamp_size); + } + + // store search tree portion of rtree + { + boost::filesystem::ifstream tree_node_file(config.ram_index_path, std::ios::binary); + io::readElementCount(tree_node_file); + auto rtree_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::R_SEARCH_TREE); + io::readRamIndex( + tree_node_file, rtree_ptr, layout_ptr->num_entries[DataLayout::R_SEARCH_TREE]); + } + + { + /* TODO: replace with io.hpp call */ + boost::filesystem::ifstream core_marker_file(config.core_data_path, std::ios::binary); + if (!core_marker_file) + { + throw util::exception("Could not open " + config.core_data_path.string() + + " for reading."); + } + + std::uint32_t number_of_core_markers = 0; + core_marker_file.read((char *)&number_of_core_markers, sizeof(uint32_t)); + /* END TODO */ + + // load core markers + std::vector unpacked_core_markers(number_of_core_markers); + core_marker_file.read((char *)unpacked_core_markers.data(), + sizeof(char) * number_of_core_markers); + + unsigned *core_marker_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::CORE_MARKER); + + for (auto i = 0u; i < number_of_core_markers; ++i) + { + BOOST_ASSERT(unpacked_core_markers[i] == 0 || unpacked_core_markers[i] == 1); + + if (unpacked_core_markers[i] == 1) + { + const unsigned bucket = i / 32; + const unsigned offset = i % 32; + const unsigned value = [&] { + unsigned return_value = 0; + if (0 != offset) + { + return_value = core_marker_ptr[bucket]; + } + return return_value; + }(); + + core_marker_ptr[bucket] = (value | (1u << offset)); + } + } + } + + // load profile properties + { + boost::filesystem::ifstream profile_properties_stream(config.properties_path); + if (!profile_properties_stream) + { + util::exception("Could not open " + config.properties_path.string() + " for reading!"); + } + io::readPropertiesCount(); + auto profile_properties_ptr = layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::PROPERTIES); + io::readProperties(profile_properties_stream, + profile_properties_ptr, + sizeof(extractor::ProfileProperties)); + } + + // Load intersection data + { + boost::filesystem::ifstream intersection_stream(config.intersection_class_path, + std::ios::binary); + if (!static_cast(intersection_stream)) + throw util::exception("Could not open " + config.intersection_class_path.string() + + " for reading."); + + if (!util::readAndCheckFingerprint(intersection_stream)) + throw util::exception("Fingerprint of " + config.intersection_class_path.string() + + " does not match or could not read from file"); + + std::vector bearing_class_id_table; + if (!util::deserializeVector(intersection_stream, bearing_class_id_table)) + throw util::exception("Failed to bearing class ids read from " + + config.names_data_path.string()); + + unsigned bearing_blocks = 0; + intersection_stream.read((char *)&bearing_blocks, sizeof(unsigned)); + unsigned sum_lengths = 0; + intersection_stream.read((char *)&sum_lengths, sizeof(unsigned)); + + std::vector bearing_offsets_data(bearing_blocks); + std::vector::BlockT> bearing_blocks_data( + bearing_blocks); + + if (bearing_blocks) + { + intersection_stream.read(reinterpret_cast(&bearing_offsets_data[0]), + bearing_blocks * sizeof(bearing_offsets_data[0])); + } + + if (bearing_blocks) + { + intersection_stream.read(reinterpret_cast(&bearing_blocks_data[0]), + bearing_blocks * sizeof(bearing_blocks_data[0])); + } + + std::uint64_t num_bearings; + intersection_stream.read(reinterpret_cast(&num_bearings), sizeof(num_bearings)); + + std::vector bearing_class_table(num_bearings); + intersection_stream.read(reinterpret_cast(&bearing_class_table[0]), + sizeof(bearing_class_table[0]) * num_bearings); + + std::vector entry_class_table; + if (!util::deserializeVector(intersection_stream, entry_class_table)) + throw util::exception("Failed to read entry classes from " + + config.intersection_class_path.string()); + + /* END TODO */ + + // load intersection classes + if (!bearing_class_id_table.empty()) + { + auto bearing_id_ptr = layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::BEARING_CLASSID); + std::copy(bearing_class_id_table.begin(), bearing_class_id_table.end(), bearing_id_ptr); + } + + if (layout_ptr->GetBlockSize(DataLayout::BEARING_OFFSETS) > 0) + { + auto *bearing_offsets_ptr = + layout_ptr->GetBlockPtr(memory_ptr, DataLayout::BEARING_OFFSETS); + std::copy( + bearing_offsets_data.begin(), bearing_offsets_data.end(), bearing_offsets_ptr); + } + + if (layout_ptr->GetBlockSize(DataLayout::BEARING_BLOCKS) > 0) + { + auto *bearing_blocks_ptr = + layout_ptr->GetBlockPtr::BlockT, true>( + memory_ptr, DataLayout::BEARING_BLOCKS); + std::copy(bearing_blocks_data.begin(), bearing_blocks_data.end(), bearing_blocks_ptr); + } + + if (!bearing_class_table.empty()) + { + auto bearing_class_ptr = layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::BEARING_VALUES); + std::copy(bearing_class_table.begin(), bearing_class_table.end(), bearing_class_ptr); + } + + if (!entry_class_table.empty()) + { + auto entry_class_ptr = layout_ptr->GetBlockPtr( + memory_ptr, DataLayout::ENTRY_CLASS); + std::copy(entry_class_table.begin(), entry_class_table.end(), entry_class_ptr); + } + } +} } }