refactor function names; consolidate readCount() functions;
remove templated types as much as possible for type safety; add more comments; clean up code, add const if possible;
This commit is contained in:
Huyen Chau Nguyen
parent
fe94977c9b
commit
cf35bbeb50
@ -123,8 +123,8 @@ class InternalDataFacade final : public BaseDataFacade
|
||||
{
|
||||
throw util::exception("Could not open " + properties_path.string() + " for reading.");
|
||||
}
|
||||
auto PropertiesSize = storage::io::readPropertiesSize();
|
||||
storage::io::readProperties(in_stream, &m_profile_properties, PropertiesSize);
|
||||
const auto properties_size = storage::io::readPropertiesCount();
|
||||
storage::io::readProperties(in_stream, &m_profile_properties, properties_size);
|
||||
}
|
||||
|
||||
void LoadLaneTupleIdPairs(const boost::filesystem::path &lane_data_path)
|
||||
@ -151,7 +151,7 @@ class InternalDataFacade final : public BaseDataFacade
|
||||
throw util::exception("Could not open " + timestamp_path.string() + " for reading.");
|
||||
}
|
||||
|
||||
auto timestamp_size = storage::io::readTimestampSize(timestamp_stream);
|
||||
const auto timestamp_size = storage::io::readNumberOfBytes(timestamp_stream);
|
||||
m_timestamp.resize(timestamp_size);
|
||||
storage::io::readTimestamp(timestamp_stream, &m_timestamp.front(), timestamp_size);
|
||||
}
|
||||
@ -164,7 +164,7 @@ class InternalDataFacade final : public BaseDataFacade
|
||||
throw util::exception("Could not open " + hsgr_path.string() + " for reading.");
|
||||
}
|
||||
|
||||
auto header = storage::io::readHSGRHeader(hsgr_input_stream);
|
||||
const auto header = storage::io::readHSGRHeader(hsgr_input_stream);
|
||||
m_check_sum = header.checksum;
|
||||
|
||||
util::ShM<QueryGraph::NodeArrayEntry, false>::vector node_list(header.number_of_nodes);
|
||||
@ -190,10 +190,10 @@ class InternalDataFacade final : public BaseDataFacade
|
||||
throw util::exception("Could not open " + nodes_file_path.string() + " for reading.");
|
||||
}
|
||||
|
||||
std::uint32_t number_of_coordinates = storage::io::readNodesSize(nodes_input_stream);
|
||||
const auto number_of_coordinates = storage::io::readElementCount(nodes_input_stream);
|
||||
m_coordinate_list.resize(number_of_coordinates);
|
||||
m_osmnodeid_list.reserve(number_of_coordinates);
|
||||
storage::io::readNodesData(
|
||||
storage::io::readNodes(
|
||||
nodes_input_stream, m_coordinate_list.data(), m_osmnodeid_list, number_of_coordinates);
|
||||
|
||||
boost::filesystem::ifstream edges_input_stream(edges_file_path, std::ios::binary);
|
||||
@ -201,7 +201,7 @@ class InternalDataFacade final : public BaseDataFacade
|
||||
{
|
||||
throw util::exception("Could not open " + edges_file_path.string() + " for reading.");
|
||||
}
|
||||
auto number_of_edges = storage::io::readEdgesSize(edges_input_stream);
|
||||
const auto number_of_edges = storage::io::readElementCount(edges_input_stream);
|
||||
m_via_geometry_list.resize(number_of_edges);
|
||||
m_name_ID_list.resize(number_of_edges);
|
||||
m_turn_instruction_list.resize(number_of_edges);
|
||||
@ -211,16 +211,16 @@ class InternalDataFacade final : public BaseDataFacade
|
||||
m_pre_turn_bearing.resize(number_of_edges);
|
||||
m_post_turn_bearing.resize(number_of_edges);
|
||||
|
||||
storage::io::readEdgesData(edges_input_stream,
|
||||
m_via_geometry_list.data(),
|
||||
m_name_ID_list.data(),
|
||||
m_turn_instruction_list.data(),
|
||||
m_lane_data_id.data(),
|
||||
m_travel_mode_list.data(),
|
||||
m_entry_class_id_list.data(),
|
||||
m_pre_turn_bearing.data(),
|
||||
m_post_turn_bearing.data(),
|
||||
number_of_edges);
|
||||
storage::io::readEdges(edges_input_stream,
|
||||
m_via_geometry_list.data(),
|
||||
m_name_ID_list.data(),
|
||||
m_turn_instruction_list.data(),
|
||||
m_lane_data_id.data(),
|
||||
m_travel_mode_list.data(),
|
||||
m_entry_class_id_list.data(),
|
||||
m_pre_turn_bearing.data(),
|
||||
m_post_turn_bearing.data(),
|
||||
number_of_edges);
|
||||
}
|
||||
|
||||
void LoadCoreInformation(const boost::filesystem::path &core_data_file)
|
||||
@ -292,7 +292,7 @@ class InternalDataFacade final : public BaseDataFacade
|
||||
}
|
||||
BOOST_ASSERT(datasources_stream);
|
||||
|
||||
auto number_of_datasources = storage::io::readDatasourceIndexesSize(datasources_stream);
|
||||
const auto number_of_datasources = storage::io::readElementCount(datasources_stream);
|
||||
if (number_of_datasources > 0)
|
||||
{
|
||||
m_datasource_list.resize(number_of_datasources);
|
||||
@ -308,15 +308,15 @@ class InternalDataFacade final : public BaseDataFacade
|
||||
}
|
||||
BOOST_ASSERT(datasourcenames_stream);
|
||||
|
||||
auto datasource_names_data = storage::io::readDatasourceNamesData(datasourcenames_stream);
|
||||
const auto datasource_names_data = storage::io::readDatasourceNames(datasourcenames_stream);
|
||||
m_datasource_names.resize(datasource_names_data.lengths.size());
|
||||
for (std::uint32_t i = 0; i < datasource_names_data.lengths.size(); ++i)
|
||||
for (std::size_t i = 0; i < datasource_names_data.lengths.size(); ++i)
|
||||
{
|
||||
auto name_begin =
|
||||
datasource_names_data.names.begin() + datasource_names_data.offsets[i];
|
||||
auto name_end = datasource_names_data.names.begin() + datasource_names_data.offsets[i] +
|
||||
datasource_names_data.lengths[i];
|
||||
m_datasource_names[i] = std::move(std::string(name_begin, name_end));
|
||||
m_datasource_names[i] = std::string(name_begin, name_end);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -17,7 +17,7 @@ namespace extractor
|
||||
struct OriginalEdgeData
|
||||
{
|
||||
explicit OriginalEdgeData(GeometryID via_geometry,
|
||||
unsigned name_id,
|
||||
NameID name_id,
|
||||
LaneDataID lane_data_id,
|
||||
guidance::TurnInstruction turn_instruction,
|
||||
EntryClassID entry_classid,
|
||||
|
||||
@ -1,10 +1,13 @@
|
||||
#ifndef OSRM_STORAGE_IO_HPP_
|
||||
#define OSRM_STORAGE_IO_HPP_
|
||||
|
||||
#include "contractor/query_edge.hpp"
|
||||
#include "extractor/extractor.hpp"
|
||||
#include "extractor/original_edge_data.hpp"
|
||||
#include "extractor/query_node.hpp"
|
||||
#include "util/fingerprint.hpp"
|
||||
#include "util/simple_logger.hpp"
|
||||
#include "util/static_graph.hpp"
|
||||
|
||||
#include <boost/filesystem/fstream.hpp>
|
||||
|
||||
@ -17,27 +20,38 @@ namespace storage
|
||||
namespace io
|
||||
{
|
||||
|
||||
inline std::size_t readPropertiesSize() { return 1; }
|
||||
|
||||
template <typename PropertiesT>
|
||||
inline void readProperties(boost::filesystem::ifstream &properties_stream,
|
||||
PropertiesT *properties,
|
||||
std::size_t properties_size)
|
||||
// Reads the count of elements that is written in the file header and returns the number
|
||||
inline std::uint64_t readElementCount(boost::filesystem::ifstream &input_stream)
|
||||
{
|
||||
properties_stream.read(reinterpret_cast<char *>(properties), properties_size);
|
||||
std::uint64_t number_of_elements = 0;
|
||||
input_stream.read((char *)&number_of_elements, sizeof(std::uint64_t));
|
||||
return number_of_elements;
|
||||
}
|
||||
|
||||
// To make function calls consistent, this function returns the fixed number of properties
|
||||
inline std::size_t readPropertiesCount() { return 1; }
|
||||
|
||||
// Returns the number of bytes in a file
|
||||
inline std::size_t readNumberOfBytes(boost::filesystem::ifstream &input_stream)
|
||||
{
|
||||
input_stream.seekg(0, input_stream.end);
|
||||
auto length = input_stream.tellg();
|
||||
input_stream.seekg(0, input_stream.beg);
|
||||
return length;
|
||||
}
|
||||
|
||||
#pragma pack(push, 1)
|
||||
struct HSGRHeader
|
||||
{
|
||||
std::uint32_t checksum;
|
||||
std::uint32_t number_of_nodes;
|
||||
std::uint32_t number_of_edges;
|
||||
std::uint64_t number_of_nodes;
|
||||
std::uint64_t number_of_edges;
|
||||
};
|
||||
#pragma pack(pop)
|
||||
static_assert(sizeof(HSGRHeader) == 12, "HSGRHeader is not packed");
|
||||
static_assert(sizeof(HSGRHeader) == 20, "HSGRHeader is not packed");
|
||||
|
||||
// Returns the checksum and the number of nodes and number of edges
|
||||
// Reads the checksum, number of nodes and number of edges written in the header file of a `.hsgr`
|
||||
// file and returns them in a HSGRHeader struct
|
||||
inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream)
|
||||
{
|
||||
const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid();
|
||||
@ -62,66 +76,65 @@ inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream)
|
||||
return header;
|
||||
}
|
||||
|
||||
// Reads the graph data of a `.hsgr` file into memory
|
||||
// Needs to be called after readHSGRHeader() to get the correct offset in the stream
|
||||
template <typename NodeT, typename EdgeT>
|
||||
void readHSGR(boost::filesystem::ifstream &input_stream,
|
||||
NodeT node_buffer[],
|
||||
std::uint32_t number_of_nodes,
|
||||
EdgeT edge_buffer[],
|
||||
std::uint32_t number_of_edges)
|
||||
using NodeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::NodeArrayEntry;
|
||||
using EdgeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::EdgeArrayEntry;
|
||||
inline void readHSGR(boost::filesystem::ifstream &input_stream,
|
||||
NodeT *node_buffer,
|
||||
const std::uint64_t number_of_nodes,
|
||||
EdgeT *edge_buffer,
|
||||
const std::uint64_t number_of_edges)
|
||||
{
|
||||
BOOST_ASSERT(node_buffer);
|
||||
BOOST_ASSERT(edge_buffer);
|
||||
input_stream.read(reinterpret_cast<char *>(node_buffer), number_of_nodes * sizeof(NodeT));
|
||||
input_stream.read(reinterpret_cast<char *>(edge_buffer), number_of_edges * sizeof(EdgeT));
|
||||
}
|
||||
// Returns the size of the timestamp in a file
|
||||
inline std::uint32_t readTimestampSize(boost::filesystem::ifstream ×tamp_input_stream)
|
||||
|
||||
// Loads properties from a `.properties` file into memory
|
||||
inline void readProperties(boost::filesystem::ifstream &properties_stream,
|
||||
extractor::ProfileProperties *properties,
|
||||
const std::size_t properties_size)
|
||||
{
|
||||
timestamp_input_stream.seekg(0, timestamp_input_stream.end);
|
||||
auto length = timestamp_input_stream.tellg();
|
||||
timestamp_input_stream.seekg(0, timestamp_input_stream.beg);
|
||||
return length;
|
||||
BOOST_ASSERT(properties);
|
||||
properties_stream.read(reinterpret_cast<char *>(properties), properties_size);
|
||||
}
|
||||
|
||||
// Reads the timestamp in a file
|
||||
// Reads the timestamp in a `.timestamp` file
|
||||
// Use readNumberOfBytes() beforehand to get the length of the file
|
||||
inline void readTimestamp(boost::filesystem::ifstream ×tamp_input_stream,
|
||||
char timestamp[],
|
||||
std::size_t timestamp_length)
|
||||
char *timestamp,
|
||||
const std::size_t timestamp_length)
|
||||
{
|
||||
BOOST_ASSERT(timestamp);
|
||||
timestamp_input_stream.read(timestamp, timestamp_length * sizeof(char));
|
||||
}
|
||||
|
||||
// Returns the number of edges in a .edges file
|
||||
inline std::uint32_t readEdgesSize(boost::filesystem::ifstream &edges_input_stream)
|
||||
// Loads datasource_indexes from .datasource_indexes into memory
|
||||
// Needs to be called after readElementCount() to get the correct offset in the stream
|
||||
inline void readDatasourceIndexes(boost::filesystem::ifstream &datasource_indexes_input_stream,
|
||||
uint8_t *datasource_buffer,
|
||||
const std::uint64_t number_of_datasource_indexes)
|
||||
{
|
||||
std::uint32_t number_of_edges;
|
||||
edges_input_stream.read((char *)&number_of_edges, sizeof(std::uint32_t));
|
||||
return number_of_edges;
|
||||
BOOST_ASSERT(datasource_buffer);
|
||||
datasource_indexes_input_stream.read(reinterpret_cast<char *>(datasource_buffer),
|
||||
number_of_datasource_indexes * sizeof(std::uint8_t));
|
||||
}
|
||||
|
||||
// Reads edge data from .edge files which includes its
|
||||
// Loads edge data from .edge files into memory which includes its
|
||||
// geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID
|
||||
// Needs to be called after readEdgesSize() to get the correct offset in the stream
|
||||
template <typename GeometryIDT,
|
||||
typename NameIDT,
|
||||
typename TurnInstructionT,
|
||||
typename LaneDataIDT,
|
||||
typename TravelModeT,
|
||||
typename EntryClassIDT,
|
||||
typename PreTurnBearingT,
|
||||
typename PostTurnBearingT>
|
||||
void readEdgesData(boost::filesystem::ifstream &edges_input_stream,
|
||||
GeometryIDT geometry_list[],
|
||||
NameIDT name_id_list[],
|
||||
TurnInstructionT turn_instruction_list[],
|
||||
LaneDataIDT lane_data_id_list[],
|
||||
TravelModeT travel_mode_list[],
|
||||
EntryClassIDT entry_class_id_list[],
|
||||
PreTurnBearingT pre_turn_bearing_list[],
|
||||
PostTurnBearingT post_turn_bearing_list[],
|
||||
std::uint32_t number_of_edges)
|
||||
// Needs to be called after readElementCount() to get the correct offset in the stream
|
||||
inline void readEdges(boost::filesystem::ifstream &edges_input_stream,
|
||||
GeometryID *geometry_list,
|
||||
NameID *name_id_list,
|
||||
extractor::guidance::TurnInstruction *turn_instruction_list,
|
||||
LaneDataID *lane_data_id_list,
|
||||
extractor::TravelMode *travel_mode_list,
|
||||
EntryClassID *entry_class_id_list,
|
||||
util::guidance::TurnBearing *pre_turn_bearing_list,
|
||||
util::guidance::TurnBearing *post_turn_bearing_list,
|
||||
const std::uint64_t number_of_edges)
|
||||
{
|
||||
BOOST_ASSERT(geometry_list);
|
||||
BOOST_ASSERT(name_id_list);
|
||||
@ -130,9 +143,10 @@ void readEdgesData(boost::filesystem::ifstream &edges_input_stream,
|
||||
BOOST_ASSERT(travel_mode_list);
|
||||
BOOST_ASSERT(entry_class_id_list);
|
||||
extractor::OriginalEdgeData current_edge_data;
|
||||
for (std::uint32_t i = 0; i < number_of_edges; ++i)
|
||||
for (std::uint64_t i = 0; i < number_of_edges; ++i)
|
||||
{
|
||||
edges_input_stream.read((char *)&(current_edge_data), sizeof(extractor::OriginalEdgeData));
|
||||
|
||||
geometry_list[i] = current_edge_data.via_geometry;
|
||||
name_id_list[i] = current_edge_data.name_id;
|
||||
turn_instruction_list[i] = current_edge_data.turn_instruction;
|
||||
@ -144,96 +158,62 @@ void readEdgesData(boost::filesystem::ifstream &edges_input_stream,
|
||||
}
|
||||
}
|
||||
|
||||
// Returns the number of nodes in a .nodes file
|
||||
inline std::uint32_t readNodesSize(boost::filesystem::ifstream &nodes_input_stream)
|
||||
{
|
||||
std::uint32_t number_of_coordinates;
|
||||
nodes_input_stream.read((char *)&number_of_coordinates, sizeof(std::uint32_t));
|
||||
return number_of_coordinates;
|
||||
}
|
||||
|
||||
// Reads coordinates and OSM node IDs from .nodes files
|
||||
// Needs to be called after readNodesSize() to get the correct offset in the stream
|
||||
template <typename CoordinateT, typename OSMNodeIDVectorT>
|
||||
void readNodesData(boost::filesystem::ifstream &nodes_input_stream,
|
||||
CoordinateT coordinate_list[],
|
||||
OSMNodeIDVectorT &osmnodeid_list,
|
||||
std::uint32_t number_of_coordinates)
|
||||
// Loads coordinates and OSM node IDs from .nodes files into memory
|
||||
// Needs to be called after readElementCount() to get the correct offset in the stream
|
||||
template <typename OSMNodeIDVectorT>
|
||||
void readNodes(boost::filesystem::ifstream &nodes_input_stream,
|
||||
util::Coordinate *coordinate_list,
|
||||
OSMNodeIDVectorT &osmnodeid_list,
|
||||
const std::uint64_t number_of_coordinates)
|
||||
{
|
||||
BOOST_ASSERT(coordinate_list);
|
||||
extractor::QueryNode current_node;
|
||||
for (std::uint32_t i = 0; i < number_of_coordinates; ++i)
|
||||
for (std::uint64_t i = 0; i < number_of_coordinates; ++i)
|
||||
{
|
||||
nodes_input_stream.read((char *)¤t_node, sizeof(extractor::QueryNode));
|
||||
coordinate_list[i] = CoordinateT(current_node.lon, current_node.lat);
|
||||
coordinate_list[i] = util::Coordinate(current_node.lon, current_node.lat);
|
||||
osmnodeid_list.push_back(current_node.node_id);
|
||||
BOOST_ASSERT(coordinate_list[i].IsValid());
|
||||
}
|
||||
}
|
||||
|
||||
// Returns the number of indexes in a .datasource_indexes file
|
||||
inline std::uint64_t
|
||||
readDatasourceIndexesSize(boost::filesystem::ifstream &datasource_indexes_input_stream)
|
||||
{
|
||||
std::uint64_t number_of_datasource_indexes;
|
||||
datasource_indexes_input_stream.read(reinterpret_cast<char *>(&number_of_datasource_indexes),
|
||||
sizeof(std::uint64_t));
|
||||
return number_of_datasource_indexes;
|
||||
}
|
||||
|
||||
// Reads datasource_indexes
|
||||
// Needs to be called after readDatasourceIndexesSize() to get the correct offset in the stream
|
||||
inline void readDatasourceIndexes(boost::filesystem::ifstream &datasource_indexes_input_stream,
|
||||
uint8_t datasource_buffer[],
|
||||
std::uint32_t number_of_datasource_indexes)
|
||||
{
|
||||
BOOST_ASSERT(datasource_buffer);
|
||||
datasource_indexes_input_stream.read(reinterpret_cast<char *>(datasource_buffer),
|
||||
number_of_datasource_indexes * sizeof(std::uint8_t));
|
||||
}
|
||||
|
||||
// Reads datasource names out of .datasource_names files and metadata such as
|
||||
// the length and offset of each name
|
||||
struct DatasourceNamesData
|
||||
{
|
||||
std::vector<char> names;
|
||||
std::vector<std::uint32_t> offsets;
|
||||
std::vector<std::uint32_t> lengths;
|
||||
std::vector<std::size_t> offsets;
|
||||
std::vector<std::size_t> lengths;
|
||||
};
|
||||
inline DatasourceNamesData
|
||||
readDatasourceNamesData(boost::filesystem::ifstream &datasource_names_input_stream)
|
||||
inline DatasourceNamesData readDatasourceNames(boost::filesystem::ifstream &datasource_names_input_stream)
|
||||
{
|
||||
DatasourceNamesData datasource_names_data;
|
||||
if (datasource_names_input_stream)
|
||||
std::string name;
|
||||
while (std::getline(datasource_names_input_stream, name))
|
||||
{
|
||||
std::string name;
|
||||
while (std::getline(datasource_names_input_stream, name))
|
||||
{
|
||||
datasource_names_data.offsets.push_back(datasource_names_data.names.size());
|
||||
datasource_names_data.lengths.push_back(name.size());
|
||||
std::copy(name.c_str(),
|
||||
name.c_str() + name.size(),
|
||||
std::back_inserter(datasource_names_data.names));
|
||||
}
|
||||
datasource_names_data.offsets.push_back(datasource_names_data.names.size());
|
||||
datasource_names_data.lengths.push_back(name.size());
|
||||
std::copy(name.c_str(),
|
||||
name.c_str() + name.size(),
|
||||
std::back_inserter(datasource_names_data.names));
|
||||
}
|
||||
return datasource_names_data;
|
||||
}
|
||||
|
||||
// Returns the number of ram indexes
|
||||
inline std::uint32_t readRamIndexSize(boost::filesystem::ifstream &ram_index_input_stream)
|
||||
{
|
||||
std::uint32_t tree_size = 0;
|
||||
ram_index_input_stream.read((char *)&tree_size, sizeof(std::uint32_t));
|
||||
return tree_size;
|
||||
}
|
||||
|
||||
// Loads ram indexes of R-Trees from `.ramIndex` files into memory
|
||||
// Needs to be called after readElementCount() to get the correct offset in the stream
|
||||
// template <bool UseSharedMemory>
|
||||
// NB Cannot be written without templated type because of cyclic depencies between
|
||||
// `static_rtree.hpp` and `io.hpp`
|
||||
template <typename RTreeNodeT>
|
||||
void readRamIndexData(boost::filesystem::ifstream &ram_index_input_stream,
|
||||
RTreeNodeT rtree_buffer[],
|
||||
std::uint32_t tree_size)
|
||||
void readRamIndex(boost::filesystem::ifstream &ram_index_input_stream,
|
||||
RTreeNodeT *rtree_buffer,
|
||||
const std::uint64_t tree_size)
|
||||
{
|
||||
BOOST_ASSERT(rtree_buffer);
|
||||
ram_index_input_stream.read(reinterpret_cast<char *>(rtree_buffer), sizeof(RTreeNodeT) * tree_size);
|
||||
ram_index_input_stream.read(reinterpret_cast<char *>(rtree_buffer),
|
||||
sizeof(RTreeNodeT) * tree_size);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -333,9 +333,9 @@ class StaticRTree
|
||||
// open tree file
|
||||
boost::filesystem::ofstream tree_node_file(tree_node_filename, std::ios::binary);
|
||||
|
||||
std::uint32_t size_of_tree = m_search_tree.size();
|
||||
std::uint64_t size_of_tree = m_search_tree.size();
|
||||
BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty");
|
||||
tree_node_file.write((char *)&size_of_tree, sizeof(std::uint32_t));
|
||||
tree_node_file.write((char *)&size_of_tree, sizeof(size_of_tree));
|
||||
tree_node_file.write((char *)&m_search_tree[0], sizeof(TreeNode) * size_of_tree);
|
||||
|
||||
MapLeafNodesFile(leaf_node_filename);
|
||||
@ -357,11 +357,10 @@ class StaticRTree
|
||||
}
|
||||
boost::filesystem::ifstream tree_node_file(node_file, std::ios::binary);
|
||||
|
||||
|
||||
std::uint32_t tree_size = storage::io::readRamIndexSize(tree_node_file);
|
||||
const auto tree_size = storage::io::readElementCount(tree_node_file);
|
||||
|
||||
m_search_tree.resize(tree_size);
|
||||
storage::io::readRamIndexData(tree_node_file, &m_search_tree[0], tree_size);
|
||||
storage::io::readRamIndex(tree_node_file, &m_search_tree[0], tree_size);
|
||||
|
||||
MapLeafNodesFile(leaf_file);
|
||||
}
|
||||
|
||||
@ -420,8 +420,8 @@ parse_turn_penalty_lookup_from_csv_files(const std::vector<std::string> &turn_pe
|
||||
local.push_back(std::move(val));
|
||||
}
|
||||
|
||||
util::SimpleLogger().Write() << "Loaded penalty file " << filename << " with " << local.size()
|
||||
<< " turn penalties";
|
||||
util::SimpleLogger().Write() << "Loaded penalty file " << filename << " with "
|
||||
<< local.size() << " turn penalties";
|
||||
|
||||
{
|
||||
Mutex::scoped_lock _{flatten_mutex};
|
||||
@ -437,8 +437,10 @@ parse_turn_penalty_lookup_from_csv_files(const std::vector<std::string> &turn_pe
|
||||
// With flattened map-ish view of all the files, sort and unique them on from,to,source
|
||||
// The greater '>' is used here since we want to give files later on higher precedence
|
||||
const auto sort_by = [](const TurnPenaltySource &lhs, const TurnPenaltySource &rhs) {
|
||||
return std::tie(lhs.segment.from, lhs.segment.via, lhs.segment.to, lhs.penalty_source.source) >
|
||||
std::tie(rhs.segment.from, rhs.segment.via, rhs.segment.to, rhs.penalty_source.source);
|
||||
return std::tie(
|
||||
lhs.segment.from, lhs.segment.via, lhs.segment.to, lhs.penalty_source.source) >
|
||||
std::tie(
|
||||
rhs.segment.from, rhs.segment.via, rhs.segment.to, rhs.penalty_source.source);
|
||||
};
|
||||
|
||||
std::stable_sort(begin(map), end(map), sort_by);
|
||||
@ -569,8 +571,8 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
|
||||
throw util::exception("Failed to open " + nodes_filename);
|
||||
}
|
||||
|
||||
unsigned number_of_nodes = 0;
|
||||
nodes_input_stream.read((char *)&number_of_nodes, sizeof(unsigned));
|
||||
std::uint64_t number_of_nodes = 0;
|
||||
nodes_input_stream.read((char *)&number_of_nodes, sizeof(std::uint64_t));
|
||||
internal_to_external_node_map.resize(number_of_nodes);
|
||||
|
||||
// Load all the query nodes into a vector
|
||||
@ -976,7 +978,7 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
|
||||
{
|
||||
// Sorting contracted edges in a way that the static query graph can read some in in-place.
|
||||
tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end());
|
||||
const unsigned contracted_edge_count = contracted_edge_list.size();
|
||||
const std::uint64_t contracted_edge_count = contracted_edge_list.size();
|
||||
util::SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count
|
||||
<< " edges";
|
||||
|
||||
@ -1033,13 +1035,13 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
|
||||
const unsigned edges_crc32 = crc32_calculator(contracted_edge_list);
|
||||
util::SimpleLogger().Write() << "Writing CRC32: " << edges_crc32;
|
||||
|
||||
const unsigned node_array_size = node_array.size();
|
||||
const std::uint64_t node_array_size = node_array.size();
|
||||
// serialize crc32, aka checksum
|
||||
hsgr_output_stream.write((char *)&edges_crc32, sizeof(unsigned));
|
||||
// serialize number of nodes
|
||||
hsgr_output_stream.write((char *)&node_array_size, sizeof(unsigned));
|
||||
hsgr_output_stream.write((char *)&node_array_size, sizeof(std::uint64_t));
|
||||
// serialize number of edges
|
||||
hsgr_output_stream.write((char *)&contracted_edge_count, sizeof(unsigned));
|
||||
hsgr_output_stream.write((char *)&contracted_edge_count, sizeof(std::uint64_t));
|
||||
// serialize all nodes
|
||||
if (node_array_size > 0)
|
||||
{
|
||||
|
||||
@ -331,7 +331,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
|
||||
}
|
||||
|
||||
// Writes a dummy value at the front that is updated later with the total length
|
||||
const unsigned length_prefix_empty_space{0};
|
||||
const std::uint64_t length_prefix_empty_space{0};
|
||||
edge_data_file.write(reinterpret_cast<const char *>(&length_prefix_empty_space),
|
||||
sizeof(length_prefix_empty_space));
|
||||
|
||||
@ -598,7 +598,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
|
||||
// Finally jump back to the empty space at the beginning and write length prefix
|
||||
edge_data_file.seekp(std::ios::beg);
|
||||
|
||||
const auto length_prefix = boost::numeric_cast<unsigned>(original_edges_counter);
|
||||
const auto length_prefix = boost::numeric_cast<std::uint64_t>(original_edges_counter);
|
||||
static_assert(sizeof(length_prefix_empty_space) == sizeof(length_prefix), "type mismatch");
|
||||
|
||||
edge_data_file.write(reinterpret_cast<const char *>(&length_prefix), sizeof(length_prefix));
|
||||
|
||||
@ -112,8 +112,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
|
||||
TIMER_START(extracting);
|
||||
|
||||
const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
|
||||
const auto number_of_threads =
|
||||
std::min(recommended_num_threads, config.requested_num_threads);
|
||||
const auto number_of_threads = std::min(recommended_num_threads, config.requested_num_threads);
|
||||
tbb::task_scheduler_init init(number_of_threads);
|
||||
|
||||
{
|
||||
@ -527,8 +526,8 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
|
||||
void Extractor::WriteNodeMapping(const std::vector<QueryNode> &internal_to_external_node_map)
|
||||
{
|
||||
boost::filesystem::ofstream node_stream(config.node_output_path, std::ios::binary);
|
||||
const unsigned size_of_mapping = internal_to_external_node_map.size();
|
||||
node_stream.write((char *)&size_of_mapping, sizeof(unsigned));
|
||||
const std::uint64_t size_of_mapping = internal_to_external_node_map.size();
|
||||
node_stream.write((char *)&size_of_mapping, sizeof(std::uint64_t));
|
||||
if (size_of_mapping > 0)
|
||||
{
|
||||
node_stream.write((char *)internal_to_external_node_map.data(),
|
||||
|
||||
@ -230,7 +230,7 @@ Storage::ReturnCode Storage::Run(int max_wait)
|
||||
throw util::exception("Could not open " + config.edges_data_path.string() +
|
||||
" for reading.");
|
||||
}
|
||||
auto number_of_original_edges = io::readEdgesSize(edges_input_stream);
|
||||
const auto number_of_original_edges = io::readElementCount(edges_input_stream);
|
||||
|
||||
// note: settings this all to the same size is correct, we extract them from the same struct
|
||||
shared_layout_ptr->SetBlockSize<NodeID>(SharedDataLayout::VIA_NODE_LIST,
|
||||
@ -256,7 +256,7 @@ Storage::ReturnCode Storage::Run(int max_wait)
|
||||
throw util::exception("Could not open " + config.hsgr_data_path.string() + " for reading.");
|
||||
}
|
||||
|
||||
auto hsgr_header = io::readHSGRHeader(hsgr_input_stream);
|
||||
const auto hsgr_header = io::readHSGRHeader(hsgr_input_stream);
|
||||
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::HSGR_CHECKSUM, 1);
|
||||
shared_layout_ptr->SetBlockSize<QueryGraph::NodeArrayEntry>(SharedDataLayout::GRAPH_NODE_LIST,
|
||||
hsgr_header.number_of_nodes);
|
||||
@ -266,13 +266,13 @@ Storage::ReturnCode Storage::Run(int max_wait)
|
||||
// load rsearch tree size
|
||||
boost::filesystem::ifstream tree_node_file(config.ram_index_path, std::ios::binary);
|
||||
|
||||
std::uint32_t tree_size = io::readRamIndexSize(tree_node_file);
|
||||
const auto tree_size = io::readElementCount(tree_node_file);
|
||||
shared_layout_ptr->SetBlockSize<RTreeNode>(SharedDataLayout::R_SEARCH_TREE, tree_size);
|
||||
|
||||
// allocate space in shared memory for profile properties
|
||||
std::size_t PropertiesSize = io::readPropertiesSize();
|
||||
const auto properties_size = io::readPropertiesCount();
|
||||
shared_layout_ptr->SetBlockSize<extractor::ProfileProperties>(SharedDataLayout::PROPERTIES,
|
||||
PropertiesSize);
|
||||
properties_size);
|
||||
|
||||
// read timestampsize
|
||||
boost::filesystem::ifstream timestamp_stream(config.timestamp_path);
|
||||
@ -280,7 +280,7 @@ Storage::ReturnCode Storage::Run(int max_wait)
|
||||
{
|
||||
throw util::exception("Could not open " + config.timestamp_path.string() + " for reading.");
|
||||
}
|
||||
std::size_t timestamp_size = io::readTimestampSize(timestamp_stream);
|
||||
const auto timestamp_size = io::readNumberOfBytes(timestamp_stream);
|
||||
shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::TIMESTAMP, timestamp_size);
|
||||
|
||||
// load core marker size
|
||||
@ -301,7 +301,7 @@ Storage::ReturnCode Storage::Run(int max_wait)
|
||||
{
|
||||
throw util::exception("Could not open " + config.core_data_path.string() + " for reading.");
|
||||
}
|
||||
auto coordinate_list_size = io::readNodesSize(nodes_input_stream);
|
||||
const auto coordinate_list_size = io::readElementCount(nodes_input_stream);
|
||||
shared_layout_ptr->SetBlockSize<util::Coordinate>(SharedDataLayout::COORDINATE_LIST,
|
||||
coordinate_list_size);
|
||||
// we'll read a list of OSM node IDs from the same data, so set the block size for the same
|
||||
@ -342,12 +342,8 @@ Storage::ReturnCode Storage::Run(int max_wait)
|
||||
throw util::exception("Could not open " + config.datasource_indexes_path.string() +
|
||||
" for reading.");
|
||||
}
|
||||
std::uint64_t number_of_compressed_datasources = 0;
|
||||
if (geometry_datasource_input_stream)
|
||||
{
|
||||
number_of_compressed_datasources =
|
||||
io::readDatasourceIndexesSize(geometry_datasource_input_stream);
|
||||
}
|
||||
const auto number_of_compressed_datasources =
|
||||
io::readElementCount(geometry_datasource_input_stream);
|
||||
shared_layout_ptr->SetBlockSize<uint8_t>(SharedDataLayout::DATASOURCES_LIST,
|
||||
number_of_compressed_datasources);
|
||||
|
||||
@ -361,19 +357,15 @@ Storage::ReturnCode Storage::Run(int max_wait)
|
||||
throw util::exception("Could not open " + config.datasource_names_path.string() +
|
||||
" for reading.");
|
||||
}
|
||||
io::DatasourceNamesData datasource_names_data;
|
||||
if (datasource_names_input_stream)
|
||||
{
|
||||
datasource_names_data = io::readDatasourceNamesData(datasource_names_input_stream);
|
||||
}
|
||||
const io::DatasourceNamesData datasource_names_data =
|
||||
io::readDatasourceNames(datasource_names_input_stream);
|
||||
|
||||
shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::DATASOURCE_NAME_DATA,
|
||||
datasource_names_data.names.size());
|
||||
shared_layout_ptr->SetBlockSize<std::uint32_t>(SharedDataLayout::DATASOURCE_NAME_OFFSETS,
|
||||
datasource_names_data.offsets.size());
|
||||
shared_layout_ptr->SetBlockSize<std::uint32_t>(SharedDataLayout::DATASOURCE_NAME_LENGTHS,
|
||||
datasource_names_data.lengths.size());
|
||||
|
||||
shared_layout_ptr->SetBlockSize<std::size_t>(SharedDataLayout::DATASOURCE_NAME_OFFSETS,
|
||||
datasource_names_data.offsets.size());
|
||||
shared_layout_ptr->SetBlockSize<std::size_t>(SharedDataLayout::DATASOURCE_NAME_LENGTHS,
|
||||
datasource_names_data.lengths.size());
|
||||
boost::filesystem::ifstream intersection_stream(config.intersection_class_path,
|
||||
std::ios::binary);
|
||||
|
||||
@ -564,16 +556,16 @@ Storage::ReturnCode Storage::Run(int max_wait)
|
||||
EntryClassID *entry_class_id_ptr = shared_layout_ptr->GetBlockPtr<EntryClassID, true>(
|
||||
shared_memory_ptr, SharedDataLayout::ENTRY_CLASSID);
|
||||
|
||||
io::readEdgesData(edges_input_stream,
|
||||
via_geometry_ptr,
|
||||
name_id_ptr,
|
||||
turn_instructions_ptr,
|
||||
lane_data_id_ptr,
|
||||
travel_mode_ptr,
|
||||
entry_class_id_ptr,
|
||||
pre_turn_bearing_ptr,
|
||||
post_turn_bearing_ptr,
|
||||
number_of_original_edges);
|
||||
io::readEdges(edges_input_stream,
|
||||
via_geometry_ptr,
|
||||
name_id_ptr,
|
||||
turn_instructions_ptr,
|
||||
lane_data_id_ptr,
|
||||
travel_mode_ptr,
|
||||
entry_class_id_ptr,
|
||||
pre_turn_bearing_ptr,
|
||||
post_turn_bearing_ptr,
|
||||
number_of_original_edges);
|
||||
edges_input_stream.close();
|
||||
|
||||
// load compressed geometry
|
||||
@ -676,7 +668,7 @@ Storage::ReturnCode Storage::Run(int max_wait)
|
||||
util::PackedVector<OSMNodeID, true> osmnodeid_list;
|
||||
osmnodeid_list.reset(osmnodeid_ptr,
|
||||
shared_layout_ptr->num_entries[SharedDataLayout::OSM_NODE_ID_LIST]);
|
||||
io::readNodesData(nodes_input_stream, coordinates_ptr, osmnodeid_list, coordinate_list_size);
|
||||
io::readNodes(nodes_input_stream, coordinates_ptr, osmnodeid_list, coordinate_list_size);
|
||||
nodes_input_stream.close();
|
||||
|
||||
// store timestamp
|
||||
@ -685,9 +677,9 @@ Storage::ReturnCode Storage::Run(int max_wait)
|
||||
io::readTimestamp(timestamp_stream, timestamp_ptr, timestamp_size);
|
||||
|
||||
// store search tree portion of rtree
|
||||
RTreeNode *rtree_ptrtest = shared_layout_ptr->GetBlockPtr<RTreeNode, true>(shared_memory_ptr,
|
||||
SharedDataLayout::R_SEARCH_TREE);
|
||||
io::readRamIndexData(tree_node_file, rtree_ptrtest, tree_size);
|
||||
RTreeNode *rtree_ptrtest = shared_layout_ptr->GetBlockPtr<RTreeNode, true>(
|
||||
shared_memory_ptr, SharedDataLayout::R_SEARCH_TREE);
|
||||
io::readRamIndex(tree_node_file, rtree_ptrtest, tree_size);
|
||||
|
||||
// load core markers
|
||||
std::vector<char> unpacked_core_markers(number_of_core_markers);
|
||||
@ -744,9 +736,8 @@ Storage::ReturnCode Storage::Run(int max_wait)
|
||||
{
|
||||
util::exception("Could not open " + config.properties_path.string() + " for reading!");
|
||||
}
|
||||
io::readProperties(profile_properties_stream,
|
||||
profile_properties_ptr,
|
||||
sizeof(extractor::ProfileProperties));
|
||||
io::readProperties(
|
||||
profile_properties_stream, profile_properties_ptr, sizeof(extractor::ProfileProperties));
|
||||
|
||||
// load intersection classes
|
||||
if (!bearing_class_id_table.empty())
|
||||
|
||||
Loading…
Reference in New Issue
Block a user