Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

refactor function names; consolidate readCount() functions;

Browse Source 
remove templated types as much as possible for type safety;
add more comments;
clean up code, add const if possible;
This commit is contained in:
Huyen Chau Nguyen
2016-10-21 15:24:55 -07:00
parent fe94977c9b
commit cf35bbeb50
8 changed files with 169 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/engine/datafacade/internal_datafacade.hpp
+21 -21
View File
@@ -123,8 +123,8 @@ class InternalDataFacade final : public BaseDataFacade
{
throw util::exception("Could not open " + properties_path.string() + " for reading.");
}
auto PropertiesSize = storage::io::readPropertiesSize();
storage::io::readProperties(in_stream, &m_profile_properties, PropertiesSize);
const auto properties_size = storage::io::readPropertiesCount();
storage::io::readProperties(in_stream, &m_profile_properties, properties_size);
}
void LoadLaneTupleIdPairs(const boost::filesystem::path &lane_data_path)
@@ -151,7 +151,7 @@ class InternalDataFacade final : public BaseDataFacade
throw util::exception("Could not open " + timestamp_path.string() + " for reading.");
}
auto timestamp_size = storage::io::readTimestampSize(timestamp_stream);
const auto timestamp_size = storage::io::readNumberOfBytes(timestamp_stream);
m_timestamp.resize(timestamp_size);
storage::io::readTimestamp(timestamp_stream, &m_timestamp.front(), timestamp_size);
}
@@ -164,7 +164,7 @@ class InternalDataFacade final : public BaseDataFacade
throw util::exception("Could not open " + hsgr_path.string() + " for reading.");
}
auto header = storage::io::readHSGRHeader(hsgr_input_stream);
const auto header = storage::io::readHSGRHeader(hsgr_input_stream);
m_check_sum = header.checksum;
util::ShM<QueryGraph::NodeArrayEntry, false>::vector node_list(header.number_of_nodes);
@@ -190,10 +190,10 @@ class InternalDataFacade final : public BaseDataFacade
throw util::exception("Could not open " + nodes_file_path.string() + " for reading.");
}
std::uint32_t number_of_coordinates = storage::io::readNodesSize(nodes_input_stream);
const auto number_of_coordinates = storage::io::readElementCount(nodes_input_stream);
m_coordinate_list.resize(number_of_coordinates);
m_osmnodeid_list.reserve(number_of_coordinates);
storage::io::readNodesData(
storage::io::readNodes(
nodes_input_stream, m_coordinate_list.data(), m_osmnodeid_list, number_of_coordinates);
boost::filesystem::ifstream edges_input_stream(edges_file_path, std::ios::binary);
@@ -201,7 +201,7 @@ class InternalDataFacade final : public BaseDataFacade
{
throw util::exception("Could not open " + edges_file_path.string() + " for reading.");
}
auto number_of_edges = storage::io::readEdgesSize(edges_input_stream);
const auto number_of_edges = storage::io::readElementCount(edges_input_stream);
m_via_geometry_list.resize(number_of_edges);
m_name_ID_list.resize(number_of_edges);
m_turn_instruction_list.resize(number_of_edges);
@@ -211,16 +211,16 @@ class InternalDataFacade final : public BaseDataFacade
m_pre_turn_bearing.resize(number_of_edges);
m_post_turn_bearing.resize(number_of_edges);
storage::io::readEdgesData(edges_input_stream,
m_via_geometry_list.data(),
m_name_ID_list.data(),
m_turn_instruction_list.data(),
m_lane_data_id.data(),
m_travel_mode_list.data(),
m_entry_class_id_list.data(),
m_pre_turn_bearing.data(),
m_post_turn_bearing.data(),
number_of_edges);
storage::io::readEdges(edges_input_stream,
m_via_geometry_list.data(),
m_name_ID_list.data(),
m_turn_instruction_list.data(),
m_lane_data_id.data(),
m_travel_mode_list.data(),
m_entry_class_id_list.data(),
m_pre_turn_bearing.data(),
m_post_turn_bearing.data(),
number_of_edges);
}
void LoadCoreInformation(const boost::filesystem::path &core_data_file)
@@ -292,7 +292,7 @@ class InternalDataFacade final : public BaseDataFacade
}
BOOST_ASSERT(datasources_stream);
auto number_of_datasources = storage::io::readDatasourceIndexesSize(datasources_stream);
const auto number_of_datasources = storage::io::readElementCount(datasources_stream);
if (number_of_datasources > 0)
{
m_datasource_list.resize(number_of_datasources);
@@ -308,15 +308,15 @@ class InternalDataFacade final : public BaseDataFacade
}
BOOST_ASSERT(datasourcenames_stream);
auto datasource_names_data = storage::io::readDatasourceNamesData(datasourcenames_stream);
const auto datasource_names_data = storage::io::readDatasourceNames(datasourcenames_stream);
m_datasource_names.resize(datasource_names_data.lengths.size());
for (std::uint32_t i = 0; i < datasource_names_data.lengths.size(); ++i)
for (std::size_t i = 0; i < datasource_names_data.lengths.size(); ++i)
{
auto name_begin =
datasource_names_data.names.begin() + datasource_names_data.offsets[i];
auto name_end = datasource_names_data.names.begin() + datasource_names_data.offsets[i] +
datasource_names_data.lengths[i];
m_datasource_names[i] = std::move(std::string(name_begin, name_end));
m_datasource_names[i] = std::string(name_begin, name_end);
}
}
include/extractor/original_edge_data.hpp
+1 -1
View File
@@ -17,7 +17,7 @@ namespace extractor
struct OriginalEdgeData
{
explicit OriginalEdgeData(GeometryID via_geometry,
unsigned name_id,
NameID name_id,
LaneDataID lane_data_id,
guidance::TurnInstruction turn_instruction,
EntryClassID entry_classid,
include/storage/io.hpp
+95 -115
View File
@@ -1,10 +1,13 @@
#ifndef OSRM_STORAGE_IO_HPP_
#define OSRM_STORAGE_IO_HPP_
#include "contractor/query_edge.hpp"
#include "extractor/extractor.hpp"
#include "extractor/original_edge_data.hpp"
#include "extractor/query_node.hpp"
#include "util/fingerprint.hpp"
#include "util/simple_logger.hpp"
#include "util/static_graph.hpp"
#include <boost/filesystem/fstream.hpp>
@@ -17,27 +20,38 @@ namespace storage
namespace io
{
inline std::size_t readPropertiesSize() { return 1; }
template <typename PropertiesT>
inline void readProperties(boost::filesystem::ifstream &properties_stream,
PropertiesT *properties,
std::size_t properties_size)
// Reads the count of elements that is written in the file header and returns the number
inline std::uint64_t readElementCount(boost::filesystem::ifstream &input_stream)
{
properties_stream.read(reinterpret_cast<char *>(properties), properties_size);
std::uint64_t number_of_elements = 0;
input_stream.read((char *)&number_of_elements, sizeof(std::uint64_t));
return number_of_elements;
}
// To make function calls consistent, this function returns the fixed number of properties
inline std::size_t readPropertiesCount() { return 1; }
// Returns the number of bytes in a file
inline std::size_t readNumberOfBytes(boost::filesystem::ifstream &input_stream)
{
input_stream.seekg(0, input_stream.end);
auto length = input_stream.tellg();
input_stream.seekg(0, input_stream.beg);
return length;
}
#pragma pack(push, 1)
struct HSGRHeader
{
std::uint32_t checksum;
std::uint32_t number_of_nodes;
std::uint32_t number_of_edges;
std::uint64_t number_of_nodes;
std::uint64_t number_of_edges;
};
#pragma pack(pop)
static_assert(sizeof(HSGRHeader) == 12, "HSGRHeader is not packed");
static_assert(sizeof(HSGRHeader) == 20, "HSGRHeader is not packed");
// Returns the checksum and the number of nodes and number of edges
// Reads the checksum, number of nodes and number of edges written in the header file of a `.hsgr`
// file and returns them in a HSGRHeader struct
inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream)
{
const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid();
@@ -62,66 +76,65 @@ inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream)
return header;
}
// Reads the graph data of a `.hsgr` file into memory
// Needs to be called after readHSGRHeader() to get the correct offset in the stream
template <typename NodeT, typename EdgeT>
void readHSGR(boost::filesystem::ifstream &input_stream,
NodeT node_buffer[],
std::uint32_t number_of_nodes,
EdgeT edge_buffer[],
std::uint32_t number_of_edges)
using NodeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::NodeArrayEntry;
using EdgeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::EdgeArrayEntry;
inline void readHSGR(boost::filesystem::ifstream &input_stream,
NodeT *node_buffer,
const std::uint64_t number_of_nodes,
EdgeT *edge_buffer,
const std::uint64_t number_of_edges)
{
BOOST_ASSERT(node_buffer);
BOOST_ASSERT(edge_buffer);
input_stream.read(reinterpret_cast<char *>(node_buffer), number_of_nodes * sizeof(NodeT));
input_stream.read(reinterpret_cast<char *>(edge_buffer), number_of_edges * sizeof(EdgeT));
}
// Returns the size of the timestamp in a file
inline std::uint32_t readTimestampSize(boost::filesystem::ifstream &timestamp_input_stream)
// Loads properties from a `.properties` file into memory
inline void readProperties(boost::filesystem::ifstream &properties_stream,
extractor::ProfileProperties *properties,
const std::size_t properties_size)
{
timestamp_input_stream.seekg(0, timestamp_input_stream.end);
auto length = timestamp_input_stream.tellg();
timestamp_input_stream.seekg(0, timestamp_input_stream.beg);
return length;
BOOST_ASSERT(properties);
properties_stream.read(reinterpret_cast<char *>(properties), properties_size);
}
// Reads the timestamp in a file
// Reads the timestamp in a `.timestamp` file
// Use readNumberOfBytes() beforehand to get the length of the file
inline void readTimestamp(boost::filesystem::ifstream &timestamp_input_stream,
char timestamp[],
std::size_t timestamp_length)
char *timestamp,
const std::size_t timestamp_length)
{
BOOST_ASSERT(timestamp);
timestamp_input_stream.read(timestamp, timestamp_length * sizeof(char));
}
// Returns the number of edges in a .edges file
inline std::uint32_t readEdgesSize(boost::filesystem::ifstream &edges_input_stream)
// Loads datasource_indexes from .datasource_indexes into memory
// Needs to be called after readElementCount() to get the correct offset in the stream
inline void readDatasourceIndexes(boost::filesystem::ifstream &datasource_indexes_input_stream,
uint8_t *datasource_buffer,
const std::uint64_t number_of_datasource_indexes)
{
std::uint32_t number_of_edges;
edges_input_stream.read((char *)&number_of_edges, sizeof(std::uint32_t));
return number_of_edges;
BOOST_ASSERT(datasource_buffer);
datasource_indexes_input_stream.read(reinterpret_cast<char *>(datasource_buffer),
number_of_datasource_indexes * sizeof(std::uint8_t));
}
// Reads edge data from .edge files which includes its
// Loads edge data from .edge files into memory which includes its
// geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID
// Needs to be called after readEdgesSize() to get the correct offset in the stream
template <typename GeometryIDT,
typename NameIDT,
typename TurnInstructionT,
typename LaneDataIDT,
typename TravelModeT,
typename EntryClassIDT,
typename PreTurnBearingT,
typename PostTurnBearingT>
void readEdgesData(boost::filesystem::ifstream &edges_input_stream,
GeometryIDT geometry_list[],
NameIDT name_id_list[],
TurnInstructionT turn_instruction_list[],
LaneDataIDT lane_data_id_list[],
TravelModeT travel_mode_list[],
EntryClassIDT entry_class_id_list[],
PreTurnBearingT pre_turn_bearing_list[],
PostTurnBearingT post_turn_bearing_list[],
std::uint32_t number_of_edges)
// Needs to be called after readElementCount() to get the correct offset in the stream
inline void readEdges(boost::filesystem::ifstream &edges_input_stream,
GeometryID *geometry_list,
NameID *name_id_list,
extractor::guidance::TurnInstruction *turn_instruction_list,
LaneDataID *lane_data_id_list,
extractor::TravelMode *travel_mode_list,
EntryClassID *entry_class_id_list,
util::guidance::TurnBearing *pre_turn_bearing_list,
util::guidance::TurnBearing *post_turn_bearing_list,
const std::uint64_t number_of_edges)
{
BOOST_ASSERT(geometry_list);
BOOST_ASSERT(name_id_list);
@@ -130,9 +143,10 @@ void readEdgesData(boost::filesystem::ifstream &edges_input_stream,
BOOST_ASSERT(travel_mode_list);
BOOST_ASSERT(entry_class_id_list);
extractor::OriginalEdgeData current_edge_data;
for (std::uint32_t i = 0; i < number_of_edges; ++i)
for (std::uint64_t i = 0; i < number_of_edges; ++i)
{
edges_input_stream.read((char *)&(current_edge_data), sizeof(extractor::OriginalEdgeData));
geometry_list[i] = current_edge_data.via_geometry;
name_id_list[i] = current_edge_data.name_id;
turn_instruction_list[i] = current_edge_data.turn_instruction;
@@ -144,96 +158,62 @@ void readEdgesData(boost::filesystem::ifstream &edges_input_stream,
}
}
// Returns the number of nodes in a .nodes file
inline std::uint32_t readNodesSize(boost::filesystem::ifstream &nodes_input_stream)
{
std::uint32_t number_of_coordinates;
nodes_input_stream.read((char *)&number_of_coordinates, sizeof(std::uint32_t));
return number_of_coordinates;
}
// Reads coordinates and OSM node IDs from .nodes files
// Needs to be called after readNodesSize() to get the correct offset in the stream
template <typename CoordinateT, typename OSMNodeIDVectorT>
void readNodesData(boost::filesystem::ifstream &nodes_input_stream,
CoordinateT coordinate_list[],
OSMNodeIDVectorT &osmnodeid_list,
std::uint32_t number_of_coordinates)
// Loads coordinates and OSM node IDs from .nodes files into memory
// Needs to be called after readElementCount() to get the correct offset in the stream
template <typename OSMNodeIDVectorT>
void readNodes(boost::filesystem::ifstream &nodes_input_stream,
util::Coordinate *coordinate_list,
OSMNodeIDVectorT &osmnodeid_list,
const std::uint64_t number_of_coordinates)
{
BOOST_ASSERT(coordinate_list);
extractor::QueryNode current_node;
for (std::uint32_t i = 0; i < number_of_coordinates; ++i)
for (std::uint64_t i = 0; i < number_of_coordinates; ++i)
{
nodes_input_stream.read((char *)&current_node, sizeof(extractor::QueryNode));
coordinate_list[i] = CoordinateT(current_node.lon, current_node.lat);
coordinate_list[i] = util::Coordinate(current_node.lon, current_node.lat);
osmnodeid_list.push_back(current_node.node_id);
BOOST_ASSERT(coordinate_list[i].IsValid());
}
}
// Returns the number of indexes in a .datasource_indexes file
inline std::uint64_t
readDatasourceIndexesSize(boost::filesystem::ifstream &datasource_indexes_input_stream)
{
std::uint64_t number_of_datasource_indexes;
datasource_indexes_input_stream.read(reinterpret_cast<char *>(&number_of_datasource_indexes),
sizeof(std::uint64_t));
return number_of_datasource_indexes;
}
// Reads datasource_indexes
// Needs to be called after readDatasourceIndexesSize() to get the correct offset in the stream
inline void readDatasourceIndexes(boost::filesystem::ifstream &datasource_indexes_input_stream,
uint8_t datasource_buffer[],
std::uint32_t number_of_datasource_indexes)
{
BOOST_ASSERT(datasource_buffer);
datasource_indexes_input_stream.read(reinterpret_cast<char *>(datasource_buffer),
number_of_datasource_indexes * sizeof(std::uint8_t));
}
// Reads datasource names out of .datasource_names files and metadata such as
// the length and offset of each name
struct DatasourceNamesData
{
std::vector<char> names;
std::vector<std::uint32_t> offsets;
std::vector<std::uint32_t> lengths;
std::vector<std::size_t> offsets;
std::vector<std::size_t> lengths;
};
inline DatasourceNamesData
readDatasourceNamesData(boost::filesystem::ifstream &datasource_names_input_stream)
inline DatasourceNamesData readDatasourceNames(boost::filesystem::ifstream &datasource_names_input_stream)
{
DatasourceNamesData datasource_names_data;
if (datasource_names_input_stream)
std::string name;
while (std::getline(datasource_names_input_stream, name))
{
std::string name;
while (std::getline(datasource_names_input_stream, name))
{
datasource_names_data.offsets.push_back(datasource_names_data.names.size());
datasource_names_data.lengths.push_back(name.size());
std::copy(name.c_str(),
name.c_str() + name.size(),
std::back_inserter(datasource_names_data.names));
}
datasource_names_data.offsets.push_back(datasource_names_data.names.size());
datasource_names_data.lengths.push_back(name.size());
std::copy(name.c_str(),
name.c_str() + name.size(),
std::back_inserter(datasource_names_data.names));
}
return datasource_names_data;
}
// Returns the number of ram indexes
inline std::uint32_t readRamIndexSize(boost::filesystem::ifstream &ram_index_input_stream)
{
std::uint32_t tree_size = 0;
ram_index_input_stream.read((char *)&tree_size, sizeof(std::uint32_t));
return tree_size;
}
// Loads ram indexes of R-Trees from `.ramIndex` files into memory
// Needs to be called after readElementCount() to get the correct offset in the stream
// template <bool UseSharedMemory>
// NB Cannot be written without templated type because of cyclic depencies between
// `static_rtree.hpp` and `io.hpp`
template <typename RTreeNodeT>
void readRamIndexData(boost::filesystem::ifstream &ram_index_input_stream,
RTreeNodeT rtree_buffer[],
std::uint32_t tree_size)
void readRamIndex(boost::filesystem::ifstream &ram_index_input_stream,
RTreeNodeT *rtree_buffer,
const std::uint64_t tree_size)
{
BOOST_ASSERT(rtree_buffer);
ram_index_input_stream.read(reinterpret_cast<char *>(rtree_buffer), sizeof(RTreeNodeT) * tree_size);
ram_index_input_stream.read(reinterpret_cast<char *>(rtree_buffer),
sizeof(RTreeNodeT) * tree_size);
}
}
include/util/static_rtree.hpp
+4 -5
View File
@@ -333,9 +333,9 @@ class StaticRTree
// open tree file
boost::filesystem::ofstream tree_node_file(tree_node_filename, std::ios::binary);
std::uint32_t size_of_tree = m_search_tree.size();
std::uint64_t size_of_tree = m_search_tree.size();
BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty");
tree_node_file.write((char *)&size_of_tree, sizeof(std::uint32_t));
tree_node_file.write((char *)&size_of_tree, sizeof(size_of_tree));
tree_node_file.write((char *)&m_search_tree[0], sizeof(TreeNode) * size_of_tree);
MapLeafNodesFile(leaf_node_filename);
@@ -357,11 +357,10 @@ class StaticRTree
}
boost::filesystem::ifstream tree_node_file(node_file, std::ios::binary);
std::uint32_t tree_size = storage::io::readRamIndexSize(tree_node_file);
const auto tree_size = storage::io::readElementCount(tree_node_file);
m_search_tree.resize(tree_size);
storage::io::readRamIndexData(tree_node_file, &m_search_tree[0], tree_size);
storage::io::readRamIndex(tree_node_file, &m_search_tree[0], tree_size);
MapLeafNodesFile(leaf_file);
}