Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

refactor loading code of .nodes and .edges files and move to io.hpp

Browse Source
This commit is contained in:
Huyen Chau Nguyen 2016-10-18 12:00:02 -07:00
parent c4f010e363
commit 7b97e1035f
3 changed files with 118 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
include/engine/datafacade/internal_datafacade.hpp
View File

@ -181,27 +181,27 @@ class InternalDataFacade final : public BaseDataFacade
util::SimpleLogger().Write() << "Data checksum is " << m_check_sum;
}
void LoadNodeAndEdgeInformation(const boost::filesystem::path &nodes_file,
const boost::filesystem::path &edges_file)
void LoadNodeAndEdgeInformation(const boost::filesystem::path &nodes_file_path,
const boost::filesystem::path &edges_file_path)
{
boost::filesystem::ifstream nodes_input_stream(nodes_file, std::ios::binary);
extractor::QueryNode current_node;
unsigned number_of_coordinates = 0;
nodes_input_stream.read((char *)&number_of_coordinates, sizeof(unsigned));
m_coordinate_list.resize(number_of_coordinates);
m_osmnodeid_list.reserve(number_of_coordinates);
for (unsigned i = 0; i < number_of_coordinates; ++i)
boost::filesystem::ifstream nodes_input_stream(nodes_file_path, std::ios::binary);
if (!nodes_input_stream)
{
nodes_input_stream.read((char *)&current_node, sizeof(extractor::QueryNode));
m_coordinate_list[i] = util::Coordinate(current_node.lon, current_node.lat);
m_osmnodeid_list.push_back(current_node.node_id);
BOOST_ASSERT(m_coordinate_list[i].IsValid());
throw util::exception("Could not open " + nodes_file_path.string() + " for reading.");
}
boost::filesystem::ifstream edges_input_stream(edges_file, std::ios::binary);
unsigned number_of_edges = 0;
edges_input_stream.read((char *)&number_of_edges, sizeof(unsigned));
std::uint32_t number_of_coordinates = storage::io::readNodesSize(nodes_input_stream);
m_coordinate_list.resize(number_of_coordinates);
m_osmnodeid_list.reserve(number_of_coordinates);
storage::io::readNodesData(
nodes_input_stream, m_coordinate_list.data(), m_osmnodeid_list, number_of_coordinates);
boost::filesystem::ifstream edges_input_stream(edges_file_path, std::ios::binary);
if (!edges_input_stream)
{
throw util::exception("Could not open " + edges_file_path.string() + " for reading.");
}
auto number_of_edges = storage::io::readEdgesSize(edges_input_stream);
m_via_geometry_list.resize(number_of_edges);
m_name_ID_list.resize(number_of_edges);
m_turn_instruction_list.resize(number_of_edges);
@ -211,20 +211,16 @@ class InternalDataFacade final : public BaseDataFacade
m_pre_turn_bearing.resize(number_of_edges);
m_post_turn_bearing.resize(number_of_edges);
extractor::OriginalEdgeData current_edge_data;
for (unsigned i = 0; i < number_of_edges; ++i)
{
edges_input_stream.read((char *)&(current_edge_data),
sizeof(extractor::OriginalEdgeData));
m_via_geometry_list[i] = current_edge_data.via_geometry;
m_name_ID_list[i] = current_edge_data.name_id;
m_turn_instruction_list[i] = current_edge_data.turn_instruction;
m_lane_data_id[i] = current_edge_data.lane_data_id;
m_travel_mode_list[i] = current_edge_data.travel_mode;
m_entry_class_id_list[i] = current_edge_data.entry_classid;
m_pre_turn_bearing[i] = current_edge_data.pre_turn_bearing;
m_post_turn_bearing[i] = current_edge_data.post_turn_bearing;
}
storage::io::readEdgesData(edges_input_stream,
m_via_geometry_list.data(),
m_name_ID_list.data(),
m_turn_instruction_list.data(),
m_lane_data_id.data(),
m_travel_mode_list.data(),
m_entry_class_id_list.data(),
m_pre_turn_bearing.data(),
m_post_turn_bearing.data(),
number_of_edges);
}
void LoadCoreInformation(const boost::filesystem::path &core_data_file)

78
include/storage/io.hpp
View File

@ -63,9 +63,9 @@ inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream)
// Needs to be called after readHSGRHeader() to get the correct offset in the stream
template <typename NodeT, typename EdgeT>
void readHSGR(boost::filesystem::ifstream &input_stream,
NodeT *node_buffer,
NodeT node_buffer[],
std::uint32_t number_of_nodes,
EdgeT *edge_buffer,
EdgeT edge_buffer[],
std::uint32_t number_of_edges)
{
input_stream.read(reinterpret_cast<char *>(node_buffer), number_of_nodes * sizeof(NodeT));
@ -74,20 +74,88 @@ void readHSGR(boost::filesystem::ifstream &input_stream,
// Returns the size of the timestamp in a file
inline std::uint32_t readTimestampSize(boost::filesystem::ifstream &timestamp_input_stream)
{
timestamp_input_stream.seekg (0, timestamp_input_stream.end);
timestamp_input_stream.seekg(0, timestamp_input_stream.end);
auto length = timestamp_input_stream.tellg();
timestamp_input_stream.seekg (0, timestamp_input_stream.beg);
timestamp_input_stream.seekg(0, timestamp_input_stream.beg);
return length;
}
// Reads the timestamp in a file
inline void readTimestamp(boost::filesystem::ifstream &timestamp_input_stream,
char *timestamp,
char timestamp[],
std::size_t timestamp_length)
{
timestamp_input_stream.read(timestamp, timestamp_length * sizeof(char));
}
// Returns the number of edges in a .edges file
inline std::uint32_t readEdgesSize(boost::filesystem::ifstream &edges_input_stream)
{
std::uint32_t number_of_edges;
edges_input_stream.read((char *)&number_of_edges, sizeof(std::uint32_t));
return number_of_edges;
}
// Reads edge data from .edge files which includes its
// geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID
template <typename GeometryIDT,
typename NameIDT,
typename TurnInstructionT,
typename LaneDataIDT,
typename TravelModeT,
typename EntryClassIDT,
typename PreTurnBearingT,
typename PostTurnBearingT>
void readEdgesData(boost::filesystem::ifstream &edges_input_stream,
GeometryIDT geometry_list[],
NameIDT name_id_list[],
TurnInstructionT turn_instruction_list[],
LaneDataIDT lane_data_id_list[],
TravelModeT travel_mode_list[],
EntryClassIDT entry_class_id_list[],
PreTurnBearingT pre_turn_bearing_list[],
PostTurnBearingT post_turn_bearing_list[],
std::uint32_t number_of_edges)
{
extractor::OriginalEdgeData current_edge_data;
for (std::uint32_t i = 0; i < number_of_edges; ++i)
{
edges_input_stream.read((char *)&(current_edge_data), sizeof(extractor::OriginalEdgeData));
geometry_list[i] = current_edge_data.via_geometry;
name_id_list[i] = current_edge_data.name_id;
turn_instruction_list[i] = current_edge_data.turn_instruction;
lane_data_id_list[i] = current_edge_data.lane_data_id;
travel_mode_list[i] = current_edge_data.travel_mode;
entry_class_id_list[i] = current_edge_data.entry_classid;
pre_turn_bearing_list[i] = current_edge_data.pre_turn_bearing;
post_turn_bearing_list[i] = current_edge_data.post_turn_bearing;
}
}
// Returns the number of nodes in a .nodes file
inline std::uint32_t readNodesSize(boost::filesystem::ifstream &nodes_input_stream)
{
std::uint32_t number_of_coordinates;
nodes_input_stream.read((char *)&number_of_coordinates, sizeof(std::uint32_t));
return number_of_coordinates;
}
// Reads coordinates and OSM node IDs from .nodes files
template <typename CoordinateT, typename OSMNodeIDVectorT>
void readNodesData(boost::filesystem::ifstream &nodes_input_stream,
CoordinateT coordinate_list[],
OSMNodeIDVectorT &osmnodeid_list,
std::uint32_t number_of_coordinates)
{
extractor::QueryNode current_node;
for (std::uint32_t i = 0; i < number_of_coordinates; ++i)
{
nodes_input_stream.read((char *)&current_node, sizeof(extractor::QueryNode));
coordinate_list[i] = CoordinateT(current_node.lon, current_node.lat);
osmnodeid_list.push_back(current_node.node_id);
BOOST_ASSERT(coordinate_list[i].IsValid());
}
}
}
}
}

49
src/storage/storage.cpp
View File

@ -1,10 +1,3 @@
#include "storage/storage.hpp"
#include "storage/io.hpp"
#include "storage/shared_barriers.hpp"
#include "storage/shared_datatype.hpp"
#include "storage/shared_memory.hpp"
#include "contractor/query_edge.hpp"
#include "extractor/compressed_edge_container.hpp"
#include "extractor/guidance/turn_instruction.hpp"
@ -12,6 +5,11 @@
#include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp"
#include "extractor/travel_mode.hpp"
#include "storage/io.hpp"
#include "storage/shared_barriers.hpp"
#include "storage/shared_datatype.hpp"
#include "storage/shared_memory.hpp"
#include "storage/storage.hpp"
#include "engine/datafacade/datafacade_base.hpp"
#include "util/coordinate.hpp"
#include "util/exception.hpp"
@ -232,8 +230,7 @@ Storage::ReturnCode Storage::Run(int max_wait)
throw util::exception("Could not open " + config.edges_data_path.string() +
" for reading.");
}
unsigned number_of_original_edges = 0;
edges_input_stream.read((char *)&number_of_original_edges, sizeof(unsigned));
auto number_of_original_edges = io::readEdgesSize(edges_input_stream);
// note: settings this all to the same size is correct, we extract them from the same struct
shared_layout_ptr->SetBlockSize<NodeID>(SharedDataLayout::VIA_NODE_LIST,
@ -305,8 +302,7 @@ Storage::ReturnCode Storage::Run(int max_wait)
{
throw util::exception("Could not open " + config.core_data_path.string() + " for reading.");
}
unsigned coordinate_list_size = 0;
nodes_input_stream.read((char *)&coordinate_list_size, sizeof(unsigned));
auto coordinate_list_size = io::readNodesSize(nodes_input_stream);
shared_layout_ptr->SetBlockSize<util::Coordinate>(SharedDataLayout::COORDINATE_LIST,
coordinate_list_size);
// we'll read a list of OSM node IDs from the same data, so set the block size for the same
@ -577,19 +573,16 @@ Storage::ReturnCode Storage::Run(int max_wait)
EntryClassID *entry_class_id_ptr = shared_layout_ptr->GetBlockPtr<EntryClassID, true>(
shared_memory_ptr, SharedDataLayout::ENTRY_CLASSID);
extractor::OriginalEdgeData current_edge_data;
for (unsigned i = 0; i < number_of_original_edges; ++i)
{
edges_input_stream.read((char *)&(current_edge_data), sizeof(extractor::OriginalEdgeData));
via_geometry_ptr[i] = current_edge_data.via_geometry;
name_id_ptr[i] = current_edge_data.name_id;
travel_mode_ptr[i] = current_edge_data.travel_mode;
lane_data_id_ptr[i] = current_edge_data.lane_data_id;
turn_instructions_ptr[i] = current_edge_data.turn_instruction;
entry_class_id_ptr[i] = current_edge_data.entry_classid;
pre_turn_bearing_ptr[i] = current_edge_data.pre_turn_bearing;
post_turn_bearing_ptr[i] = current_edge_data.post_turn_bearing;
}
io::readEdgesData(edges_input_stream,
via_geometry_ptr,
name_id_ptr,
turn_instructions_ptr,
lane_data_id_ptr,
travel_mode_ptr,
entry_class_id_ptr,
pre_turn_bearing_ptr,
post_turn_bearing_ptr,
number_of_original_edges);
edges_input_stream.close();
// load compressed geometry
@ -691,13 +684,7 @@ Storage::ReturnCode Storage::Run(int max_wait)
osmnodeid_list.reset(osmnodeid_ptr,
shared_layout_ptr->num_entries[SharedDataLayout::OSM_NODE_ID_LIST]);
extractor::QueryNode current_node;
for (unsigned i = 0; i < coordinate_list_size; ++i)
{
nodes_input_stream.read((char *)&current_node, sizeof(extractor::QueryNode));
coordinates_ptr[i] = util::Coordinate(current_node.lon, current_node.lat);
osmnodeid_list.push_back(current_node.node_id);
}
io::readNodesData(nodes_input_stream, coordinates_ptr, osmnodeid_list, coordinate_list_size);
nodes_input_stream.close();
// store timestamp