Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Refactored loading code for .hsgr file

Browse Source
This commit is contained in:
Patrick Niklaus 2016-10-18 01:32:52 +02:00 committed by Patrick Niklaus
parent ae157d0b4f
commit b7ee38eca7
5 changed files with 102 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
include/engine/datafacade/internal_datafacade.hpp
View File

@ -14,6 +14,7 @@
#include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp"
#include "storage/storage_config.hpp"
#include "storage/io.hpp"
#include "engine/geospatial_query.hpp"
#include "util/graph_loader.hpp"
#include "util/guidance/turn_lanes.hpp"
@ -150,21 +151,26 @@ class InternalDataFacade final : public BaseDataFacade
void LoadGraph(const boost::filesystem::path &hsgr_path)
{
util::ShM<QueryGraph::NodeArrayEntry, false>::vector node_list;
util::ShM<QueryGraph::EdgeArrayEntry, false>::vector edge_list;
boost::filesystem::ifstream hsgr_input_stream(hsgr_path);
if (!hsgr_input_stream)
{
throw util::exception("Could not open " + hsgr_path.string() + " for reading.");
}
util::SimpleLogger().Write() << "loading graph from " << hsgr_path.string();
auto header = storage::io::readHSGRHeader(hsgr_input_stream);
m_check_sum = header.checksum;
m_number_of_nodes = readHSGRFromStream(hsgr_path, node_list, edge_list, &m_check_sum);
util::ShM<QueryGraph::NodeArrayEntry, false>::vector node_list(header.number_of_nodes);
util::ShM<QueryGraph::EdgeArrayEntry, false>::vector edge_list(header.number_of_edges);
storage::io::readHSGR(hsgr_input_stream,
node_list.data(),
header.number_of_nodes,
edge_list.data(),
header.number_of_edges);
BOOST_ASSERT_MSG(0 != node_list.size(), "node list empty");
// BOOST_ASSERT_MSG(0 != edge_list.size(), "edge list empty");
util::SimpleLogger().Write() << "loaded " << node_list.size() << " nodes and "
<< edge_list.size() << " edges";
m_query_graph = std::unique_ptr<QueryGraph>(new QueryGraph(node_list, edge_list));
BOOST_ASSERT_MSG(0 == node_list.size(), "node list not flushed");
BOOST_ASSERT_MSG(0 == edge_list.size(), "edge list not flushed");
util::SimpleLogger().Write() << "Data checksum is " << m_check_sum;
}

68
include/storage/io.hpp Normal file
View File

@ -0,0 +1,68 @@
#ifndef OSRM_STORAGE_IO_HPP_
#define OSRM_STORAGE_IO_HPP_
#include "util/fingerprint.hpp"
#include "util/simple_logger.hpp"
#include <boost/filesystem/fstream.hpp>
#include <tuple>
namespace osrm
{
namespace storage
{
namespace io
{
#pragma pack(push, 1)
struct HSGRHeader
{
std::uint32_t checksum;
std::uint32_t number_of_nodes;
std::uint32_t number_of_edges;
};
#pragma pack(pop)
static_assert(sizeof(HSGRHeader) == 12, "HSGRHeader is not packed");
// Returns the checksum and the number of nodes and number of edges
inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream)
{
const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid();
util::FingerPrint fingerprint_loaded;
input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(util::FingerPrint));
if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid))
{
util::SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}
HSGRHeader header;
input_stream.read(reinterpret_cast<char *>(&header.checksum), sizeof(header.checksum));
input_stream.read(reinterpret_cast<char *>(&header.number_of_nodes), sizeof(header.number_of_nodes));
input_stream.read(reinterpret_cast<char *>(&header.number_of_edges), sizeof(header.number_of_edges));
BOOST_ASSERT_MSG(0 != header.number_of_nodes, "number of nodes is zero");
// number of edges can be zero, this is the case in a few test fixtures
return header;
}
// Needs to be called after getHSGRSize() to get the correct offset in the stream
//
template <typename NodeT, typename EdgeT>
void readHSGR(boost::filesystem::ifstream &input_stream,
NodeT *node_buffer,
std::uint32_t number_of_nodes,
EdgeT *edge_buffer,
std::uint32_t number_of_edges)
{
input_stream.read(reinterpret_cast<char *>(node_buffer), number_of_nodes * sizeof(NodeT));
input_stream.read(reinterpret_cast<char *>(edge_buffer), number_of_edges * sizeof(EdgeT));
}
}
}
}
#endif

1
include/util/fingerprint.hpp
View File

@ -32,6 +32,7 @@ class FingerPrint
boost::uuids::uuid named_uuid;
};
static_assert(sizeof(FingerPrint) == 152, "FingerPrint has unexpected size");
static_assert(std::is_trivial<FingerPrint>::value, "FingerPrint needs to be trivial.");
}
}

50
include/util/graph_loader.hpp
View File

@ -148,56 +148,6 @@ NodeID loadEdgesFromFile(std::istream &input_stream,
return m;
}
template <typename NodeT, typename EdgeT>
unsigned readHSGRFromStream(const boost::filesystem::path &hsgr_file,
std::vector<NodeT> &node_list,
std::vector<EdgeT> &edge_list,
unsigned *check_sum)
{
if (!boost::filesystem::exists(hsgr_file))
{
throw exception("hsgr file does not exist");
}
if (0 == boost::filesystem::file_size(hsgr_file))
{
throw exception("hsgr file is empty");
}
boost::filesystem::ifstream hsgr_input_stream(hsgr_file, std::ios::binary);
const FingerPrint fingerprint_valid = FingerPrint::GetValid();
FingerPrint fingerprint_loaded;
hsgr_input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(FingerPrint));
if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid))
{
SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}
unsigned number_of_nodes = 0;
unsigned number_of_edges = 0;
hsgr_input_stream.read(reinterpret_cast<char *>(check_sum), sizeof(unsigned));
hsgr_input_stream.read(reinterpret_cast<char *>(&number_of_nodes), sizeof(unsigned));
BOOST_ASSERT_MSG(0 != number_of_nodes, "number of nodes is zero");
hsgr_input_stream.read(reinterpret_cast<char *>(&number_of_edges), sizeof(unsigned));
SimpleLogger().Write() << "number_of_nodes: " << number_of_nodes
<< ", number_of_edges: " << number_of_edges;
// BOOST_ASSERT_MSG( 0 != number_of_edges, "number of edges is zero");
node_list.resize(number_of_nodes);
hsgr_input_stream.read(reinterpret_cast<char *>(&node_list[0]),
number_of_nodes * sizeof(NodeT));
edge_list.resize(number_of_edges);
if (number_of_edges > 0)
{
hsgr_input_stream.read(reinterpret_cast<char *>(&edge_list[0]),
number_of_edges * sizeof(EdgeT));
}
return number_of_nodes;
}
}
}

60
src/storage/storage.cpp
View File

@ -1,3 +1,10 @@
#include "storage/storage.hpp"
#include "storage/io.hpp"
#include "storage/shared_barriers.hpp"
#include "storage/shared_datatype.hpp"
#include "storage/shared_memory.hpp"
#include "contractor/query_edge.hpp"
#include "extractor/compressed_edge_container.hpp"
#include "extractor/guidance/turn_instruction.hpp"
@ -5,10 +12,6 @@
#include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp"
#include "extractor/travel_mode.hpp"
#include "storage/shared_barriers.hpp"
#include "storage/shared_datatype.hpp"
#include "storage/shared_memory.hpp"
#include "storage/storage.hpp"
#include "engine/datafacade/datafacade_base.hpp"
#include "util/coordinate.hpp"
#include "util/exception.hpp"
@ -252,37 +255,12 @@ Storage::ReturnCode Storage::Run(int max_wait)
throw util::exception("Could not open " + config.hsgr_data_path.string() + " for reading.");
}
util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid();
util::FingerPrint fingerprint_loaded;
hsgr_input_stream.read((char *)&fingerprint_loaded, sizeof(util::FingerPrint));
if (fingerprint_loaded.TestGraphUtil(fingerprint_valid))
{
util::SimpleLogger().Write(logDEBUG) << "Fingerprint checked out ok";
}
else
{
util::SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build. "
"Reprocess to get rid of this warning.";
}
// load checksum
unsigned checksum = 0;
hsgr_input_stream.read((char *)&checksum, sizeof(unsigned));
auto hsgr_header = io::readHSGRHeader(hsgr_input_stream);
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::HSGR_CHECKSUM, 1);
// load graph node size
unsigned number_of_graph_nodes = 0;
hsgr_input_stream.read((char *)&number_of_graph_nodes, sizeof(unsigned));
BOOST_ASSERT_MSG((0 != number_of_graph_nodes), "number of nodes is zero");
shared_layout_ptr->SetBlockSize<QueryGraph::NodeArrayEntry>(SharedDataLayout::GRAPH_NODE_LIST,
number_of_graph_nodes);
// load graph edge size
unsigned number_of_graph_edges = 0;
hsgr_input_stream.read((char *)&number_of_graph_edges, sizeof(unsigned));
// BOOST_ASSERT_MSG(0 != number_of_graph_edges, "number of graph edges is zero");
hsgr_header.number_of_nodes);
shared_layout_ptr->SetBlockSize<QueryGraph::EdgeArrayEntry>(SharedDataLayout::GRAPH_EDGE_LIST,
number_of_graph_edges);
hsgr_header.number_of_edges);
// load rsearch tree size
boost::filesystem::ifstream tree_node_file(config.ram_index_path, std::ios::binary);
@ -477,7 +455,7 @@ Storage::ReturnCode Storage::Run(int max_wait)
// hsgr checksum
unsigned *checksum_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(
shared_memory_ptr, SharedDataLayout::HSGR_CHECKSUM);
*checksum_ptr = checksum;
*checksum_ptr = hsgr_header.checksum;
// ram index file name
char *file_index_path_ptr = shared_layout_ptr->GetBlockPtr<char, true>(
@ -726,21 +704,17 @@ Storage::ReturnCode Storage::Run(int max_wait)
QueryGraph::NodeArrayEntry *graph_node_list_ptr =
shared_layout_ptr->GetBlockPtr<QueryGraph::NodeArrayEntry, true>(
shared_memory_ptr, SharedDataLayout::GRAPH_NODE_LIST);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST) > 0)
{
hsgr_input_stream.read((char *)graph_node_list_ptr,
shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST));
}
// load the edges of the search graph
QueryGraph::EdgeArrayEntry *graph_edge_list_ptr =
shared_layout_ptr->GetBlockPtr<QueryGraph::EdgeArrayEntry, true>(
shared_memory_ptr, SharedDataLayout::GRAPH_EDGE_LIST);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST) > 0)
{
hsgr_input_stream.read((char *)graph_edge_list_ptr,
shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST));
}
io::readHSGR(hsgr_input_stream,
graph_node_list_ptr,
hsgr_header.number_of_nodes,
graph_edge_list_ptr,
hsgr_header.number_of_edges);
hsgr_input_stream.close();
// load profile properties