Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Move .osrm file to tar format

Browse Source
This commit is contained in:
Patrick Niklaus
2018-03-20 00:47:34 +00:00
parent 06f28ffd34
commit f407afa694
10 changed files with 91 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/extractor/extraction_containers.hpp
+4 -4
View File
@@ -7,7 +7,7 @@
#include "extractor/restriction.hpp"
#include "extractor/scripting_environment.hpp"
#include "storage/io.hpp"
#include "storage/tar_fwd.hpp"
namespace osrm
{
@@ -27,9 +27,9 @@ class ExtractionContainers
void PrepareRestrictions();
void PrepareEdges(ScriptingEnvironment &scripting_environment);
void WriteNodes(storage::io::FileWriter &file_out) const;
void WriteEdges(storage::io::FileWriter &file_out) const;
void WriteMetadata(storage::io::FileWriter &file_out) const;
void WriteNodes(storage::tar::FileWriter &file_out) const;
void WriteEdges(storage::tar::FileWriter &file_out) const;
void WriteMetadata(storage::tar::FileWriter &file_out) const;
void WriteCharData(const std::string &file_name);
public:
include/extractor/files.hpp
+37 -2
View File
@@ -4,6 +4,7 @@
#include "extractor/edge_based_edge.hpp"
#include "extractor/node_data_container.hpp"
#include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp"
#include "extractor/serialization.hpp"
#include "extractor/turn_lane_types.hpp"
@@ -368,7 +369,7 @@ inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPen
// writes .osrm.restrictions
template <typename ConditionalRestrictionsT>
inline void writeConditionalRestrictions(const boost::filesystem::path &path,
const ConditionalRestrictionsT &conditional_restrictions)
const ConditionalRestrictionsT &conditional_restrictions)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
@@ -378,13 +379,47 @@ inline void writeConditionalRestrictions(const boost::filesystem::path &path,
// read .osrm.restrictions
template <typename ConditionalRestrictionsT>
inline void readConditionalRestrictions(const boost::filesystem::path &path, ConditionalRestrictionsT &conditional_restrictions)
inline void readConditionalRestrictions(const boost::filesystem::path &path,
ConditionalRestrictionsT &conditional_restrictions)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
serialization::read(reader, "/common/conditional_restrictions", conditional_restrictions);
}
// reads .osrm file which is a temporary file of osrm-extract
template <typename BarrierOutIter, typename TrafficSignalsOutIter, typename PackedOSMIDsT>
void readRawNBGraph(const boost::filesystem::path &path,
BarrierOutIter barriers,
TrafficSignalsOutIter traffic_signals,
std::vector<util::Coordinate> &coordinates,
PackedOSMIDsT &osm_node_ids,
std::vector<extractor::NodeBasedEdge> &edge_list,
std::vector<extractor::NodeBasedEdgeAnnotation> &annotations)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
auto number_of_nodes = reader.ReadElementCount64("/extractor/nodes");
coordinates.resize(number_of_nodes);
osm_node_ids.reserve(number_of_nodes);
auto index = 0;
auto decode = [&](const auto &current_node) {
coordinates[index].lon = current_node.lon;
coordinates[index].lat = current_node.lat;
osm_node_ids.push_back(current_node.node_id);
index++;
};
reader.ReadStreaming<extractor::QueryNode>("/extractor/nodes", boost::make_function_output_iterator(decode));
reader.ReadStreaming<NodeID>("/extractor/barriers", barriers);
reader.ReadStreaming<NodeID>("/extractor/traffic_lights", traffic_signals);
storage::serialization::read(reader, "/extractor/edges", edge_list);
storage::serialization::read(reader, "/extractor/annotations", annotations);
}
}
}
}
include/util/graph_loader.hpp
-127
View File
@@ -1,127 +0,0 @@
#ifndef GRAPH_LOADER_HPP
#define GRAPH_LOADER_HPP
#include "extractor/node_based_edge.hpp"
#include "extractor/packed_osm_ids.hpp"
#include "extractor/query_node.hpp"
#include "extractor/restriction.hpp"
#include "storage/io.hpp"
#include "util/exception.hpp"
#include "util/fingerprint.hpp"
#include "util/log.hpp"
#include "util/packed_vector.hpp"
#include "util/typedefs.hpp"
#include <boost/assert.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/function_output_iterator.hpp>
#include <tbb/parallel_sort.h>
#include <cmath>
#include <fstream>
#include <ios>
#include <unordered_set>
#include <vector>
namespace osrm
{
namespace util
{
/**
* Reads the beginning of an .osrm file and produces:
* - barrier nodes
* - traffic lights
* - nodes indexed by their internal (non-osm) id
*/
template <typename BarrierOutIter, typename TrafficSignalsOutIter>
NodeID loadNodesFromFile(storage::io::FileReader &file_reader,
BarrierOutIter barriers,
TrafficSignalsOutIter traffic_signals,
std::vector<util::Coordinate> &coordinates,
extractor::PackedOSMIDs &osm_node_ids)
{
auto number_of_nodes = file_reader.ReadElementCount64();
Log() << "Importing number_of_nodes new = " << number_of_nodes << " nodes ";
coordinates.resize(number_of_nodes);
osm_node_ids.reserve(number_of_nodes);
auto index = 0;
auto decode = [&](const extractor::QueryNode &current_node) {
coordinates[index].lon = current_node.lon;
coordinates[index].lat = current_node.lat;
osm_node_ids.push_back(current_node.node_id);
index++;
};
file_reader.ReadStreaming<extractor::QueryNode>(boost::make_function_output_iterator(decode),
number_of_nodes);
auto num_barriers = file_reader.ReadElementCount64();
file_reader.ReadStreaming<NodeID>(barriers, num_barriers);
auto num_lights = file_reader.ReadElementCount64();
file_reader.ReadStreaming<NodeID>(traffic_signals, num_lights);
return number_of_nodes;
}
/**
* Reads a .osrm file and produces the edges.
*/
inline EdgeID loadEdgesFromFile(storage::io::FileReader &file_reader,
std::vector<extractor::NodeBasedEdge> &edge_list)
{
auto number_of_edges = file_reader.ReadElementCount64();
edge_list.resize(number_of_edges);
Log() << " and " << number_of_edges << " edges ";
file_reader.ReadInto(edge_list.data(), number_of_edges);
BOOST_ASSERT(edge_list.size() > 0);
#ifndef NDEBUG
Log() << "Validating loaded edges...";
tbb::parallel_sort(
edge_list.begin(),
edge_list.end(),
[](const extractor::NodeBasedEdge &lhs, const extractor::NodeBasedEdge &rhs) {
return (lhs.source < rhs.source) ||
(lhs.source == rhs.source && lhs.target < rhs.target);
});
for (auto i = 1u; i < edge_list.size(); ++i)
{
const auto &edge = edge_list[i];
const auto &prev_edge = edge_list[i - 1];
BOOST_ASSERT_MSG(edge.weight > 0, "loaded null weight");
BOOST_ASSERT_MSG(edge.flags.forward, "edge must be oriented in forward direction");
BOOST_ASSERT_MSG(edge.source != edge.target, "loaded edges contain a loop");
BOOST_ASSERT_MSG(edge.source != prev_edge.source || edge.target != prev_edge.target,
"loaded edges contain a multi edge");
}
#endif
Log() << "Graph loaded ok and has " << edge_list.size() << " edges";
return number_of_edges;
}
inline EdgeID loadAnnotationData(storage::io::FileReader &file_reader,
std::vector<extractor::NodeBasedEdgeAnnotation> &metadata)
{
auto const meta_data_count = file_reader.ReadElementCount64();
metadata.resize(meta_data_count);
file_reader.ReadInto(metadata.data(), meta_data_count);
return meta_data_count;
}
}
}
#endif // GRAPH_LOADER_HPP