Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Move .osrm file to tar format

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-20 00:47:34 +00:00
parent 06f28ffd34
commit f407afa694
10 changed files with 91 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -700,7 +700,7 @@ target_link_libraries(osrm_customize ${CUSTOMIZER_LIBRARIES} osrm_update osrm_st
target_link_libraries(osrm_store ${STORAGE_LIBRARIES})
# BUILD_COMPONENTS
add_executable(osrm-components src/tools/components.cpp $<TARGET_OBJECTS:UTIL>)
add_executable(osrm-components src/tools/components.cpp $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
target_link_libraries(osrm-components ${TBB_LIBRARIES} ${BOOST_BASE_LIBRARIES} ${UTIL_LIBRARIES})
install(TARGETS osrm-components DESTINATION bin)

8
include/extractor/extraction_containers.hpp
View File

@ -7,7 +7,7 @@
#include "extractor/restriction.hpp"
#include "extractor/scripting_environment.hpp"
#include "storage/io.hpp"
#include "storage/tar_fwd.hpp"
namespace osrm
{
@ -27,9 +27,9 @@ class ExtractionContainers
void PrepareRestrictions();
void PrepareEdges(ScriptingEnvironment &scripting_environment);
void WriteNodes(storage::io::FileWriter &file_out) const;
void WriteEdges(storage::io::FileWriter &file_out) const;
void WriteMetadata(storage::io::FileWriter &file_out) const;
void WriteNodes(storage::tar::FileWriter &file_out) const;
void WriteEdges(storage::tar::FileWriter &file_out) const;
void WriteMetadata(storage::tar::FileWriter &file_out) const;
void WriteCharData(const std::string &file_name);
public:

39
include/extractor/files.hpp
View File

@ -4,6 +4,7 @@
#include "extractor/edge_based_edge.hpp"
#include "extractor/node_data_container.hpp"
#include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp"
#include "extractor/serialization.hpp"
#include "extractor/turn_lane_types.hpp"
@ -368,7 +369,7 @@ inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPen
// writes .osrm.restrictions
template <typename ConditionalRestrictionsT>
inline void writeConditionalRestrictions(const boost::filesystem::path &path,
const ConditionalRestrictionsT &conditional_restrictions)
const ConditionalRestrictionsT &conditional_restrictions)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
@ -378,13 +379,47 @@ inline void writeConditionalRestrictions(const boost::filesystem::path &path,
// read .osrm.restrictions
template <typename ConditionalRestrictionsT>
inline void readConditionalRestrictions(const boost::filesystem::path &path, ConditionalRestrictionsT &conditional_restrictions)
inline void readConditionalRestrictions(const boost::filesystem::path &path,
ConditionalRestrictionsT &conditional_restrictions)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
serialization::read(reader, "/common/conditional_restrictions", conditional_restrictions);
}
// reads .osrm file which is a temporary file of osrm-extract
template <typename BarrierOutIter, typename TrafficSignalsOutIter, typename PackedOSMIDsT>
void readRawNBGraph(const boost::filesystem::path &path,
BarrierOutIter barriers,
TrafficSignalsOutIter traffic_signals,
std::vector<util::Coordinate> &coordinates,
PackedOSMIDsT &osm_node_ids,
std::vector<extractor::NodeBasedEdge> &edge_list,
std::vector<extractor::NodeBasedEdgeAnnotation> &annotations)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
auto number_of_nodes = reader.ReadElementCount64("/extractor/nodes");
coordinates.resize(number_of_nodes);
osm_node_ids.reserve(number_of_nodes);
auto index = 0;
auto decode = [&](const auto &current_node) {
coordinates[index].lon = current_node.lon;
coordinates[index].lat = current_node.lat;
osm_node_ids.push_back(current_node.node_id);
index++;
};
reader.ReadStreaming<extractor::QueryNode>("/extractor/nodes", boost::make_function_output_iterator(decode));
reader.ReadStreaming<NodeID>("/extractor/barriers", barriers);
reader.ReadStreaming<NodeID>("/extractor/traffic_lights", traffic_signals);
storage::serialization::read(reader, "/extractor/edges", edge_list);
storage::serialization::read(reader, "/extractor/annotations", annotations);
}
}
}
}

127
include/util/graph_loader.hpp
View File

@ -1,127 +0,0 @@
#ifndef GRAPH_LOADER_HPP
#define GRAPH_LOADER_HPP
#include "extractor/node_based_edge.hpp"
#include "extractor/packed_osm_ids.hpp"
#include "extractor/query_node.hpp"
#include "extractor/restriction.hpp"
#include "storage/io.hpp"
#include "util/exception.hpp"
#include "util/fingerprint.hpp"
#include "util/log.hpp"
#include "util/packed_vector.hpp"
#include "util/typedefs.hpp"
#include <boost/assert.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/function_output_iterator.hpp>
#include <tbb/parallel_sort.h>
#include <cmath>
#include <fstream>
#include <ios>
#include <unordered_set>
#include <vector>
namespace osrm
{
namespace util
{
/**
* Reads the beginning of an .osrm file and produces:
* - barrier nodes
* - traffic lights
* - nodes indexed by their internal (non-osm) id
*/
template <typename BarrierOutIter, typename TrafficSignalsOutIter>
NodeID loadNodesFromFile(storage::io::FileReader &file_reader,
BarrierOutIter barriers,
TrafficSignalsOutIter traffic_signals,
std::vector<util::Coordinate> &coordinates,
extractor::PackedOSMIDs &osm_node_ids)
{
auto number_of_nodes = file_reader.ReadElementCount64();
Log() << "Importing number_of_nodes new = " << number_of_nodes << " nodes ";
coordinates.resize(number_of_nodes);
osm_node_ids.reserve(number_of_nodes);
auto index = 0;
auto decode = [&](const extractor::QueryNode &current_node) {
coordinates[index].lon = current_node.lon;
coordinates[index].lat = current_node.lat;
osm_node_ids.push_back(current_node.node_id);
index++;
};
file_reader.ReadStreaming<extractor::QueryNode>(boost::make_function_output_iterator(decode),
number_of_nodes);
auto num_barriers = file_reader.ReadElementCount64();
file_reader.ReadStreaming<NodeID>(barriers, num_barriers);
auto num_lights = file_reader.ReadElementCount64();
file_reader.ReadStreaming<NodeID>(traffic_signals, num_lights);
return number_of_nodes;
}
/**
* Reads a .osrm file and produces the edges.
*/
inline EdgeID loadEdgesFromFile(storage::io::FileReader &file_reader,
std::vector<extractor::NodeBasedEdge> &edge_list)
{
auto number_of_edges = file_reader.ReadElementCount64();
edge_list.resize(number_of_edges);
Log() << " and " << number_of_edges << " edges ";
file_reader.ReadInto(edge_list.data(), number_of_edges);
BOOST_ASSERT(edge_list.size() > 0);
#ifndef NDEBUG
Log() << "Validating loaded edges...";
tbb::parallel_sort(
edge_list.begin(),
edge_list.end(),
[](const extractor::NodeBasedEdge &lhs, const extractor::NodeBasedEdge &rhs) {
return (lhs.source < rhs.source) ||
(lhs.source == rhs.source && lhs.target < rhs.target);
});
for (auto i = 1u; i < edge_list.size(); ++i)
{
const auto &edge = edge_list[i];
const auto &prev_edge = edge_list[i - 1];
BOOST_ASSERT_MSG(edge.weight > 0, "loaded null weight");
BOOST_ASSERT_MSG(edge.flags.forward, "edge must be oriented in forward direction");
BOOST_ASSERT_MSG(edge.source != edge.target, "loaded edges contain a loop");
BOOST_ASSERT_MSG(edge.source != prev_edge.source || edge.target != prev_edge.target,
"loaded edges contain a multi edge");
}
#endif
Log() << "Graph loaded ok and has " << edge_list.size() << " edges";
return number_of_edges;
}
inline EdgeID loadAnnotationData(storage::io::FileReader &file_reader,
std::vector<extractor::NodeBasedEdgeAnnotation> &metadata)
{
auto const meta_data_count = file_reader.ReadElementCount64();
metadata.resize(meta_data_count);
file_reader.ReadInto(metadata.data(), meta_data_count);
return meta_data_count;
}
}
}
#endif // GRAPH_LOADER_HPP

1
src/contractor/contractor.cpp
View File

@ -19,7 +19,6 @@
#include "util/exception_utils.hpp"
#include "util/exclude_flag.hpp"
#include "util/filtered_graph.hpp"
#include "util/graph_loader.hpp"
#include "util/integer_range.hpp"
#include "util/log.hpp"
#include "util/static_graph.hpp"

78
src/extractor/extraction_containers.cpp
View File

@ -131,15 +131,15 @@ void ExtractionContainers::PrepareData(ScriptingEnvironment &scripting_environme
const std::string &osrm_path,
const std::string &name_file_name)
{
storage::io::FileWriter file_out(osrm_path, storage::io::FileWriter::GenerateFingerprint);
storage::tar::FileWriter writer(osrm_path, storage::tar::FileWriter::GenerateFingerprint);
PrepareNodes();
WriteNodes(file_out);
WriteNodes(writer);
PrepareEdges(scripting_environment);
all_nodes_list.clear(); // free all_nodes_list before allocation of normal_edges
all_nodes_list.shrink_to_fit();
WriteEdges(file_out);
WriteMetadata(file_out);
WriteEdges(writer);
WriteMetadata(writer);
/* Sort these so that searching is a bit faster later on */
{
@ -276,8 +276,8 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
{
if (edge_iterator->result.osm_source_id < node_iterator->node_id)
{
util::Log(logDEBUG) << "Found invalid node reference "
<< edge_iterator->result.source;
util::Log(logDEBUG)
<< "Found invalid node reference " << edge_iterator->result.source;
edge_iterator->result.source = SPECIAL_NODEID;
++edge_iterator;
continue;
@ -531,7 +531,7 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
}
}
void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const
void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) const
{
std::vector<NodeBasedEdge> normal_edges;
normal_edges.reserve(all_edges_list.size());
@ -558,8 +558,7 @@ void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const
throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF);
}
file_out.WriteElementCount64(normal_edges.size());
file_out.WriteFrom(normal_edges.data(), normal_edges.size());
storage::serialization::write(writer, "/extractor/edges", normal_edges);
TIMER_STOP(write_edges);
log << "ok, after " << TIMER_SEC(write_edges) << "s";
@ -567,31 +566,21 @@ void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const
}
}
void ExtractionContainers::WriteMetadata(storage::io::FileWriter &file_out) const
void ExtractionContainers::WriteMetadata(storage::tar::FileWriter &writer) const
{
util::UnbufferedLog log;
log << "Writing way meta-data ... " << std::flush;
TIMER_START(write_meta_data);
file_out.WriteElementCount64(all_edges_annotation_data_list.size());
file_out.WriteFrom(all_edges_annotation_data_list.data(),
all_edges_annotation_data_list.size());
storage::serialization::write(writer, "/extractor/annotations", all_edges_annotation_data_list);
TIMER_STOP(write_meta_data);
log << "ok, after " << TIMER_SEC(write_meta_data) << "s";
log << " -- Metadata contains << " << all_edges_annotation_data_list.size() << " entries.";
}
void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
void ExtractionContainers::WriteNodes(storage::tar::FileWriter &writer) const
{
{
// write dummy value, will be overwritten later
util::UnbufferedLog log;
log << "setting number of nodes ... " << std::flush;
file_out.WriteElementCount64(max_internal_node_id);
log << "ok";
}
{
util::UnbufferedLog log;
log << "Confirming/Writing used nodes ... ";
@ -599,28 +588,35 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
// identify all used nodes by a merging step of two sorted lists
auto node_iterator = all_nodes_list.begin();
auto node_id_iterator = used_node_id_list.begin();
const auto used_node_id_list_end = used_node_id_list.end();
const auto all_nodes_list_end = all_nodes_list.end();
while (node_id_iterator != used_node_id_list_end && node_iterator != all_nodes_list_end)
{
if (*node_id_iterator < node_iterator->node_id)
{
++node_id_iterator;
continue;
}
if (*node_id_iterator > node_iterator->node_id)
const auto encode = [&]() {
BOOST_ASSERT(node_id_iterator != used_node_id_list.end());
BOOST_ASSERT(node_iterator != all_nodes_list_end);
BOOST_ASSERT(*node_id_iterator >= node_iterator->node_id);
while (*node_id_iterator > node_iterator->node_id &&
node_iterator != all_nodes_list_end)
{
++node_iterator;
continue;
}
if (node_iterator == all_nodes_list_end || *node_id_iterator < node_iterator->node_id)
{
throw util::exception(
"Invalid OSM data: Referenced non-existing node with ID " +
std::to_string(static_cast<std::uint64_t>(*node_id_iterator)));
}
BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
file_out.WriteFrom((*node_iterator));
++node_id_iterator;
++node_iterator;
}
return *node_iterator++;
};
writer.WriteElementCount64("/extractor/nodes", used_node_id_list.size());
writer.WriteStreaming<QueryNode>(
"/extractor/nodes",
boost::make_function_input_iterator(encode, boost::infinite()),
used_node_id_list.size());
TIMER_STOP(write_nodes);
log << "ok, after " << TIMER_SEC(write_nodes) << "s";
}
@ -639,7 +635,7 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
internal_barrier_nodes.push_back(node_id);
}
}
storage::serialization::write(file_out, internal_barrier_nodes);
storage::serialization::write(writer, "/extractor/barriers", internal_barrier_nodes);
log << "ok, after " << TIMER_SEC(write_nodes) << "s";
}
@ -657,7 +653,8 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
internal_traffic_signals.push_back(node_id);
}
}
storage::serialization::write(file_out, internal_traffic_signals);
storage::serialization::write(
writer, "/extractor/traffic_lights", internal_traffic_signals);
log << "ok, after " << TIMER_SEC(write_nodes) << "s";
}
@ -1035,8 +1032,9 @@ void ExtractionContainers::PrepareRestrictions()
// translate the turn from one segment onto another into a node restriction (the ways can
// only
// be connected at a single location)
auto const get_node_restriction_from_OSM_ids = [&](
auto const from_id, auto const to_id, const OSMNodeID via_node) {
auto const get_node_restriction_from_OSM_ids = [&](auto const from_id,
auto const to_id,
const OSMNodeID via_node) {
auto const from_segment_itr = referenced_ways.find(from_id);
if (from_segment_itr->second.way_id != from_id)
{

1
src/extractor/extractor.cpp
View File

@ -23,7 +23,6 @@
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
#include "util/graph_loader.hpp"
#include "util/integer_range.hpp"
#include "util/log.hpp"
#include "util/name_table.hpp"

17
src/extractor/node_based_graph_factory.cpp
View File

@ -1,7 +1,7 @@
#include "extractor/node_based_graph_factory.hpp"
#include "extractor/graph_compressor.hpp"
#include "extractor/files.hpp"
#include "storage/io.hpp"
#include "util/graph_loader.hpp"
#include "util/log.hpp"
#include "util/timing_util.hpp"
@ -34,28 +34,19 @@ NodeBasedGraphFactory::NodeBasedGraphFactory(
// load the data serialised during the extraction run
void NodeBasedGraphFactory::LoadDataFromFile(const boost::filesystem::path &input_file)
{
// the extraction_containers serialise all data necessary to create the node-based graph into a
// single file, the *.osrm file. It contains nodes, basic information about which of these nodes
// are traffic signals/stop signs. It also contains Edges and purely annotative meta-data
storage::io::FileReader file_reader(input_file, storage::io::FileReader::VerifyFingerprint);
auto barriers_iter = inserter(barriers, end(barriers));
auto traffic_signals_iter = inserter(traffic_signals, end(traffic_signals));
const auto number_of_node_based_nodes = util::loadNodesFromFile(
file_reader, barriers_iter, traffic_signals_iter, coordinates, osm_node_ids);
std::vector<NodeBasedEdge> edge_list;
util::loadEdgesFromFile(file_reader, edge_list);
files::readRawNBGraph(input_file, barriers_iter, traffic_signals_iter, coordinates, osm_node_ids, edge_list, annotation_data);
const auto number_of_node_based_nodes = coordinates.size();
if (edge_list.empty())
{
throw util::exception("Node-based-graph (" + input_file.string() + ") contains no edges." +
SOURCE_REF);
}
util::loadAnnotationData(file_reader, annotation_data);
// at this point, the data isn't compressed, but since we update the graph in-place, we assign
// it here.
compressed_output_graph =

15
src/tools/components.cpp
View File

@ -1,9 +1,11 @@
#include "extractor/files.hpp"
#include "extractor/packed_osm_ids.hpp"
#include "extractor/tarjan_scc.hpp"
#include "util/coordinate.hpp"
#include "util/coordinate_calculation.hpp"
#include "util/dynamic_graph.hpp"
#include "util/fingerprint.hpp"
#include "util/graph_loader.hpp"
#include "util/log.hpp"
#include "util/static_graph.hpp"
#include "util/typedefs.hpp"
@ -36,16 +38,13 @@ std::size_t loadGraph(const std::string &path,
extractor::PackedOSMIDs &osm_node_ids,
std::vector<TarjanEdge> &graph_edge_list)
{
storage::io::FileReader file_reader(path, storage::io::FileReader::VerifyFingerprint);
std::vector<extractor::NodeBasedEdge> edge_list;
std::vector<extractor::NodeBasedEdgeAnnotation> annotation_data;
auto nop = boost::make_function_output_iterator([](auto) {});
const auto number_of_nodes =
util::loadNodesFromFile(file_reader, nop, nop, coordinate_list, osm_node_ids);
util::loadEdgesFromFile(file_reader, edge_list);
extractor::files::readRawNBGraph(
path, nop, nop, coordinate_list, osm_node_ids, edge_list, annotation_data);
// Building a node-based graph
for (const auto &input_edge : edge_list)
@ -66,7 +65,7 @@ std::size_t loadGraph(const std::string &path,
}
}
return number_of_nodes;
return osm_node_ids.size();
}
struct FeatureWriter

1
src/updater/updater.cpp
View File

@ -16,7 +16,6 @@
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
#include "util/for_each_pair.hpp"
#include "util/graph_loader.hpp"
#include "util/integer_range.hpp"
#include "util/log.hpp"
#include "util/opening_hours.hpp"