From f407afa69470ac437aaa07839e1ba472cdbb2542 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 20 Mar 2018 00:47:34 +0000 Subject: [PATCH] Move .osrm file to tar format --- CMakeLists.txt | 2 +- include/extractor/extraction_containers.hpp | 8 +- include/extractor/files.hpp | 39 +++++- include/util/graph_loader.hpp | 127 -------------------- src/contractor/contractor.cpp | 1 - src/extractor/extraction_containers.cpp | 78 ++++++------ src/extractor/extractor.cpp | 1 - src/extractor/node_based_graph_factory.cpp | 17 +-- src/tools/components.cpp | 15 ++- src/updater/updater.cpp | 1 - 10 files changed, 91 insertions(+), 198 deletions(-) delete mode 100644 include/util/graph_loader.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a85b681e..3e6e40828 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -700,7 +700,7 @@ target_link_libraries(osrm_customize ${CUSTOMIZER_LIBRARIES} osrm_update osrm_st target_link_libraries(osrm_store ${STORAGE_LIBRARIES}) # BUILD_COMPONENTS -add_executable(osrm-components src/tools/components.cpp $) +add_executable(osrm-components src/tools/components.cpp $ $) target_link_libraries(osrm-components ${TBB_LIBRARIES} ${BOOST_BASE_LIBRARIES} ${UTIL_LIBRARIES}) install(TARGETS osrm-components DESTINATION bin) diff --git a/include/extractor/extraction_containers.hpp b/include/extractor/extraction_containers.hpp index 7b5ae88ac..184db7739 100644 --- a/include/extractor/extraction_containers.hpp +++ b/include/extractor/extraction_containers.hpp @@ -7,7 +7,7 @@ #include "extractor/restriction.hpp" #include "extractor/scripting_environment.hpp" -#include "storage/io.hpp" +#include "storage/tar_fwd.hpp" namespace osrm { @@ -27,9 +27,9 @@ class ExtractionContainers void PrepareRestrictions(); void PrepareEdges(ScriptingEnvironment &scripting_environment); - void WriteNodes(storage::io::FileWriter &file_out) const; - void WriteEdges(storage::io::FileWriter &file_out) const; - void WriteMetadata(storage::io::FileWriter &file_out) const; + void WriteNodes(storage::tar::FileWriter &file_out) const; + void WriteEdges(storage::tar::FileWriter &file_out) const; + void WriteMetadata(storage::tar::FileWriter &file_out) const; void WriteCharData(const std::string &file_name); public: diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 94f85b61d..598215d32 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -4,6 +4,7 @@ #include "extractor/edge_based_edge.hpp" #include "extractor/node_data_container.hpp" #include "extractor/profile_properties.hpp" +#include "extractor/query_node.hpp" #include "extractor/serialization.hpp" #include "extractor/turn_lane_types.hpp" @@ -368,7 +369,7 @@ inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPen // writes .osrm.restrictions template inline void writeConditionalRestrictions(const boost::filesystem::path &path, - const ConditionalRestrictionsT &conditional_restrictions) + const ConditionalRestrictionsT &conditional_restrictions) { const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; storage::tar::FileWriter writer{path, fingerprint}; @@ -378,13 +379,47 @@ inline void writeConditionalRestrictions(const boost::filesystem::path &path, // read .osrm.restrictions template -inline void readConditionalRestrictions(const boost::filesystem::path &path, ConditionalRestrictionsT &conditional_restrictions) +inline void readConditionalRestrictions(const boost::filesystem::path &path, + ConditionalRestrictionsT &conditional_restrictions) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; serialization::read(reader, "/common/conditional_restrictions", conditional_restrictions); } + +// reads .osrm file which is a temporary file of osrm-extract +template +void readRawNBGraph(const boost::filesystem::path &path, + BarrierOutIter barriers, + TrafficSignalsOutIter traffic_signals, + std::vector &coordinates, + PackedOSMIDsT &osm_node_ids, + std::vector &edge_list, + std::vector &annotations) +{ + const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; + storage::tar::FileReader reader{path, fingerprint}; + + auto number_of_nodes = reader.ReadElementCount64("/extractor/nodes"); + coordinates.resize(number_of_nodes); + osm_node_ids.reserve(number_of_nodes); + auto index = 0; + auto decode = [&](const auto ¤t_node) { + coordinates[index].lon = current_node.lon; + coordinates[index].lat = current_node.lat; + osm_node_ids.push_back(current_node.node_id); + index++; + }; + reader.ReadStreaming("/extractor/nodes", boost::make_function_output_iterator(decode)); + + reader.ReadStreaming("/extractor/barriers", barriers); + + reader.ReadStreaming("/extractor/traffic_lights", traffic_signals); + + storage::serialization::read(reader, "/extractor/edges", edge_list); + storage::serialization::read(reader, "/extractor/annotations", annotations); +} } } } diff --git a/include/util/graph_loader.hpp b/include/util/graph_loader.hpp deleted file mode 100644 index 208c1e81f..000000000 --- a/include/util/graph_loader.hpp +++ /dev/null @@ -1,127 +0,0 @@ -#ifndef GRAPH_LOADER_HPP -#define GRAPH_LOADER_HPP - -#include "extractor/node_based_edge.hpp" -#include "extractor/packed_osm_ids.hpp" -#include "extractor/query_node.hpp" -#include "extractor/restriction.hpp" -#include "storage/io.hpp" -#include "util/exception.hpp" -#include "util/fingerprint.hpp" -#include "util/log.hpp" -#include "util/packed_vector.hpp" -#include "util/typedefs.hpp" - -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include -#include - -namespace osrm -{ -namespace util -{ - -/** - * Reads the beginning of an .osrm file and produces: - * - barrier nodes - * - traffic lights - * - nodes indexed by their internal (non-osm) id - */ -template -NodeID loadNodesFromFile(storage::io::FileReader &file_reader, - BarrierOutIter barriers, - TrafficSignalsOutIter traffic_signals, - std::vector &coordinates, - extractor::PackedOSMIDs &osm_node_ids) -{ - auto number_of_nodes = file_reader.ReadElementCount64(); - Log() << "Importing number_of_nodes new = " << number_of_nodes << " nodes "; - - coordinates.resize(number_of_nodes); - osm_node_ids.reserve(number_of_nodes); - - auto index = 0; - auto decode = [&](const extractor::QueryNode ¤t_node) { - coordinates[index].lon = current_node.lon; - coordinates[index].lat = current_node.lat; - osm_node_ids.push_back(current_node.node_id); - index++; - }; - - file_reader.ReadStreaming(boost::make_function_output_iterator(decode), - number_of_nodes); - - auto num_barriers = file_reader.ReadElementCount64(); - file_reader.ReadStreaming(barriers, num_barriers); - - auto num_lights = file_reader.ReadElementCount64(); - file_reader.ReadStreaming(traffic_signals, num_lights); - - return number_of_nodes; -} - -/** - * Reads a .osrm file and produces the edges. - */ -inline EdgeID loadEdgesFromFile(storage::io::FileReader &file_reader, - std::vector &edge_list) -{ - auto number_of_edges = file_reader.ReadElementCount64(); - - edge_list.resize(number_of_edges); - Log() << " and " << number_of_edges << " edges "; - - file_reader.ReadInto(edge_list.data(), number_of_edges); - - BOOST_ASSERT(edge_list.size() > 0); - -#ifndef NDEBUG - Log() << "Validating loaded edges..."; - tbb::parallel_sort( - edge_list.begin(), - edge_list.end(), - [](const extractor::NodeBasedEdge &lhs, const extractor::NodeBasedEdge &rhs) { - return (lhs.source < rhs.source) || - (lhs.source == rhs.source && lhs.target < rhs.target); - }); - for (auto i = 1u; i < edge_list.size(); ++i) - { - const auto &edge = edge_list[i]; - const auto &prev_edge = edge_list[i - 1]; - - BOOST_ASSERT_MSG(edge.weight > 0, "loaded null weight"); - BOOST_ASSERT_MSG(edge.flags.forward, "edge must be oriented in forward direction"); - - BOOST_ASSERT_MSG(edge.source != edge.target, "loaded edges contain a loop"); - BOOST_ASSERT_MSG(edge.source != prev_edge.source || edge.target != prev_edge.target, - "loaded edges contain a multi edge"); - } -#endif - - Log() << "Graph loaded ok and has " << edge_list.size() << " edges"; - - return number_of_edges; -} - -inline EdgeID loadAnnotationData(storage::io::FileReader &file_reader, - std::vector &metadata) -{ - auto const meta_data_count = file_reader.ReadElementCount64(); - metadata.resize(meta_data_count); - file_reader.ReadInto(metadata.data(), meta_data_count); - return meta_data_count; -} -} -} - -#endif // GRAPH_LOADER_HPP diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp index c14926ecf..3aa874d39 100644 --- a/src/contractor/contractor.cpp +++ b/src/contractor/contractor.cpp @@ -19,7 +19,6 @@ #include "util/exception_utils.hpp" #include "util/exclude_flag.hpp" #include "util/filtered_graph.hpp" -#include "util/graph_loader.hpp" #include "util/integer_range.hpp" #include "util/log.hpp" #include "util/static_graph.hpp" diff --git a/src/extractor/extraction_containers.cpp b/src/extractor/extraction_containers.cpp index 8390347fd..6336d9890 100644 --- a/src/extractor/extraction_containers.cpp +++ b/src/extractor/extraction_containers.cpp @@ -131,15 +131,15 @@ void ExtractionContainers::PrepareData(ScriptingEnvironment &scripting_environme const std::string &osrm_path, const std::string &name_file_name) { - storage::io::FileWriter file_out(osrm_path, storage::io::FileWriter::GenerateFingerprint); + storage::tar::FileWriter writer(osrm_path, storage::tar::FileWriter::GenerateFingerprint); PrepareNodes(); - WriteNodes(file_out); + WriteNodes(writer); PrepareEdges(scripting_environment); all_nodes_list.clear(); // free all_nodes_list before allocation of normal_edges all_nodes_list.shrink_to_fit(); - WriteEdges(file_out); - WriteMetadata(file_out); + WriteEdges(writer); + WriteMetadata(writer); /* Sort these so that searching is a bit faster later on */ { @@ -276,8 +276,8 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm { if (edge_iterator->result.osm_source_id < node_iterator->node_id) { - util::Log(logDEBUG) << "Found invalid node reference " - << edge_iterator->result.source; + util::Log(logDEBUG) + << "Found invalid node reference " << edge_iterator->result.source; edge_iterator->result.source = SPECIAL_NODEID; ++edge_iterator; continue; @@ -531,7 +531,7 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm } } -void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const +void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) const { std::vector normal_edges; normal_edges.reserve(all_edges_list.size()); @@ -558,8 +558,7 @@ void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF); } - file_out.WriteElementCount64(normal_edges.size()); - file_out.WriteFrom(normal_edges.data(), normal_edges.size()); + storage::serialization::write(writer, "/extractor/edges", normal_edges); TIMER_STOP(write_edges); log << "ok, after " << TIMER_SEC(write_edges) << "s"; @@ -567,31 +566,21 @@ void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const } } -void ExtractionContainers::WriteMetadata(storage::io::FileWriter &file_out) const +void ExtractionContainers::WriteMetadata(storage::tar::FileWriter &writer) const { util::UnbufferedLog log; log << "Writing way meta-data ... " << std::flush; TIMER_START(write_meta_data); - file_out.WriteElementCount64(all_edges_annotation_data_list.size()); - file_out.WriteFrom(all_edges_annotation_data_list.data(), - all_edges_annotation_data_list.size()); + storage::serialization::write(writer, "/extractor/annotations", all_edges_annotation_data_list); TIMER_STOP(write_meta_data); log << "ok, after " << TIMER_SEC(write_meta_data) << "s"; log << " -- Metadata contains << " << all_edges_annotation_data_list.size() << " entries."; } -void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const +void ExtractionContainers::WriteNodes(storage::tar::FileWriter &writer) const { - { - // write dummy value, will be overwritten later - util::UnbufferedLog log; - log << "setting number of nodes ... " << std::flush; - file_out.WriteElementCount64(max_internal_node_id); - log << "ok"; - } - { util::UnbufferedLog log; log << "Confirming/Writing used nodes ... "; @@ -599,28 +588,35 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const // identify all used nodes by a merging step of two sorted lists auto node_iterator = all_nodes_list.begin(); auto node_id_iterator = used_node_id_list.begin(); - const auto used_node_id_list_end = used_node_id_list.end(); const auto all_nodes_list_end = all_nodes_list.end(); - while (node_id_iterator != used_node_id_list_end && node_iterator != all_nodes_list_end) - { - if (*node_id_iterator < node_iterator->node_id) - { - ++node_id_iterator; - continue; - } - if (*node_id_iterator > node_iterator->node_id) + const auto encode = [&]() { + BOOST_ASSERT(node_id_iterator != used_node_id_list.end()); + BOOST_ASSERT(node_iterator != all_nodes_list_end); + BOOST_ASSERT(*node_id_iterator >= node_iterator->node_id); + while (*node_id_iterator > node_iterator->node_id && + node_iterator != all_nodes_list_end) { ++node_iterator; - continue; + } + if (node_iterator == all_nodes_list_end || *node_id_iterator < node_iterator->node_id) + { + throw util::exception( + "Invalid OSM data: Referenced non-existing node with ID " + + std::to_string(static_cast(*node_id_iterator))); } BOOST_ASSERT(*node_id_iterator == node_iterator->node_id); - file_out.WriteFrom((*node_iterator)); - ++node_id_iterator; - ++node_iterator; - } + return *node_iterator++; + }; + + writer.WriteElementCount64("/extractor/nodes", used_node_id_list.size()); + writer.WriteStreaming( + "/extractor/nodes", + boost::make_function_input_iterator(encode, boost::infinite()), + used_node_id_list.size()); + TIMER_STOP(write_nodes); log << "ok, after " << TIMER_SEC(write_nodes) << "s"; } @@ -639,7 +635,7 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const internal_barrier_nodes.push_back(node_id); } } - storage::serialization::write(file_out, internal_barrier_nodes); + storage::serialization::write(writer, "/extractor/barriers", internal_barrier_nodes); log << "ok, after " << TIMER_SEC(write_nodes) << "s"; } @@ -657,7 +653,8 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const internal_traffic_signals.push_back(node_id); } } - storage::serialization::write(file_out, internal_traffic_signals); + storage::serialization::write( + writer, "/extractor/traffic_lights", internal_traffic_signals); log << "ok, after " << TIMER_SEC(write_nodes) << "s"; } @@ -1035,8 +1032,9 @@ void ExtractionContainers::PrepareRestrictions() // translate the turn from one segment onto another into a node restriction (the ways can // only // be connected at a single location) - auto const get_node_restriction_from_OSM_ids = [&]( - auto const from_id, auto const to_id, const OSMNodeID via_node) { + auto const get_node_restriction_from_OSM_ids = [&](auto const from_id, + auto const to_id, + const OSMNodeID via_node) { auto const from_segment_itr = referenced_ways.find(from_id); if (from_segment_itr->second.way_id != from_id) { diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 377666d83..fe0d02e7f 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -23,7 +23,6 @@ #include "util/exception.hpp" #include "util/exception_utils.hpp" -#include "util/graph_loader.hpp" #include "util/integer_range.hpp" #include "util/log.hpp" #include "util/name_table.hpp" diff --git a/src/extractor/node_based_graph_factory.cpp b/src/extractor/node_based_graph_factory.cpp index aef2d4491..d89239850 100644 --- a/src/extractor/node_based_graph_factory.cpp +++ b/src/extractor/node_based_graph_factory.cpp @@ -1,7 +1,7 @@ #include "extractor/node_based_graph_factory.hpp" #include "extractor/graph_compressor.hpp" +#include "extractor/files.hpp" #include "storage/io.hpp" -#include "util/graph_loader.hpp" #include "util/log.hpp" #include "util/timing_util.hpp" @@ -34,28 +34,19 @@ NodeBasedGraphFactory::NodeBasedGraphFactory( // load the data serialised during the extraction run void NodeBasedGraphFactory::LoadDataFromFile(const boost::filesystem::path &input_file) { - // the extraction_containers serialise all data necessary to create the node-based graph into a - // single file, the *.osrm file. It contains nodes, basic information about which of these nodes - // are traffic signals/stop signs. It also contains Edges and purely annotative meta-data - storage::io::FileReader file_reader(input_file, storage::io::FileReader::VerifyFingerprint); - auto barriers_iter = inserter(barriers, end(barriers)); auto traffic_signals_iter = inserter(traffic_signals, end(traffic_signals)); - - const auto number_of_node_based_nodes = util::loadNodesFromFile( - file_reader, barriers_iter, traffic_signals_iter, coordinates, osm_node_ids); - std::vector edge_list; - util::loadEdgesFromFile(file_reader, edge_list); + files::readRawNBGraph(input_file, barriers_iter, traffic_signals_iter, coordinates, osm_node_ids, edge_list, annotation_data); + + const auto number_of_node_based_nodes = coordinates.size(); if (edge_list.empty()) { throw util::exception("Node-based-graph (" + input_file.string() + ") contains no edges." + SOURCE_REF); } - util::loadAnnotationData(file_reader, annotation_data); - // at this point, the data isn't compressed, but since we update the graph in-place, we assign // it here. compressed_output_graph = diff --git a/src/tools/components.cpp b/src/tools/components.cpp index 5a7c2ca62..056edb460 100644 --- a/src/tools/components.cpp +++ b/src/tools/components.cpp @@ -1,9 +1,11 @@ +#include "extractor/files.hpp" +#include "extractor/packed_osm_ids.hpp" #include "extractor/tarjan_scc.hpp" + #include "util/coordinate.hpp" #include "util/coordinate_calculation.hpp" #include "util/dynamic_graph.hpp" #include "util/fingerprint.hpp" -#include "util/graph_loader.hpp" #include "util/log.hpp" #include "util/static_graph.hpp" #include "util/typedefs.hpp" @@ -36,16 +38,13 @@ std::size_t loadGraph(const std::string &path, extractor::PackedOSMIDs &osm_node_ids, std::vector &graph_edge_list) { - storage::io::FileReader file_reader(path, storage::io::FileReader::VerifyFingerprint); - std::vector edge_list; + std::vector annotation_data; auto nop = boost::make_function_output_iterator([](auto) {}); - const auto number_of_nodes = - util::loadNodesFromFile(file_reader, nop, nop, coordinate_list, osm_node_ids); - - util::loadEdgesFromFile(file_reader, edge_list); + extractor::files::readRawNBGraph( + path, nop, nop, coordinate_list, osm_node_ids, edge_list, annotation_data); // Building a node-based graph for (const auto &input_edge : edge_list) @@ -66,7 +65,7 @@ std::size_t loadGraph(const std::string &path, } } - return number_of_nodes; + return osm_node_ids.size(); } struct FeatureWriter diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index 78672a5ca..ffebb7fcf 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -16,7 +16,6 @@ #include "util/exception.hpp" #include "util/exception_utils.hpp" #include "util/for_each_pair.hpp" -#include "util/graph_loader.hpp" #include "util/integer_range.hpp" #include "util/log.hpp" #include "util/opening_hours.hpp"