From b1dfbce6758f1f289d9fe2631010238913c5865d Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 19 Mar 2018 14:09:25 +0000 Subject: [PATCH] Port .ebg_nodes to tar file --- include/extractor/files.hpp | 12 ++++++------ include/extractor/node_data_container.hpp | 14 +++++++++----- include/extractor/serialization.hpp | 19 ++++++++----------- src/storage/storage.cpp | 14 +++----------- 4 files changed, 26 insertions(+), 33 deletions(-) diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 9d19045a6..ccfafac10 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -210,10 +210,10 @@ inline void readNodeData(const boost::filesystem::path &path, NodeDataT &node_da std::is_same::value || std::is_same::value, ""); - const auto fingerprint = storage::io::FileReader::VerifyFingerprint; - storage::io::FileReader reader{path, fingerprint}; + const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; + storage::tar::FileReader reader{path, fingerprint}; - serialization::read(reader, node_data); + serialization::read(reader, "/common/ebg_node_data", node_data); } // writes .osrm.ebg_nodes @@ -224,10 +224,10 @@ inline void writeNodeData(const boost::filesystem::path &path, const NodeDataT & std::is_same::value || std::is_same::value, ""); - const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; - storage::io::FileWriter writer{path, fingerprint}; + const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; + storage::tar::FileWriter writer{path, fingerprint}; - serialization::write(writer, node_data); + serialization::write(writer, "/common/ebg_node_data", node_data); } // reads .osrm.tls diff --git a/include/extractor/node_data_container.hpp b/include/extractor/node_data_container.hpp index 45e0ebba9..e60dfa2e2 100644 --- a/include/extractor/node_data_container.hpp +++ b/include/extractor/node_data_container.hpp @@ -6,8 +6,8 @@ #include "extractor/node_based_edge.hpp" #include "extractor/travel_mode.hpp" -#include "storage/io_fwd.hpp" #include "storage/shared_memory_ownership.hpp" +#include "storage/tar_fwd.hpp" #include "util/permutation.hpp" #include "util/typedefs.hpp" @@ -29,11 +29,13 @@ template class EdgeBasedNodeDataContainerImpl; namespace serialization { template -void read(storage::io::FileReader &reader, +void read(storage::tar::FileReader &reader, + const std::string &name, detail::EdgeBasedNodeDataContainerImpl &ebn_data); template -void write(storage::io::FileWriter &writer, +void write(storage::tar::FileWriter &writer, + const std::string &name, const detail::EdgeBasedNodeDataContainerImpl &ebn_data); } @@ -89,10 +91,12 @@ template class EdgeBasedNodeDataContainerImpl return annotation_data[nodes[node_id].annotation_id].classes; } - friend void serialization::read(storage::io::FileReader &reader, + friend void serialization::read(storage::tar::FileReader &reader, + const std::string &name, EdgeBasedNodeDataContainerImpl &ebn_data_container); friend void - serialization::write(storage::io::FileWriter &writer, + serialization::write(storage::tar::FileWriter &writer, + const std::string &name, const EdgeBasedNodeDataContainerImpl &ebn_data_container); template > diff --git a/include/extractor/serialization.hpp b/include/extractor/serialization.hpp index 46ec155ab..32d643027 100644 --- a/include/extractor/serialization.hpp +++ b/include/extractor/serialization.hpp @@ -111,25 +111,22 @@ inline void write(storage::tar::FileWriter &writer, } template -inline void read(storage::io::FileReader &reader, +inline void read(storage::tar::FileReader &reader, + const std::string &name, detail::EdgeBasedNodeDataContainerImpl &node_data_container) { - // read header (separate sizes for both vectors) - reader.ReadElementCount64(); - reader.ReadElementCount64(); // read actual data - storage::serialization::read(reader, node_data_container.nodes); - storage::serialization::read(reader, node_data_container.annotation_data); + storage::serialization::read(reader, name + "/nodes", node_data_container.nodes); + storage::serialization::read(reader, name + "/annotations", node_data_container.annotation_data); } template -inline void write(storage::io::FileWriter &writer, +inline void write(storage::tar::FileWriter &writer, + const std::string& name, const detail::EdgeBasedNodeDataContainerImpl &node_data_container) { - writer.WriteElementCount64(node_data_container.NumberOfNodes()); - writer.WriteElementCount64(node_data_container.NumberOfAnnotations()); - storage::serialization::write(writer, node_data_container.nodes); - storage::serialization::write(writer, node_data_container.annotation_data); + storage::serialization::write(writer, name + "/nodes", node_data_container.nodes); + storage::serialization::write(writer, name + "/annotations", node_data_container.annotation_data); } inline void read(storage::io::FileReader &reader, NodeRestriction &restriction) diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 1bc1ecb7d..c6b68931a 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -278,17 +278,6 @@ void Storage::PopulateLayout(DataLayout &layout) make_block(number_of_original_edges)); } - { - io::FileReader nodes_data_file(config.GetPath(".osrm.ebg_nodes"), - io::FileReader::VerifyFingerprint); - const auto nodes_number = nodes_data_file.ReadElementCount64(); - const auto annotations_number = nodes_data_file.ReadElementCount64(); - layout.SetBlock(DataLayout::EDGE_BASED_NODE_DATA_LIST, - make_block(nodes_number)); - layout.SetBlock(DataLayout::ANNOTATION_DATA_LIST, - make_block(annotations_number)); - } - // load rsearch tree size { io::FileReader tree_node_file(config.GetPath(".osrm.ramIndex"), @@ -407,6 +396,8 @@ void Storage::PopulateLayout(DataLayout &layout) {"/common/segment_data/reverse_durations/packed", DataLayout::GEOMETRIES_REV_DURATION_LIST}, {"/common/segment_data/forward_data_sources", DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST}, {"/common/segment_data/reverse_data_sources", DataLayout::GEOMETRIES_REV_DATASOURCES_LIST}, + {"/common/ebg_node_data/nodes", DataLayout::EDGE_BASED_NODE_DATA_LIST}, + {"/common/ebg_node_data/annotations", DataLayout::ANNOTATION_DATA_LIST}, }; std::vector blocks; @@ -423,6 +414,7 @@ void Storage::PopulateLayout(DataLayout &layout) {REQUIRED, config.GetPath(".osrm.nbg_nodes")}, {REQUIRED, config.GetPath(".osrm.datasource_names")}, {REQUIRED, config.GetPath(".osrm.geometry")}, + {REQUIRED, config.GetPath(".osrm.ebg_nodes")}, }; for (const auto &file : tar_files)