Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Separate serialization and files in extractor

Browse Source
This commit is contained in:
Patrick Niklaus 2017-04-01 18:44:49 +00:00 committed by Patrick Niklaus
parent 603e2ee7de
commit 08d62cd5e3
8 changed files with 112 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
include/extractor/files.hpp Normal file
View File

@ -0,0 +1,72 @@
#ifndef OSRM_EXTRACTOR_FILES_HPP
#define OSRM_EXTRACTOR_FILES_HPP
#include "extractor/seralization.hpp"
#include <boost/assert.hpp>
namespace osrm
{
namespace extractor
{
namespace files
{
// reads .osrm.cnbg_to_ebg
inline void readNBGMapping(const boost::filesystem::path &path, std::vector<NBGToEBG> &mapping)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
serialization::read(reader, mapping);
}
// writes .osrm.cnbg_to_ebg
inline void writeNBGMapping(const boost::filesystem::path &path, const std::vector<NBGToEBG> &mapping)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
serialization::write(writer, mapping);
}
// reads .osrm.datasource_names
inline void readDatasources(const boost::filesystem::path &path, Datasources &sources)
{
const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
storage::io::FileReader reader{path, fingerprint};
serialization::read(reader, sources);
}
// writes .osrm.datasource_names
inline void writeDatasources(const boost::filesystem::path &path, Datasources &sources)
{
const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
storage::io::FileWriter writer{path, fingerprint};
serialization::write(writer, sources);
}
// reads .osrm.geometry
inline void readSegmentData(const boost::filesystem::path &path, SegmentDataContainer &segment_data)
{
const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
storage::io::FileReader reader{path, fingerprint};
serialization::read(reader, segment_data);
}
// writes .osrm.geometry
inline void writeSegmentData(const boost::filesystem::path &path, const SegmentDataContainer &segment_data)
{
const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
storage::io::FileWriter writer{path, fingerprint};
serialization::write(writer, segment_data);
}
}
}
}
#endif

20
include/extractor/segment_data_container.hpp
View File

@ -17,6 +17,15 @@
namespace osrm
{
namespace storage
{
namespace io
{
class FileReader;
class FileWriter;
}
}
namespace extractor
{
@ -27,13 +36,13 @@ namespace detail
template <storage::Ownership Ownership> class SegmentDataContainerImpl;
}
namespace io
namespace serialization
{
template <storage::Ownership Ownership>
inline void read(const boost::filesystem::path &path,
inline void read(storage::io::FileReader &reader,
detail::SegmentDataContainerImpl<Ownership> &segment_data);
template <storage::Ownership Ownership>
inline void write(const boost::filesystem::path &path,
inline void write(storage::io::FileWriter &writer,
const detail::SegmentDataContainerImpl<Ownership> &segment_data);
}
@ -190,10 +199,11 @@ template <storage::Ownership Ownership> class SegmentDataContainerImpl
auto GetNumberOfGeometries() const { return index.size() - 1; }
auto GetNumberOfSegments() const { return fwd_weights.size(); }
friend void io::read<Ownership>(const boost::filesystem::path &path,
friend void
serialization::read<Ownership>(storage::io::FileReader &reader,
detail::SegmentDataContainerImpl<Ownership> &segment_data);
friend void
io::write<Ownership>(const boost::filesystem::path &path,
serialization::write<Ownership>(storage::io::FileWriter &writer,
const detail::SegmentDataContainerImpl<Ownership> &segment_data);
private:

34
include/extractor/io.hpp → include/extractor/seralization.hpp
View File

@ -13,47 +13,32 @@ namespace osrm
{
namespace extractor
{
namespace io
namespace serialization
{
inline void read(const boost::filesystem::path &path, std::vector<NBGToEBG> &mapping)
inline void read(storage::io::FileReader &reader, std::vector<NBGToEBG> &mapping)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(mapping);
}
inline void write(const boost::filesystem::path &path, const std::vector<NBGToEBG> &mapping)
inline void write(storage::io::FileWriter &writer, const std::vector<NBGToEBG> &mapping)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter reader{path, fingerprint};
reader.SerializeVector(mapping);
writer.SerializeVector(mapping);
}
inline void read(const boost::filesystem::path &path, Datasources &sources)
inline void read(storage::io::FileReader &reader, Datasources &sources)
{
const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.ReadInto(sources);
}
inline void write(const boost::filesystem::path &path, Datasources &sources)
inline void write(storage::io::FileWriter &writer, Datasources &sources)
{
const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.WriteFrom(sources);
}
template <>
inline void read(const boost::filesystem::path &path, SegmentDataContainer &segment_data)
inline void read(storage::io::FileReader &reader, SegmentDataContainer &segment_data)
{
const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
storage::io::FileReader reader{path, fingerprint};
auto num_indices = reader.ReadElementCount32();
segment_data.index.resize(num_indices);
reader.ReadInto(segment_data.index.data(), num_indices);
@ -75,11 +60,8 @@ inline void read(const boost::filesystem::path &path, SegmentDataContainer &segm
}
template <>
inline void write(const boost::filesystem::path &path, const SegmentDataContainer &segment_data)
inline void write(storage::io::FileWriter &writer, const SegmentDataContainer &segment_data)
{
const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.WriteElementCount32(segment_data.index.size());
writer.WriteFrom(segment_data.index);

4
src/extractor/edge_based_graph_factory.cpp
View File

@ -2,7 +2,7 @@
#include "extractor/edge_based_edge.hpp"
#include "extractor/guidance/turn_analysis.hpp"
#include "extractor/guidance/turn_lane_handler.hpp"
#include "extractor/io.hpp"
#include "extractor/files.hpp"
#include "extractor/scripting_environment.hpp"
#include "extractor/suffix_table.hpp"
@ -205,7 +205,7 @@ void EdgeBasedGraphFactory::Run(ScriptingEnvironment &scripting_environment,
m_edge_based_node_weights.reserve(m_max_edge_id + 1);
{
auto mapping = GenerateEdgeExpandedNodes();
io::write(cnbg_ebg_mapping_path, mapping);
files::writeNBGMapping(cnbg_ebg_mapping_path, mapping);
}
TIMER_STOP(generate_nodes);

4
src/extractor/extractor.cpp
View File

@ -5,7 +5,7 @@
#include "extractor/extraction_node.hpp"
#include "extractor/extraction_way.hpp"
#include "extractor/extractor_callbacks.hpp"
#include "extractor/io.hpp"
#include "extractor/files.hpp"
#include "extractor/raster_source.hpp"
#include "extractor/restriction_parser.hpp"
#include "extractor/scripting_environment.hpp"
@ -512,7 +512,7 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
});
WriteTurnLaneData(config.turn_lane_descriptions_file_name);
io::write(config.geometry_output_path, *compressed_edge_container.ToSegmentData());
files::writeSegmentData(config.geometry_output_path, *compressed_edge_container.ToSegmentData());
edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list);
edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list);

4
src/partition/partitioner.cpp
View File

@ -9,7 +9,7 @@
#include "partition/recursive_bisection.hpp"
#include "partition/remove_unconnected.hpp"
#include "extractor/io.hpp"
#include "extractor/files.hpp"
#include "util/coordinate.hpp"
#include "util/geojson_debug_logger.hpp"
@ -126,7 +126,7 @@ int Partitioner::Run(const PartitionConfig &config)
// For details see #3205
std::vector<extractor::NBGToEBG> mapping;
extractor::io::read(config.cnbg_ebg_mapping_path.string(), mapping);
extractor::files::readNBGMapping(config.cnbg_ebg_mapping_path.string(), mapping);
util::Log() << "Loaded node based graph to edge based graph mapping";
auto edge_based_graph = LoadEdgeBasedGraph(config.edge_based_graph_path.string());

4
src/storage/storage.cpp
View File

@ -4,7 +4,7 @@
#include "extractor/compressed_edge_container.hpp"
#include "extractor/edge_based_edge.hpp"
#include "extractor/guidance/turn_instruction.hpp"
#include "extractor/io.hpp"
#include "extractor/files.hpp"
#include "extractor/original_edge_data.hpp"
#include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp"
@ -686,7 +686,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
{
const auto datasources_names_ptr = layout.GetBlockPtr<extractor::Datasources, true>(
memory_ptr, DataLayout::DATASOURCES_NAMES);
extractor::io::read(config.datasource_names_path, *datasources_names_ptr);
extractor::files::readDatasources(config.datasource_names_path, *datasources_names_ptr);
}
// Loading list of coordinates

12
src/updater/updater.cpp
View File

@ -2,12 +2,14 @@
#include "updater/csv_source.hpp"
#include "extractor/files.hpp"
#include "extractor/compressed_edge_container.hpp"
#include "extractor/edge_based_graph_factory.hpp"
#include "extractor/io.hpp"
#include "extractor/seralization.hpp"
#include "extractor/node_based_edge.hpp"
#include "storage/io.hpp"
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
#include "util/for_each_pair.hpp"
@ -85,7 +87,7 @@ void checkWeightsConsistency(
using OriginalEdgeData = osrm::extractor::OriginalEdgeData;
extractor::SegmentDataContainer segment_data;
extractor::io::read(config.geometry_path, segment_data);
extractor::files::readSegmentData(config.geometry_path, segment_data);
Reader edges_input_file(config.osrm_input_path.string() + ".edges", Reader::HasNoFingerprint);
std::vector<OriginalEdgeData> current_edge_data(edges_input_file.ReadElementCount64());
@ -361,7 +363,7 @@ void saveDatasourcesNames(const UpdaterConfig &config)
source++;
}
extractor::io::write(config.datasource_names_path, sources);
extractor::files::writeDatasources(config.datasource_names_path, sources);
}
std::vector<std::uint64_t>
@ -459,7 +461,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
if (update_edge_weights || update_turn_penalties)
{
const auto load_segment_data = [&] {
extractor::io::read(config.geometry_path, segment_data);
extractor::files::readSegmentData(config.geometry_path, segment_data);
};
const auto load_edge_data = [&] {
@ -503,7 +505,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
updated_segments =
updateSegmentData(config, profile_properties, segment_speed_lookup, segment_data);
// Now save out the updated compressed geometries
extractor::io::write(config.geometry_path, segment_data);
extractor::files::writeSegmentData(config.geometry_path, segment_data);
TIMER_STOP(segment);
util::Log() << "Updating segment data took " << TIMER_MSEC(segment) << "ms.";
}