Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Port .geometry file to tar format

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-19 13:26:00 +00:00
parent bced9a5a6d
commit da5aebaef3
4 changed files with 51 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/extractor/files.hpp
View File

@ -183,10 +183,10 @@ inline void readSegmentData(const boost::filesystem::path &path, SegmentDataT &s
static_assert(std::is_same<SegmentDataContainer, SegmentDataT>::value || static_assert(std::is_same<SegmentDataContainer, SegmentDataT>::value ||
std::is_same<SegmentDataView, SegmentDataT>::value, std::is_same<SegmentDataView, SegmentDataT>::value,
""); "");
const auto fingerprint = storage::io::FileReader::VerifyFingerprint; const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint}; storage::tar::FileReader reader{path, fingerprint};
serialization::read(reader, segment_data); serialization::read(reader, "/common/segment_data", segment_data);
} }
// writes .osrm.geometry // writes .osrm.geometry
@ -196,10 +196,10 @@ inline void writeSegmentData(const boost::filesystem::path &path, const SegmentD
static_assert(std::is_same<SegmentDataContainer, SegmentDataT>::value || static_assert(std::is_same<SegmentDataContainer, SegmentDataT>::value ||
std::is_same<SegmentDataView, SegmentDataT>::value, std::is_same<SegmentDataView, SegmentDataT>::value,
""); "");
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint}; storage::tar::FileWriter writer{path, fingerprint};
serialization::write(writer, segment_data); serialization::write(writer, "/common/segment_data", segment_data);
} }
// reads .osrm.ebg_nodes // reads .osrm.ebg_nodes

14
include/extractor/segment_data_container.hpp
View File

@ -5,8 +5,8 @@
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
#include "util/vector_view.hpp" #include "util/vector_view.hpp"
#include "storage/io_fwd.hpp"
#include "storage/shared_memory_ownership.hpp" #include "storage/shared_memory_ownership.hpp"
#include "storage/tar_fwd.hpp"
#include <boost/filesystem/path.hpp> #include <boost/filesystem/path.hpp>
#include <boost/range/adaptor/reversed.hpp> #include <boost/range/adaptor/reversed.hpp>
@ -32,10 +32,12 @@ template <storage::Ownership Ownership> class SegmentDataContainerImpl;
namespace serialization namespace serialization
{ {
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::SegmentDataContainerImpl<Ownership> &segment_data); detail::SegmentDataContainerImpl<Ownership> &segment_data);
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::SegmentDataContainerImpl<Ownership> &segment_data); const detail::SegmentDataContainerImpl<Ownership> &segment_data);
} }
@ -200,10 +202,12 @@ template <storage::Ownership Ownership> class SegmentDataContainerImpl
auto GetNumberOfSegments() const { return fwd_weights.size(); } auto GetNumberOfSegments() const { return fwd_weights.size(); }
friend void friend void
serialization::read<Ownership>(storage::io::FileReader &reader, serialization::read<Ownership>(storage::tar::FileReader &reader,
const std::string &name,
detail::SegmentDataContainerImpl<Ownership> &segment_data); detail::SegmentDataContainerImpl<Ownership> &segment_data);
friend void serialization::write<Ownership>( friend void serialization::write<Ownership>(
storage::io::FileWriter &writer, storage::tar::FileWriter &writer,
const std::string &name,
const detail::SegmentDataContainerImpl<Ownership> &segment_data); const detail::SegmentDataContainerImpl<Ownership> &segment_data);
private: private:

46
include/extractor/serialization.hpp
View File

@ -64,12 +64,12 @@ inline void write(storage::tar::FileWriter &writer,
} }
// read/write for datasources file // read/write for datasources file
inline void read(storage::tar::FileReader &reader, const std::string& name, Datasources &sources) inline void read(storage::tar::FileReader &reader, const std::string &name, Datasources &sources)
{ {
sources = reader.ReadOne<Datasources>(name); sources = reader.ReadOne<Datasources>(name);
} }
inline void write(storage::tar::FileWriter &writer, const std::string& name, Datasources &sources) inline void write(storage::tar::FileWriter &writer, const std::string &name, Datasources &sources)
{ {
writer.WriteElementCount64(name, 1); writer.WriteElementCount64(name, 1);
writer.WriteOne(name, sources); writer.WriteOne(name, sources);
@ -77,31 +77,37 @@ inline void write(storage::tar::FileWriter &writer, const std::string& name, Dat
// read/write for segment data file // read/write for segment data file
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::SegmentDataContainerImpl<Ownership> &segment_data) detail::SegmentDataContainerImpl<Ownership> &segment_data)
{ {
storage::serialization::read(reader, segment_data.index); storage::serialization::read(reader, name + "/index", segment_data.index);
storage::serialization::read(reader, segment_data.nodes); storage::serialization::read(reader, name + "/nodes", segment_data.nodes);
util::serialization::read(reader, segment_data.fwd_weights); util::serialization::read(reader, name + "/forward_weights", segment_data.fwd_weights);
util::serialization::read(reader, segment_data.rev_weights); util::serialization::read(reader, name + "/reverse_weights", segment_data.rev_weights);
util::serialization::read(reader, segment_data.fwd_durations); util::serialization::read(reader, name + "/forward_durations", segment_data.fwd_durations);
util::serialization::read(reader, segment_data.rev_durations); util::serialization::read(reader, name + "/reverse_durations", segment_data.rev_durations);
storage::serialization::read(reader, segment_data.fwd_datasources); storage::serialization::read(
storage::serialization::read(reader, segment_data.rev_datasources); reader, name + "/forward_data_sources", segment_data.fwd_datasources);
storage::serialization::read(
reader, name + "/reverse_data_sources", segment_data.rev_datasources);
} }
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::SegmentDataContainerImpl<Ownership> &segment_data) const detail::SegmentDataContainerImpl<Ownership> &segment_data)
{ {
storage::serialization::write(writer, segment_data.index); storage::serialization::write(writer, name + "/index", segment_data.index);
storage::serialization::write(writer, segment_data.nodes); storage::serialization::write(writer, name + "/nodes", segment_data.nodes);
util::serialization::write(writer, segment_data.fwd_weights); util::serialization::write(writer, name + "/forward_weights", segment_data.fwd_weights);
util::serialization::write(writer, segment_data.rev_weights); util::serialization::write(writer, name + "/reverse_weights", segment_data.rev_weights);
util::serialization::write(writer, segment_data.fwd_durations); util::serialization::write(writer, name + "/forward_durations", segment_data.fwd_durations);
util::serialization::write(writer, segment_data.rev_durations); util::serialization::write(writer, name + "/reverse_durations", segment_data.rev_durations);
storage::serialization::write(writer, segment_data.fwd_datasources); storage::serialization::write(
storage::serialization::write(writer, segment_data.rev_datasources); writer, name + "/forward_data_sources", segment_data.fwd_datasources);
storage::serialization::write(
writer, name + "/reverse_data_sources", segment_data.rev_datasources);
} }
template <storage::Ownership Ownership> template <storage::Ownership Ownership>

55
src/storage/storage.cpp
View File

@ -328,50 +328,6 @@ void Storage::PopulateLayout(DataLayout &layout)
make_block<TurnPenalty>(number_of_penalties)); make_block<TurnPenalty>(number_of_penalties));
} }
// load geometries sizes
{
io::FileReader reader(config.GetPath(".osrm.geometry"), io::FileReader::VerifyFingerprint);
const auto number_of_geometries_indices = reader.ReadVectorSize<unsigned>();
layout.SetBlock(DataLayout::GEOMETRIES_INDEX,
make_block<unsigned>(number_of_geometries_indices));
const auto number_of_compressed_geometries = reader.ReadVectorSize<NodeID>();
layout.SetBlock(DataLayout::GEOMETRIES_NODE_LIST,
make_block<NodeID>(number_of_compressed_geometries));
reader.ReadElementCount64(); // number of segments
const auto number_of_segment_weight_blocks =
reader.ReadVectorSize<extractor::SegmentDataView::SegmentWeightVector::block_type>();
reader.ReadElementCount64(); // number of segments
auto number_of_rev_weight_blocks =
reader.ReadVectorSize<extractor::SegmentDataView::SegmentWeightVector::block_type>();
BOOST_ASSERT(number_of_rev_weight_blocks == number_of_segment_weight_blocks);
(void)number_of_rev_weight_blocks;
reader.ReadElementCount64(); // number of segments
const auto number_of_segment_duration_blocks =
reader.ReadVectorSize<extractor::SegmentDataView::SegmentDurationVector::block_type>();
layout.SetBlock(DataLayout::GEOMETRIES_FWD_WEIGHT_LIST,
make_block<extractor::SegmentDataView::SegmentWeightVector::block_type>(
number_of_segment_weight_blocks));
layout.SetBlock(DataLayout::GEOMETRIES_REV_WEIGHT_LIST,
make_block<extractor::SegmentDataView::SegmentWeightVector::block_type>(
number_of_segment_weight_blocks));
layout.SetBlock(DataLayout::GEOMETRIES_FWD_DURATION_LIST,
make_block<extractor::SegmentDataView::SegmentDurationVector::block_type>(
number_of_segment_duration_blocks));
layout.SetBlock(DataLayout::GEOMETRIES_REV_DURATION_LIST,
make_block<extractor::SegmentDataView::SegmentDurationVector::block_type>(
number_of_segment_duration_blocks));
layout.SetBlock(DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST,
make_block<DatasourceID>(number_of_compressed_geometries));
layout.SetBlock(DataLayout::GEOMETRIES_REV_DATASOURCES_LIST,
make_block<DatasourceID>(number_of_compressed_geometries));
}
{ {
// Loading turn lane data // Loading turn lane data
io::FileReader lane_data_file(config.GetPath(".osrm.tld"), io::FileReader lane_data_file(config.GetPath(".osrm.tld"),
@ -443,6 +399,14 @@ void Storage::PopulateLayout(DataLayout &layout)
{"/common/coordinates", DataLayout::COORDINATE_LIST}, {"/common/coordinates", DataLayout::COORDINATE_LIST},
{"/common/osm_node_ids/packed", DataLayout::OSM_NODE_ID_LIST}, {"/common/osm_node_ids/packed", DataLayout::OSM_NODE_ID_LIST},
{"/common/data_sources_names", DataLayout::DATASOURCES_NAMES}, {"/common/data_sources_names", DataLayout::DATASOURCES_NAMES},
{"/common/segment_data/index", DataLayout::GEOMETRIES_INDEX},
{"/common/segment_data/nodes", DataLayout::GEOMETRIES_NODE_LIST},
{"/common/segment_data/forward_weights/packed", DataLayout::GEOMETRIES_FWD_WEIGHT_LIST},
{"/common/segment_data/reverse_weights/packed", DataLayout::GEOMETRIES_REV_WEIGHT_LIST},
{"/common/segment_data/forward_durations/packed", DataLayout::GEOMETRIES_FWD_DURATION_LIST},
{"/common/segment_data/reverse_durations/packed", DataLayout::GEOMETRIES_REV_DURATION_LIST},
{"/common/segment_data/forward_data_sources", DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST},
{"/common/segment_data/reverse_data_sources", DataLayout::GEOMETRIES_REV_DATASOURCES_LIST},
}; };
std::vector<NamedBlock> blocks; std::vector<NamedBlock> blocks;
@ -457,7 +421,8 @@ void Storage::PopulateLayout(DataLayout &layout)
{REQUIRED, config.GetPath(".osrm.icd")}, {REQUIRED, config.GetPath(".osrm.icd")},
{REQUIRED, config.GetPath(".osrm.properties")}, {REQUIRED, config.GetPath(".osrm.properties")},
{REQUIRED, config.GetPath(".osrm.nbg_nodes")}, {REQUIRED, config.GetPath(".osrm.nbg_nodes")},
{REQUIRED, config.GetPath(".osrm.datasource_names")} {REQUIRED, config.GetPath(".osrm.datasource_names")},
{REQUIRED, config.GetPath(".osrm.geometry")},
}; };
for (const auto &file : tar_files) for (const auto &file : tar_files)