Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Port .edges file to tar

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-20 01:05:11 +00:00
parent f407afa694
commit a594008e57
4 changed files with 50 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/guidance/files.hpp
View File

@ -27,10 +27,10 @@ inline void readTurnData(const boost::filesystem::path &path,
std::is_same<guidance::TurnDataView, TurnDataT>::value || std::is_same<guidance::TurnDataView, TurnDataT>::value ||
std::is_same<guidance::TurnDataExternalContainer, TurnDataT>::value, std::is_same<guidance::TurnDataExternalContainer, TurnDataT>::value,
""); "");
const auto fingerprint = storage::io::FileReader::VerifyFingerprint; const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint}; storage::tar::FileReader reader{path, fingerprint};
serialization::read(reader, turn_data, connectivity_checksum); serialization::read(reader, "/common/turn_data", turn_data, connectivity_checksum);
} }
// writes .osrm.edges // writes .osrm.edges
@ -43,10 +43,10 @@ inline void writeTurnData(const boost::filesystem::path &path,
std::is_same<guidance::TurnDataView, TurnDataT>::value || std::is_same<guidance::TurnDataView, TurnDataT>::value ||
std::is_same<guidance::TurnDataExternalContainer, TurnDataT>::value, std::is_same<guidance::TurnDataExternalContainer, TurnDataT>::value,
""); "");
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint}; storage::tar::FileWriter writer{path, fingerprint};
serialization::write(writer, turn_data, connectivity_checksum); serialization::write(writer, "/common/turn_data", turn_data, connectivity_checksum);
} }
} }
} }

43
include/guidance/serialization.hpp
View File

@ -3,7 +3,7 @@
#include "guidance/turn_data_container.hpp" #include "guidance/turn_data_container.hpp"
#include "storage/io.hpp" #include "storage/tar.hpp"
#include "storage/serialization.hpp" #include "storage/serialization.hpp"
#include <boost/assert.hpp> #include <boost/assert.hpp>
@ -17,29 +17,42 @@ namespace serialization
// read/write for turn data file // read/write for turn data file
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, inline void read(storage::tar::FileReader &reader,
const std::string &name,
guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container, guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container,
std::uint32_t &connectivity_checksum) std::uint32_t &connectivity_checksum)
{ {
storage::serialization::read(reader, turn_data_container.turn_instructions); storage::serialization::read(
storage::serialization::read(reader, turn_data_container.lane_data_ids); reader, name + "/turn_instructions", turn_data_container.turn_instructions);
storage::serialization::read(reader, turn_data_container.entry_class_ids); storage::serialization::read(
storage::serialization::read(reader, turn_data_container.pre_turn_bearings); reader, name + "/lane_data_ids", turn_data_container.lane_data_ids);
storage::serialization::read(reader, turn_data_container.post_turn_bearings); storage::serialization::read(
reader.ReadInto(connectivity_checksum); reader, name + "/entry_class_ids", turn_data_container.entry_class_ids);
storage::serialization::read(
reader, name + "/pre_turn_bearings", turn_data_container.pre_turn_bearings);
storage::serialization::read(
reader, name + "/post_turn_bearings", turn_data_container.post_turn_bearings);
reader.ReadInto(name + "/connectivity_checksum", connectivity_checksum);
} }
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container, const guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container,
const std::uint32_t connectivity_checksum) const std::uint32_t connectivity_checksum)
{ {
storage::serialization::write(writer, turn_data_container.turn_instructions); storage::serialization::write(
storage::serialization::write(writer, turn_data_container.lane_data_ids); writer, name + "/turn_instructions", turn_data_container.turn_instructions);
storage::serialization::write(writer, turn_data_container.entry_class_ids); storage::serialization::write(
storage::serialization::write(writer, turn_data_container.pre_turn_bearings); writer, name + "/lane_data_ids", turn_data_container.lane_data_ids);
storage::serialization::write(writer, turn_data_container.post_turn_bearings); storage::serialization::write(
writer.WriteFrom(connectivity_checksum); writer, name + "/entry_class_ids", turn_data_container.entry_class_ids);
storage::serialization::write(
writer, name + "/pre_turn_bearings", turn_data_container.pre_turn_bearings);
storage::serialization::write(
writer, name + "/post_turn_bearings", turn_data_container.post_turn_bearings);
writer.WriteElementCount64(name + "/connectivity_checksum", 1);
writer.WriteFrom(name + "/connectivity_checksum", connectivity_checksum);
} }
} }
} }

14
include/guidance/turn_data_container.hpp
View File

@ -5,8 +5,8 @@
#include "guidance/turn_bearing.hpp" #include "guidance/turn_bearing.hpp"
#include "guidance/turn_instruction.hpp" #include "guidance/turn_instruction.hpp"
#include "storage/io_fwd.hpp"
#include "storage/shared_memory_ownership.hpp" #include "storage/shared_memory_ownership.hpp"
#include "storage/tar_fwd.hpp"
#include "util/vector_view.hpp" #include "util/vector_view.hpp"
@ -24,12 +24,14 @@ template <storage::Ownership Ownership> class TurnDataContainerImpl;
namespace serialization namespace serialization
{ {
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
void read(storage::io::FileReader &reader, void read(storage::tar::FileReader &reader,
const std::string &name,
detail::TurnDataContainerImpl<Ownership> &turn_data, detail::TurnDataContainerImpl<Ownership> &turn_data,
std::uint32_t &connectivity_checksum); std::uint32_t &connectivity_checksum);
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
void write(storage::io::FileWriter &writer, void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::TurnDataContainerImpl<Ownership> &turn_data, const detail::TurnDataContainerImpl<Ownership> &turn_data,
const std::uint32_t connectivity_checksum); const std::uint32_t connectivity_checksum);
} }
@ -97,10 +99,12 @@ template <storage::Ownership Ownership> class TurnDataContainerImpl
others.begin(), others.end(), [this](const TurnData &other) { push_back(other); }); others.begin(), others.end(), [this](const TurnData &other) { push_back(other); });
} }
friend void serialization::read<Ownership>(storage::io::FileReader &reader, friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
const std::string &name,
TurnDataContainerImpl &turn_data_container, TurnDataContainerImpl &turn_data_container,
std::uint32_t &connectivity_checksum); std::uint32_t &connectivity_checksum);
friend void serialization::write<Ownership>(storage::io::FileWriter &writer, friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
const std::string &name,
const TurnDataContainerImpl &turn_data_container, const TurnDataContainerImpl &turn_data_container,
const std::uint32_t connectivity_checksum); const std::uint32_t connectivity_checksum);

24
src/storage/storage.cpp
View File

@ -250,23 +250,6 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlock(DataLayout::NAME_CHAR_DATA, make_block<char>(name_file.GetSize())); layout.SetBlock(DataLayout::NAME_CHAR_DATA, make_block<char>(name_file.GetSize()));
} }
// Loading information for original edges
{
io::FileReader edges_file(config.GetPath(".osrm.edges"), io::FileReader::VerifyFingerprint);
const auto number_of_original_edges = edges_file.ReadElementCount64();
// note: settings this all to the same size is correct, we extract them from the same struct
layout.SetBlock(DataLayout::PRE_TURN_BEARING,
make_block<guidance::TurnBearing>(number_of_original_edges));
layout.SetBlock(DataLayout::POST_TURN_BEARING,
make_block<guidance::TurnBearing>(number_of_original_edges));
layout.SetBlock(DataLayout::TURN_INSTRUCTION,
make_block<guidance::TurnInstruction>(number_of_original_edges));
layout.SetBlock(DataLayout::LANE_DATA_ID, make_block<LaneDataID>(number_of_original_edges));
layout.SetBlock(DataLayout::ENTRY_CLASSID,
make_block<EntryClassID>(number_of_original_edges));
}
// load rsearch tree size // load rsearch tree size
{ {
io::FileReader tree_node_file(config.GetPath(".osrm.ramIndex"), io::FileReader tree_node_file(config.GetPath(".osrm.ramIndex"),
@ -356,6 +339,12 @@ void Storage::PopulateLayout(DataLayout &layout)
{"/common/maneuver_overrides/node_sequences", DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES}, {"/common/maneuver_overrides/node_sequences", DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES},
{"/common/turn_penalty/weight", DataLayout::TURN_WEIGHT_PENALTIES}, {"/common/turn_penalty/weight", DataLayout::TURN_WEIGHT_PENALTIES},
{"/common/turn_penalty/duration", DataLayout::TURN_DURATION_PENALTIES}, {"/common/turn_penalty/duration", DataLayout::TURN_DURATION_PENALTIES},
{"/common/turn_data/pre_turn_bearings", DataLayout::PRE_TURN_BEARING},
{"/common/turn_data/post_turn_bearings", DataLayout::POST_TURN_BEARING},
{"/common/turn_data/turn_instructions", DataLayout::TURN_INSTRUCTION},
{"/common/turn_data/lane_data_ids", DataLayout::LANE_DATA_ID},
{"/common/turn_data/entry_class_ids", DataLayout::ENTRY_CLASSID},
{"/common/turn_data/connectivity_checksum", DataLayout::IGNORE_BLOCK},
}; };
std::vector<NamedBlock> blocks; std::vector<NamedBlock> blocks;
@ -378,6 +367,7 @@ void Storage::PopulateLayout(DataLayout &layout)
{REQUIRED, config.GetPath(".osrm.maneuver_overrides")}, {REQUIRED, config.GetPath(".osrm.maneuver_overrides")},
{REQUIRED, config.GetPath(".osrm.turn_weight_penalties")}, {REQUIRED, config.GetPath(".osrm.turn_weight_penalties")},
{REQUIRED, config.GetPath(".osrm.turn_duration_penalties")}, {REQUIRED, config.GetPath(".osrm.turn_duration_penalties")},
{REQUIRED, config.GetPath(".osrm.edges")},
}; };
for (const auto &file : tar_files) for (const auto &file : tar_files)