Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Swtich EBG to tar file

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-17 01:37:02 +00:00
parent c61198b26b
commit cf5f6be472
5 changed files with 52 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
include/extractor/files.hpp
View File

@ -83,11 +83,13 @@ void writeEdgeBasedGraph(const boost::filesystem::path &path,
{ {
static_assert(std::is_same<typename EdgeBasedEdgeVector::value_type, EdgeBasedEdge>::value, ""); static_assert(std::is_same<typename EdgeBasedEdgeVector::value_type, EdgeBasedEdge>::value, "");
storage::io::FileWriter writer(path, storage::io::FileWriter::GenerateFingerprint); storage::tar::FileWriter writer(path, storage::tar::FileWriter::GenerateFingerprint);
writer.WriteElementCount64(number_of_edge_based_nodes); writer.WriteElementCount64("/common/number_of_edge_based_nodes", 1);
storage::serialization::write(writer, edge_based_edge_list); writer.WriteOne("/common/number_of_edge_based_nodes", number_of_edge_based_nodes);
writer.WriteOne(connectivity_checksum); storage::serialization::write(writer, "/common/edge_based_edge_list", edge_based_edge_list);
writer.WriteElementCount64("/common/connectivity_checksum", 1);
writer.WriteOne("/common/connectivity_checksum", connectivity_checksum);
} }
template <typename EdgeBasedEdgeVector> template <typename EdgeBasedEdgeVector>
@ -98,11 +100,11 @@ void readEdgeBasedGraph(const boost::filesystem::path &path,
{ {
static_assert(std::is_same<typename EdgeBasedEdgeVector::value_type, EdgeBasedEdge>::value, ""); static_assert(std::is_same<typename EdgeBasedEdgeVector::value_type, EdgeBasedEdge>::value, "");
storage::io::FileReader reader(path, storage::io::FileReader::VerifyFingerprint); storage::tar::FileReader reader(path, storage::tar::FileReader::VerifyFingerprint);
number_of_edge_based_nodes = reader.ReadElementCount64(); number_of_edge_based_nodes = reader.ReadOne<EdgeID>("/common/number_of_edge_based_nodes");
storage::serialization::read(reader, edge_based_edge_list); storage::serialization::read(reader, "/common/edge_based_edge_list", edge_based_edge_list);
reader.ReadInto(connectivity_checksum); connectivity_checksum = reader.ReadOne<std::uint32_t>("/common/connectivity_checksum");
} }
// reads .osrm.nodes // reads .osrm.nodes

14
include/storage/serialization.hpp
View File

@ -67,6 +67,20 @@ inline void write(storage::io::FileWriter &writer, const util::DeallocatingVecto
writer.WriteFrom(vec.bucket_list.back(), last_block_size); writer.WriteFrom(vec.bucket_list.back(), last_block_size);
} }
template <typename T>
inline void read(storage::tar::FileReader &reader, const std::string& name, util::DeallocatingVector<T> &vec)
{
vec.resize(reader.ReadElementCount64(name));
reader.ReadStreaming<T>(name, vec.begin(), vec.size());
}
template <typename T>
inline void write(storage::tar::FileWriter &writer, const std::string& name, const util::DeallocatingVector<T> &vec)
{
writer.WriteElementCount64(name, vec.size());
writer.WriteStreaming<T>(name, vec.begin(), vec.size());
}
#if USE_STXXL_LIBRARY #if USE_STXXL_LIBRARY
template <typename T> inline void read(storage::io::FileReader &reader, stxxl::vector<T> &vec) template <typename T> inline void read(storage::io::FileReader &reader, stxxl::vector<T> &vec)
{ {

8
include/util/deallocating_vector.hpp
View File

@ -228,6 +228,14 @@ template <typename ElementT> class DeallocatingVector
return *this; return *this;
} }
DeallocatingVector(std::initializer_list<ElementT> elements)
{
for (auto && elem : elements)
{
emplace_back(std::move(elem));
}
}
~DeallocatingVector() { clear(); } ~DeallocatingVector() { clear(); }
friend void swap<>(DeallocatingVector<ElementT> &lhs, DeallocatingVector<ElementT> &rhs); friend void swap<>(DeallocatingVector<ElementT> &lhs, DeallocatingVector<ElementT> &rhs);

3
src/storage/storage.cpp
View File

@ -394,9 +394,6 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlock(DataLayout::DATASOURCES_NAMES, make_block<extractor::Datasources>(1)); layout.SetBlock(DataLayout::DATASOURCES_NAMES, make_block<extractor::Datasources>(1));
} }
{
}
{ {
// Loading turn lane data // Loading turn lane data
io::FileReader lane_data_file(config.GetPath(".osrm.tld"), io::FileReader lane_data_file(config.GetPath(".osrm.tld"),

20
unit_tests/storage/serialization.cpp
View File

@ -112,4 +112,24 @@ BOOST_AUTO_TEST_CASE(tar_serialize_unsigned_vector)
} }
} }
BOOST_AUTO_TEST_CASE(tar_serialize_deallocting_vector)
{
TemporaryFile tmp;
{
std::vector<util::DeallocatingVector<unsigned>> data = {
{}, {0}, {1, 2, 3}, {4, 5, 6, 7, 8, 9, 10, 11}, {12, 13, 14, 15, 16, 17, 18, 19, 20}};
for (const auto &v : data)
{
{
tar::FileWriter writer(tmp.path, tar::FileWriter::GenerateFingerprint);
storage::serialization::write(writer, "my_unsigned_vector", v);
}
std::vector<unsigned> result;
tar::FileReader reader(tmp.path, tar::FileReader::VerifyFingerprint);
storage::serialization::read(reader, "my_unsigned_vector", result);
BOOST_CHECK_EQUAL_COLLECTIONS(v.begin(), v.end(), result.begin(), result.end());
}
}
}
BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END()