Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Port .osrm.icd file over to tar

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-16 16:39:18 +00:00
parent c664d0392a
commit 5d1b4ce71d
8 changed files with 98 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -169,7 +169,7 @@ add_executable(osrm-routed src/tools/routed.cpp $<TARGET_OBJECTS:SERVER> $<TARGE
add_executable(osrm-datastore src/tools/store.cpp $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm src/osrm/osrm.cpp $<TARGET_OBJECTS:ENGINE> $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL> )
add_library(osrm_contract src/osrm/contractor.cpp $<TARGET_OBJECTS:CONTRACTOR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_extract src/osrm/extractor.cpp $<TARGET_OBJECTS:EXTRACTOR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_extract src/osrm/extractor.cpp $<TARGET_OBJECTS:EXTRACTOR> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_guidance $<TARGET_OBJECTS:GUIDANCE> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_partition src/osrm/partitioner.cpp $<TARGET_OBJECTS:PARTITIONER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_customize src/osrm/customizer.cpp $<TARGET_OBJECTS:CUSTOMIZER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)

12
include/extractor/files.hpp
View File

@ -33,10 +33,10 @@ inline void writeIntersections(const boost::filesystem::path &path,
std::is_same<IntersectionBearingsView, IntersectionBearingsT>::value,
"");
storage::io::FileWriter writer(path, storage::io::FileWriter::GenerateFingerprint);
storage::tar::FileWriter writer(path, storage::tar::FileWriter::GenerateFingerprint);
serialization::write(writer, intersection_bearings);
storage::serialization::write(writer, entry_classes);
serialization::write(writer, "/common/intersection_bearings", intersection_bearings);
storage::serialization::write(writer, "/common/entry_classes", entry_classes);
}
// read the .osrm.icd file
@ -49,10 +49,10 @@ inline void readIntersections(const boost::filesystem::path &path,
std::is_same<IntersectionBearingsView, IntersectionBearingsT>::value,
"");
storage::io::FileReader reader(path, storage::io::FileReader::VerifyFingerprint);
storage::tar::FileReader reader(path, storage::tar::FileReader::VerifyFingerprint);
serialization::read(reader, intersection_bearings);
storage::serialization::read(reader, entry_classes);
serialization::read(reader, "/common/intersection_bearings", intersection_bearings);
storage::serialization::read(reader, "/common/entry_classes", entry_classes);
}
// reads .osrm.properties

13
include/extractor/intersection_bearings_container.hpp
View File

@ -2,6 +2,7 @@
#define OSRM_EXTRACTOR_BEARING_CONTAINER_HPP
#include "storage/shared_memory_ownership.hpp"
#include "storage/tar_fwd.hpp"
#include "util/guidance/bearing_class.hpp"
#include "util/range_table.hpp"
@ -21,11 +22,13 @@ template <storage::Ownership Ownership> class IntersectionBearingsContainer;
namespace serialization
{
template <storage::Ownership Ownership>
void read(storage::io::FileReader &reader,
void read(storage::tar::FileReader &reader,
const std::string& name,
detail::IntersectionBearingsContainer<Ownership> &turn_data);
template <storage::Ownership Ownership>
void write(storage::io::FileWriter &writer,
void write(storage::tar::FileWriter &writer,
const std::string& name,
const detail::IntersectionBearingsContainer<Ownership> &turn_data);
}
@ -83,10 +86,12 @@ template <storage::Ownership Ownership> class IntersectionBearingsContainer
return result;
}
friend void serialization::read<Ownership>(storage::io::FileReader &reader,
friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
const std::string &name,
IntersectionBearingsContainer &turn_data_container);
friend void
serialization::write<Ownership>(storage::io::FileWriter &writer,
serialization::write<Ownership>(storage::tar::FileWriter &writer,
const std::string &name,
const IntersectionBearingsContainer &turn_data_container);
private:

18
include/extractor/serialization.hpp
View File

@ -25,21 +25,23 @@ namespace serialization
// read/write for bearing data
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader,
inline void read(storage::tar::FileReader &reader,
const std::string& name,
detail::IntersectionBearingsContainer<Ownership> &intersection_bearings)
{
storage::serialization::read(reader, intersection_bearings.values);
storage::serialization::read(reader, intersection_bearings.node_to_class_id);
util::serialization::read(reader, intersection_bearings.class_id_to_ranges_table);
storage::serialization::read(reader, name + "/bearing_values", intersection_bearings.values);
storage::serialization::read(reader, name + "/node_to_class_id", intersection_bearings.node_to_class_id);
util::serialization::read(reader, name + "/class_id_to_ranges", intersection_bearings.class_id_to_ranges_table);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
inline void write(storage::tar::FileWriter &writer,
const std::string& name,
const detail::IntersectionBearingsContainer<Ownership> &intersection_bearings)
{
storage::serialization::write(writer, intersection_bearings.values);
storage::serialization::write(writer, intersection_bearings.node_to_class_id);
util::serialization::write(writer, intersection_bearings.class_id_to_ranges_table);
storage::serialization::write(writer, name + "/bearing_values", intersection_bearings.values);
storage::serialization::write(writer, name + "/node_to_class_id", intersection_bearings.node_to_class_id);
util::serialization::write(writer, name + "/class_id_to_ranges", intersection_bearings.class_id_to_ranges_table);
}
// read/write for properties file

16
include/util/range_table.hpp
View File

@ -1,8 +1,8 @@
#ifndef RANGE_TABLE_HPP
#define RANGE_TABLE_HPP
#include "storage/io.hpp"
#include "storage/shared_memory_ownership.hpp"
#include "storage/tar_fwd.hpp"
#include "util/integer_range.hpp"
#include "util/vector_view.hpp"
@ -25,10 +25,14 @@ class RangeTable;
namespace serialization
{
template <unsigned BlockSize, storage::Ownership Ownership>
void write(storage::io::FileWriter &writer, const util::RangeTable<BlockSize, Ownership> &table);
void write(storage::tar::FileWriter &writer,
const std::string &name,
const util::RangeTable<BlockSize, Ownership> &table);
template <unsigned BlockSize, storage::Ownership Ownership>
void read(storage::io::FileReader &reader, util::RangeTable<BlockSize, Ownership> &table);
void read(storage::tar::FileReader &reader,
const std::string &name,
util::RangeTable<BlockSize, Ownership> &table);
}
/**
@ -177,9 +181,11 @@ template <unsigned BLOCK_SIZE, storage::Ownership Ownership> class RangeTable
return irange(begin_idx, end_idx);
}
friend void serialization::write<BLOCK_SIZE, Ownership>(storage::io::FileWriter &writer,
friend void serialization::write<BLOCK_SIZE, Ownership>(storage::tar::FileWriter &writer,
const std::string &name,
const RangeTable &table);
friend void serialization::read<BLOCK_SIZE, Ownership>(storage::io::FileReader &reader,
friend void serialization::read<BLOCK_SIZE, Ownership>(storage::tar::FileReader &reader,
const std::string &name,
RangeTable &table);
private:

16
include/util/serialization.hpp
View File

@ -31,6 +31,22 @@ void read(storage::io::FileReader &reader, util::RangeTable<BlockSize, Ownership
storage::serialization::read(reader, table.diff_blocks);
}
template <unsigned BlockSize, storage::Ownership Ownership>
void write(storage::tar::FileWriter &writer, const std::string& name, const util::RangeTable<BlockSize, Ownership> &table)
{
writer.WriteOne(name + "/sum_lengths.meta", table.sum_lengths);
storage::serialization::write(writer, name + "/block_offsets", table.block_offsets);
storage::serialization::write(writer, name + "/diff_blocks", table.diff_blocks);
}
template <unsigned BlockSize, storage::Ownership Ownership>
void read(storage::tar::FileReader &reader, const std::string& name, util::RangeTable<BlockSize, Ownership> &table)
{
table.sum_lengths = reader.ReadOne<unsigned>(name + "/sum_lengths.meta");
storage::serialization::read(reader, name + "/block_offsets", table.block_offsets);
storage::serialization::read(reader, name + "/diff_blocks", table.diff_blocks);
}
template <typename T, std::size_t Bits, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Bits, Ownership> &vec)
{

32
src/storage/storage.cpp
View File

@ -399,30 +399,6 @@ void Storage::PopulateLayout(DataLayout &layout)
}
{
io::FileReader reader(config.GetPath(".osrm.icd"), io::FileReader::VerifyFingerprint);
auto num_discreate_bearings = reader.ReadVectorSize<DiscreteBearing>();
layout.SetBlock(DataLayout::BEARING_VALUES,
make_block<DiscreteBearing>(num_discreate_bearings));
auto num_bearing_classes = reader.ReadVectorSize<BearingClassID>();
layout.SetBlock(DataLayout::BEARING_CLASSID,
make_block<BearingClassID>(num_bearing_classes));
reader.Skip<std::uint32_t>(1); // sum_lengths
const auto bearing_blocks = reader.ReadVectorSize<unsigned>();
const auto bearing_offsets =
reader
.ReadVectorSize<typename util::RangeTable<16, storage::Ownership::View>::BlockT>();
layout.SetBlock(DataLayout::BEARING_OFFSETS, make_block<unsigned>(bearing_blocks));
layout.SetBlock(DataLayout::BEARING_BLOCKS,
make_block<typename util::RangeTable<16, storage::Ownership::View>::BlockT>(
bearing_offsets));
auto num_entry_classes = reader.ReadVectorSize<util::guidance::EntryClass>();
layout.SetBlock(DataLayout::ENTRY_CLASS,
make_block<util::guidance::EntryClass>(num_entry_classes));
}
{
@ -487,6 +463,11 @@ void Storage::PopulateLayout(DataLayout &layout)
{"/ch/edge_filter/5", DataLayout::CH_EDGE_FILTER_5},
{"/ch/edge_filter/6", DataLayout::CH_EDGE_FILTER_6},
{"/ch/edge_filter/7", DataLayout::CH_EDGE_FILTER_7},
{"/common/intersection_bearings/bearing_values", DataLayout::BEARING_VALUES},
{"/common/intersection_bearings/node_to_class_id", DataLayout::BEARING_CLASSID},
{"/common/intersection_bearings/class_id_to_ranges/block_offsets", DataLayout::BEARING_OFFSETS},
{"/common/intersection_bearings/class_id_to_ranges/diff_blocks", DataLayout::BEARING_BLOCKS},
{"/common/entry_classes", DataLayout::ENTRY_CLASS},
};
std::vector<NamedBlock> blocks;
@ -497,7 +478,8 @@ void Storage::PopulateLayout(DataLayout &layout)
{OPTIONAL, config.GetPath(".osrm.cells")},
{OPTIONAL, config.GetPath(".osrm.partition")},
{OPTIONAL, config.GetPath(".osrm.cell_metrics")},
{OPTIONAL, config.GetPath(".osrm.hsgr")}
{OPTIONAL, config.GetPath(".osrm.hsgr")},
{REQUIRED, config.GetPath(".osrm.icd")}
};
for (const auto &file : tar_files)

38
unit_tests/util/serialization.cpp Normal file
View File

@ -0,0 +1,38 @@
#include "util/serialization.hpp"
#include "../common/range_tools.hpp"
#include "../common/temporary_file.hpp"
#include <boost/filesystem.hpp>
#include <boost/test/unit_test.hpp>
BOOST_AUTO_TEST_SUITE(serialization)
using namespace osrm;
using namespace osrm::util;
BOOST_AUTO_TEST_CASE(tar_serialize_range_table)
{
TemporaryFile tmp;
{
constexpr unsigned BLOCK_SIZE = 16;
using TestRangeTable = RangeTable<BLOCK_SIZE, osrm::storage::Ownership::Container>;
std::vector<TestRangeTable> data = {
};
for (const auto &v : data)
{
{
storage::tar::FileWriter writer(tmp.path, storage::tar::FileWriter::GenerateFingerprint);
util::serialization::write(writer, "my_range_table", v);
}
TestRangeTable result;
storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
util::serialization::read(reader, "my_range_table", result);
}
}
}
BOOST_AUTO_TEST_SUITE_END()