Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Port .names file to tar

Browse Source
This commit is contained in:
Patrick Niklaus
2018-03-21 11:10:02 +00:00
parent a594008e57
commit b8260e44fa
47 changed files with 459 additions and 438 deletions
Download Patch File Download Diff File Expand all files Collapse all files
unit_tests/util/indexed_data.cpp
+4 -66
View File
@@ -35,29 +35,7 @@ BOOST_AUTO_TEST_CASE(check_variable_group_block_bitops)
template <typename IndexedData, typename Offsets, typename Data>
void test_rw(const Offsets &offsets, const Data &data)
{
TemporaryFile file;
IndexedData indexed_data;
{
storage::io::FileWriter writer(file.path, storage::io::FileWriter::HasNoFingerprint);
indexed_data.write(writer, offsets.begin(), offsets.end(), data.begin());
}
storage::io::FileReader reader(file.path, storage::io::FileReader::HasNoFingerprint);
auto length = reader.GetSize();
std::string str(length, '\0');
reader.ReadInto(const_cast<char *>(str.data()), length);
#if 0
std::cout << "\n" << typeid(IndexedData).name() << "\nsaved size = " << str.size() << "\n";
for (auto c : str)
std::cout << std::hex << std::setw(2) << std::setfill('0')
<< (int)((unsigned char)c) << " ";
std::cout << std::dec << "\n";
#endif
indexed_data.reset(str.c_str(), str.c_str() + str.size());
IndexedData indexed_data(offsets.begin(), offsets.end(), data.begin());
for (std::size_t index = 0; index < offsets.size() - 1; ++index)
{
@@ -151,56 +129,16 @@ BOOST_AUTO_TEST_CASE(check_max_size)
name_offsets = {0, 256};
BOOST_CHECK_THROW(test_fixed(), osrm::util::exception);
name_offsets = {0, 255};
test_fixed();
}
BOOST_AUTO_TEST_CASE(check_corrupted_memory)
{
std::vector<unsigned char> buf;
auto test_variable = [&buf]() {
IndexedData<VariableGroupBlock<16, std::vector<unsigned char>>> indexed_data;
indexed_data.reset(&buf[0], &buf[buf.size()]);
const auto result = indexed_data.at(0);
return std::string(reinterpret_cast<const char *>(&result[0]), result.size());
};
// Use LE internal representation
buf = {0, 42};
BOOST_CHECK_THROW(test_variable(), osrm::util::exception);
buf = {1, 0, 0, 0, 0};
BOOST_CHECK_THROW(test_variable(), osrm::util::exception);
buf = {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42};
BOOST_CHECK_THROW(test_variable(), osrm::util::exception);
buf = {1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 42};
BOOST_CHECK_THROW(test_variable(), osrm::util::exception);
buf = {1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0xF0, 0x9F, 0x90, 0xBC};
BOOST_CHECK_EQUAL(test_variable(), "🐼");
BOOST_CHECK_NO_THROW(test_fixed());
}
BOOST_AUTO_TEST_CASE(check_string_view)
{
TemporaryFile file;
std::string name_data = "hellostringview";
std::vector<std::uint32_t> name_offsets = {0, 5, 11, 15};
IndexedData<VariableGroupBlock<16, StringView>> indexed_data;
{
storage::io::FileWriter writer(file.path, storage::io::FileWriter::HasNoFingerprint);
indexed_data.write(writer, name_offsets.begin(), name_offsets.end(), name_data.begin());
}
storage::io::FileReader reader(file.path, storage::io::FileReader::HasNoFingerprint);
auto length = reader.GetSize();
std::string str(length, '\0');
reader.ReadInto(const_cast<char *>(str.data()), length);
indexed_data.reset(str.c_str(), str.c_str() + str.size());
IndexedData<VariableGroupBlock<16, StringView>> indexed_data(
name_offsets.begin(), name_offsets.end(), name_data.begin());
BOOST_CHECK_EQUAL(indexed_data.at(0), "hello");
BOOST_CHECK_EQUAL(indexed_data.at(1), "string");
unit_tests/util/name_table.cpp
-136
View File
@@ -1,136 +0,0 @@
#include "util/name_table.hpp"
#include "common/temporary_file.hpp"
#include "util/exception.hpp"
#include <boost/filesystem.hpp>
#include <boost/test/test_case_template.hpp>
#include <boost/test/unit_test.hpp>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <typeinfo>
#include <vector>
//#include <valgrind/callgrind.h>
BOOST_AUTO_TEST_SUITE(name_table)
using namespace osrm;
using namespace osrm::util;
std::string PrapareNameTableData(std::vector<std::string> &data, bool fill_all)
{
NameTable::IndexedData indexed_data;
std::vector<unsigned char> name_char_data;
std::vector<std::uint32_t> name_offsets;
for (auto s : data)
{
name_offsets.push_back(name_char_data.size());
std::copy(s.begin(), s.end(), std::back_inserter(name_char_data));
if (fill_all)
{
std::string tmp;
tmp = s + "_des";
name_offsets.push_back(name_char_data.size());
std::copy(tmp.begin(), tmp.end(), std::back_inserter(name_char_data));
tmp = s + "_pro";
name_offsets.push_back(name_char_data.size());
std::copy(tmp.begin(), tmp.end(), std::back_inserter(name_char_data));
tmp = s + "_ref";
name_offsets.push_back(name_char_data.size());
std::copy(tmp.begin(), tmp.end(), std::back_inserter(name_char_data));
tmp = s + "_ext";
name_offsets.push_back(name_char_data.size());
std::copy(tmp.begin(), tmp.end(), std::back_inserter(name_char_data));
}
else
{
name_offsets.push_back(name_char_data.size());
name_offsets.push_back(name_char_data.size());
name_offsets.push_back(name_char_data.size());
name_offsets.push_back(name_char_data.size());
}
}
name_offsets.push_back(name_char_data.size());
TemporaryFile file;
{
storage::io::FileWriter writer(file.path, storage::io::FileWriter::HasNoFingerprint);
indexed_data.write(
writer, name_offsets.begin(), name_offsets.end(), name_char_data.begin());
}
storage::io::FileReader reader(file.path, storage::io::FileReader::HasNoFingerprint);
auto length = reader.GetSize();
std::string str(length, '\0');
reader.ReadInto(const_cast<char *>(str.data()), length);
return str;
}
BOOST_AUTO_TEST_CASE(check_name_table_fill)
{
std::vector<std::string> expected_names = {
"", "A", "check_name", "ccc", "dDDd", "E", "ff", "ggg", "hhhh", "I", "jj", "", "kkk",
"llll", "M", "nn", "ooo", "pppp", "q", "r", "S", "T", "", "u", "V", "W",
"X", "Y", "Z", "", "", "", "", "", "", "", "0", ""};
auto data = PrapareNameTableData(expected_names, true);
NameTable name_table;
name_table.reset(&data[0], &data[data.size()]);
for (std::size_t index = 0; index < expected_names.size(); ++index)
{
const NameID id = 5 * index;
BOOST_CHECK_EQUAL(name_table.GetNameForID(id), expected_names[index]);
BOOST_CHECK_EQUAL(name_table.GetRefForID(id), expected_names[index] + "_ref");
BOOST_CHECK_EQUAL(name_table.GetDestinationsForID(id), expected_names[index] + "_des");
BOOST_CHECK_EQUAL(name_table.GetPronunciationForID(id), expected_names[index] + "_pro");
BOOST_CHECK_EQUAL(name_table.GetExitsForID(id), expected_names[index] + "_ext");
}
}
BOOST_AUTO_TEST_CASE(check_name_table_nofill)
{
std::vector<std::string> expected_names = {
"", "A", "check_name", "ccc", "dDDd", "E", "ff", "ggg", "hhhh", "I", "jj", "", "kkk",
"llll", "M", "nn", "ooo", "pppp", "q", "r", "S", "T", "", "u", "V", "W",
"X", "Y", "Z", "", "", "", "", "", "", "", "0", ""};
auto data = PrapareNameTableData(expected_names, false);
NameTable name_table;
name_table.reset(&data[0], &data[data.size()]);
// CALLGRIND_START_INSTRUMENTATION;
for (std::size_t index = 0; index < expected_names.size(); ++index)
{
const NameID id = 5 * index;
BOOST_CHECK_EQUAL(name_table.GetNameForID(id), expected_names[index]);
BOOST_CHECK(name_table.GetRefForID(id).empty());
BOOST_CHECK(name_table.GetDestinationsForID(id).empty());
BOOST_CHECK(name_table.GetPronunciationForID(id).empty());
BOOST_CHECK(name_table.GetExitsForID(id).empty());
}
// CALLGRIND_STOP_INSTRUMENTATION;
}
BOOST_AUTO_TEST_CASE(check_invalid_ids)
{
NameTable name_table;
BOOST_CHECK_EQUAL(name_table.GetNameForID(INVALID_NAMEID), "");
BOOST_CHECK_EQUAL(name_table.GetRefForID(INVALID_NAMEID), "");
BOOST_CHECK_EQUAL(name_table.GetDestinationsForID(INVALID_NAMEID), "");
BOOST_CHECK_EQUAL(name_table.GetPronunciationForID(INVALID_NAMEID), "");
BOOST_CHECK_EQUAL(name_table.GetExitsForID(INVALID_NAMEID), "");
}
BOOST_AUTO_TEST_SUITE_END()
unit_tests/util/serialization.cpp
+40
View File
@@ -71,6 +71,46 @@ BOOST_AUTO_TEST_CASE(tar_serialize_packed_vector)
TestPackedVector result;
storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
util::serialization::read(reader, "my_packed_vector", result);
CHECK_EQUAL_COLLECTIONS(result, v);
}
}
}
BOOST_AUTO_TEST_CASE(tar_serialize_variable_indexed_data)
{
TemporaryFile tmp;
{
using TestIndexedData = IndexedData<VariableGroupBlock<16, std::string>>;
std::vector<std::vector<unsigned>> offset_data = {
{5, 8, 8},
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17},
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
};
std::vector<std::string> char_data = {
"HalloFoo",
"ABCDEFGHIJKLMNOPQR",
"ABCDEFGHIJKLMNOP",
};
for (const auto i : util::irange<std::size_t>(0, offset_data.size()))
{
TestIndexedData indexed {offset_data[i].begin(), offset_data[i].end(), char_data[i].begin()};
{
storage::tar::FileWriter writer(tmp.path,
storage::tar::FileWriter::GenerateFingerprint);
util::serialization::write(writer, "my_indexed_data", indexed);
}
TestIndexedData result;
storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
util::serialization::read(reader, "my_indexed_data", result);
for(auto j : util::irange<std::size_t>(0, offset_data[i].size() - 1))
{
BOOST_CHECK_EQUAL(indexed.at(j), result.at(j));
}
}
}
}