Fix vector<bool> serialization for tar files and add unit tests

include/contractor/files.hpp

@@ -67,10 +67,9 @@ inline void writeGraph(const boost::filesystem::path &path,
     util::serialization::write(writer, "/ch/contracted_graph", graph);
 
     writer.WriteElementCount64("/ch/edge_filter", edge_filter.size());
-    auto id = 0;
+    for (const auto index : util::irange<std::size_t>(0, edge_filter.size()))
-    for (const auto &filter : edge_filter)
     {
-        storage::serialization::write(writer, "/ch/edge_filter/" + std::to_string(id++), filter);
+        storage::serialization::write(writer, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
     }
 
     writer.WriteElementCount64("/ch/connectivity_checksum", 1);

include/storage/serialization.hpp

@@ -139,7 +139,7 @@ template <typename T> void write(io::FileWriter &writer, const std::vector<T> &d
 template <typename T> void read(io::FileReader &reader, util::vector_view<T> &data)
 {
     const auto count = reader.ReadElementCount64();
-    BOOST_ASSERT(data.size() == count);
+    data.resize(count);
     reader.ReadInto(data.data(), count);
 }
 
@@ -174,7 +174,7 @@ inline void unpackBits(T &data, std::size_t index, std::size_t count, unsigned c
 template <typename VectorT> void readBoolVector(io::FileReader &reader, VectorT &data)
 {
     const auto count = reader.ReadElementCount64();
-    BOOST_ASSERT(data.size() == count);
+    data.resize(count);
     std::uint64_t index = 0;
     for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
     {
@@ -191,7 +191,7 @@ template <typename VectorT> void writeBoolVector(io::FileWriter &writer, const V
     std::uint64_t index = 0;
     for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
     {
-        writer.WriteOne<unsigned char>(packBits(data, CHAR_BIT * index, CHAR_BIT));
+        writer.WriteOne<unsigned char>(packBits(data, index, CHAR_BIT));
     }
     if (count > index)
         writer.WriteOne<unsigned char>(packBits(data, index, count - index));
@@ -201,10 +201,10 @@ template <typename VectorT>
 void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data)
 {
     const auto count = reader.ReadElementCount64(name);
-    BOOST_ASSERT(data.size() == count);
+    data.resize(count);
     std::uint64_t index = 0;
 
-    const auto decode = [&data, &index, count](const char block) {
+    const auto decode = [&](const unsigned char block) {
         auto read_size = std::min<std::size_t>(count - index, CHAR_BIT);
         unpackBits(data, index, read_size, block);
         index += CHAR_BIT;
@@ -222,12 +222,12 @@ void writeBoolVector(tar::FileWriter &writer, const std::string &name, const Vec
 
     const auto encode = [&]() {
         auto write_size = std::min<std::size_t>(count - index, CHAR_BIT);
-        auto packed = packBits(data, CHAR_BIT * index, write_size);
+        auto packed = packBits(data, index, write_size);
         index += CHAR_BIT;
         return packed;
     };
 
-    std::uint64_t number_of_blocks = std::ceil(count / CHAR_BIT);
+    std::uint64_t number_of_blocks = std::ceil((double)count / CHAR_BIT);
     writer.WriteStreaming<unsigned char>(
         name, boost::make_function_input_iterator(encode, boost::infinite()), number_of_blocks);
 }

include/util/vector_view.hpp

@@ -108,6 +108,15 @@ template <typename DataT> class vector_view
 
     std::size_t size() const { return m_size; }
 
+    void resize(const size_t size) {
+        if (size > m_size)
+        {
+            throw util::exception("Trying to resize a view to a larger size.");
+        }
+        m_size = size;
+    }
+
+
     bool empty() const { return 0 == size(); }
 
     DataT &operator[](const unsigned index)
@@ -185,7 +194,18 @@ template <> class vector_view<bool>
         return m_ptr[bucket] & (1u << offset);
     }
 
-    void reset(unsigned *, std::size_t size) { m_size = size; }
+    void reset(unsigned * ptr, std::size_t size) {
+        m_ptr = ptr;
+        m_size = size;
+    }
+
+    void resize(const size_t size) {
+        if (size > m_size)
+        {
+            throw util::exception("Trying to resize a view to a larger size.");
+        }
+        m_size = size;
+    }
 
     std::size_t size() const { return m_size; }
 

unit_tests/CMakeLists.txt

@@ -160,19 +160,19 @@ target_include_directories(updater-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(contractor-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(storage-tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 
-target_link_libraries(engine-tests ${ENGINE_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
+target_link_libraries(engine-tests ${ENGINE_LIBRARIES} $<TARGET_OBJECTS:MICROTAR> ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(extractor-tests osrm_extract osrm_guidance ${EXTRACTOR_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
-target_link_libraries(partitioner-tests ${PARTITIONER_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
+target_link_libraries(partitioner-tests osrm_partition ${PARTITIONER_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
-target_link_libraries(customizer-tests ${CUSTOMIZER_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
+target_link_libraries(customizer-tests osrm_customize ${CUSTOMIZER_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
-target_link_libraries(updater-tests ${UPDATER_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
+target_link_libraries(updater-tests osrm_update ${UPDATER_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(library-tests osrm ${ENGINE_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(library-extract-tests osrm_extract osrm_guidance ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(library-contract-tests osrm_contract ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(library-customize-tests osrm_customize ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(library-partition-tests osrm_partition ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(server-tests osrm ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
-target_link_libraries(util-tests ${UTIL_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
+target_link_libraries(util-tests ${UTIL_LIBRARIES} $<TARGET_OBJECTS:MICROTAR> ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
-target_link_libraries(contractor-tests ${CONTRACTOR_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
+target_link_libraries(contractor-tests osrm_contract ${CONTRACTOR_LIBRARIES} $<TARGET_OBJECTS:MICROTAR> ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 target_link_libraries(storage-tests osrm_store ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
 
 add_custom_target(tests

unit_tests/common/temporary_file.hpp

@@ -6,6 +6,7 @@
 struct TemporaryFile
 {
     TemporaryFile() : path(boost::filesystem::unique_path()) {}
+    TemporaryFile(const std::string& path) : path(path){}
 
     ~TemporaryFile() { boost::filesystem::remove(path); }
 

unit_tests/contractor/files.cpp

@@ -0,0 +1,54 @@
+#include "contractor/files.hpp"
+#include "contractor/graph_contractor_adaptors.hpp"
+
+#include "../common/range_tools.hpp"
+#include "../common/temporary_file.hpp"
+#include "helper.hpp"
+
+#include <boost/test/unit_test.hpp>
+
+
+BOOST_AUTO_TEST_SUITE(tar)
+
+using namespace osrm;
+using namespace osrm::contractor;
+using namespace osrm::unit_test;
+
+BOOST_AUTO_TEST_CASE(read_write_hsgr)
+{
+    auto reference_checksum = 0xFF00FF00;
+    auto reference_connectivity_checksum = 0xDEADBEEF;
+    std::vector<TestEdge> edges = {TestEdge{0, 1, 3},
+                                   TestEdge{0, 5, 1},
+                                   TestEdge{1, 3, 3},
+                                   TestEdge{1, 4, 1},
+                                   TestEdge{3, 1, 1},
+                                   TestEdge{4, 3, 1},
+                                   TestEdge{5, 1, 1}};
+    auto reference_graph = QueryGraph {6, toEdges<QueryEdge>(makeGraph(edges))};
+    std::vector<std::vector<bool>> reference_filters = {
+        {false, false, true, true, false, false, true},
+        {true, false, true, false, true, false, true},
+        {false, false, false, false, false, false, false},
+        {true, true, true, true, true, true, true},
+    };
+
+    TemporaryFile tmp {TEST_DATA_DIR "/read_write_hsgr_test.osrm.hsgr"};
+    contractor::files::writeGraph(tmp.path, reference_checksum, reference_graph, reference_filters, reference_connectivity_checksum);
+
+    unsigned checksum;
+    unsigned connectivity_checksum;
+    QueryGraph graph;
+    std::vector<std::vector<bool>> filters;
+    contractor::files::readGraph(tmp.path, checksum, graph, filters, connectivity_checksum);
+
+    BOOST_CHECK_EQUAL(checksum, reference_checksum);
+    BOOST_CHECK_EQUAL(connectivity_checksum, reference_connectivity_checksum);
+    BOOST_CHECK_EQUAL(filters.size(), reference_filters.size());
+    CHECK_EQUAL_COLLECTIONS(filters[0], reference_filters[0]);
+    CHECK_EQUAL_COLLECTIONS(filters[1], reference_filters[1]);
+    CHECK_EQUAL_COLLECTIONS(filters[2], reference_filters[2]);
+    CHECK_EQUAL_COLLECTIONS(filters[3], reference_filters[3]);
+}
+
+BOOST_AUTO_TEST_SUITE_END()

unit_tests/contractor/graph_contractor.cpp

@@ -1,6 +1,7 @@
 #include "contractor/graph_contractor.hpp"
 
 #include "../common/range_tools.hpp"
+#include "helper.hpp"
 
 #include <boost/test/test_case_template.hpp>
 #include <boost/test/unit_test.hpp>
@@ -9,32 +10,10 @@
 
 using namespace osrm;
 using namespace osrm::contractor;
+using namespace osrm::unit_test;
 
 BOOST_AUTO_TEST_SUITE(graph_contractor)
 
-using TestEdge = std::tuple<unsigned, unsigned, int>;
-ContractorGraph makeGraph(const std::vector<TestEdge> &edges)
-{
-    std::vector<ContractorEdge> input_edges;
-    auto id = 0u;
-    auto max_id = 0u;
-    for (const auto &edge : edges)
-    {
-        unsigned start;
-        unsigned target;
-        int weight;
-        std::tie(start, target, weight) = edge;
-        max_id = std::max(std::max(start, target), max_id);
-        input_edges.push_back(ContractorEdge{
-            start, target, ContractorEdgeData{weight, weight * 2, id++, 0, false, true, false}});
-        input_edges.push_back(ContractorEdge{
-            target, start, ContractorEdgeData{weight, weight * 2, id++, 0, false, false, true}});
-    }
-    std::sort(input_edges.begin(), input_edges.end());
-
-    return ContractorGraph{max_id + 1, std::move(input_edges)};
-}
-
 BOOST_AUTO_TEST_CASE(contract_graph)
 {
     tbb::task_scheduler_init scheduler(1);

unit_tests/contractor/helper.hpp

@@ -0,0 +1,38 @@
+#ifndef OSRM_UNIT_TESTS_CONTRACTOR_HELPER_HPP_
+#define OSRM_UNIT_TESTS_CONTRACTOR_HELPER_HPP_
+
+#include "contractor/contractor_graph.hpp"
+
+namespace osrm
+{
+namespace unit_test
+{
+
+using TestEdge = std::tuple<unsigned, unsigned, int>;
+inline contractor::ContractorGraph makeGraph(const std::vector<TestEdge> &edges)
+{
+    std::vector<contractor::ContractorEdge> input_edges;
+    auto id = 0u;
+    auto max_id = 0u;
+    for (const auto &edge : edges)
+    {
+        unsigned start;
+        unsigned target;
+        int weight;
+        std::tie(start, target, weight) = edge;
+        max_id = std::max(std::max(start, target), max_id);
+        input_edges.push_back(contractor::ContractorEdge{
+            start, target, contractor::ContractorEdgeData{weight, weight * 2, id++, 0, false, true, false}});
+        input_edges.push_back(contractor::ContractorEdge{
+            target, start, contractor::ContractorEdgeData{weight, weight * 2, id++, 0, false, false, true}});
+    }
+    std::sort(input_edges.begin(), input_edges.end());
+
+    return contractor::ContractorGraph{max_id + 1, std::move(input_edges)};
+}
+
+}
+}
+
+
+#endif

unit_tests/storage/serialization.cpp

@@ -0,0 +1,75 @@
+#include "storage/serialization.hpp"
+
+#include "../common/range_tools.hpp"
+#include "../common/temporary_file.hpp"
+
+#include <boost/filesystem.hpp>
+#include <boost/test/unit_test.hpp>
+
+BOOST_AUTO_TEST_SUITE(serialization)
+
+using namespace osrm;
+using namespace osrm::storage;
+
+BOOST_AUTO_TEST_CASE(pack_test)
+{
+    std::vector<bool> v = {0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1};
+
+    BOOST_CHECK_EQUAL(storage::serialization::detail::packBits(v, 0, 8), 0x2e);
+    BOOST_CHECK_EQUAL(storage::serialization::detail::packBits(v, 5, 7), 0x65);
+    BOOST_CHECK_EQUAL(storage::serialization::detail::packBits(v, 6, 8), 0x95);
+    BOOST_CHECK_EQUAL(storage::serialization::detail::packBits(v, 11, 1), 0x01);
+}
+
+BOOST_AUTO_TEST_CASE(unpack_test)
+{
+    std::vector<bool> v(14), expected = {0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1};
+
+    storage::serialization::detail::unpackBits(v, 0, 8, 0x2e);
+    storage::serialization::detail::unpackBits(v, 5, 7, 0x65);
+    storage::serialization::detail::unpackBits(v, 6, 8, 0x95);
+    storage::serialization::detail::unpackBits(v, 11, 1, 0x01);
+    BOOST_CHECK_EQUAL_COLLECTIONS(v.begin(), v.end(), expected.begin(), expected.end());
+}
+
+BOOST_AUTO_TEST_CASE(bin_serialize_bool_vector)
+{
+    TemporaryFile tmp;
+    {
+        std::vector<std::vector<bool>> data = {
+            {}, {0}, {1, 1, 1}, {1, 1, 0, 0, 1, 1, 0, 0}, {1, 1, 0, 0, 1, 1, 0, 0, 1}};
+        for (const auto &v : data)
+        {
+            {
+                io::FileWriter writer(tmp.path, io::FileWriter::GenerateFingerprint);
+                storage::serialization::write(writer, v);
+            }
+            std::vector<bool> result;
+            io::FileReader reader(tmp.path, io::FileReader::VerifyFingerprint);
+            storage::serialization::read(reader, result);
+            BOOST_CHECK_EQUAL_COLLECTIONS(v.begin(), v.end(), result.begin(), result.end());
+        }
+    }
+}
+
+BOOST_AUTO_TEST_CASE(tar_serialize_bool_vector)
+{
+    TemporaryFile tmp;
+    {
+        std::vector<std::vector<bool>> data = {
+            {}, {0}, {1, 1, 1}, {1, 1, 0, 0, 1, 1, 0, 0}, {1, 1, 0, 0, 1, 1, 0, 0, 1}};
+        for (const auto &v : data)
+        {
+            {
+                tar::FileWriter writer(tmp.path, tar::FileWriter::GenerateFingerprint);
+                storage::serialization::write(writer, "my_boolean_vector", v);
+            }
+            std::vector<bool> result;
+            tar::FileReader reader(tmp.path, tar::FileReader::VerifyFingerprint);
+            storage::serialization::read(reader, "my_boolean_vector", result);
+            BOOST_CHECK_EQUAL_COLLECTIONS(v.begin(), v.end(), result.begin(), result.end());
+        }
+    }
+}
+
+BOOST_AUTO_TEST_SUITE_END()

unit_tests/storage/tar.cpp

@@ -1,6 +1,7 @@
 #include "storage/tar.hpp"
 
 #include "../common/range_tools.hpp"
+#include "../common/temporary_file.hpp"
 
 #include <boost/test/unit_test.hpp>
 
@@ -50,7 +51,7 @@ BOOST_AUTO_TEST_CASE(read_tar_file)
 
 BOOST_AUTO_TEST_CASE(write_tar_file)
 {
-    boost::filesystem::path tmp_path(TEST_DATA_DIR "/tar_write_test.tar");
+    TemporaryFile tmp {TEST_DATA_DIR "/tar_write_test.tar"};
 
     std::uint64_t single_64bit_integer = 0xDEADBEEFAABBCCDD;
     std::uint32_t single_32bit_integer = 0xDEADBEEF;
@@ -60,7 +61,7 @@ BOOST_AUTO_TEST_CASE(write_tar_file)
         0, 1, 2, 3, 4, 1ULL << 62, 0, 1 << 22, 0xFFFFFFFFFFFFFFFF};
 
     {
-        storage::tar::FileWriter writer(tmp_path, storage::tar::FileWriter::GenerateFingerprint);
+        storage::tar::FileWriter writer(tmp.path, storage::tar::FileWriter::GenerateFingerprint);
         writer.WriteOne("foo/single_64bit_integer", single_64bit_integer);
         writer.WriteOne("bar/single_32bit_integer", single_32bit_integer);
         writer.WriteElementCount64("baz/bla/64bit_vector", vector_64bit.size());
@@ -69,7 +70,7 @@ BOOST_AUTO_TEST_CASE(write_tar_file)
         writer.WriteFrom("32bit_vector", vector_32bit.data(), vector_32bit.size());
     }
 
-    storage::tar::FileReader reader(tmp_path, storage::tar::FileReader::VerifyFingerprint);
+    storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
 
     BOOST_CHECK_EQUAL(reader.ReadOne<std::uint32_t>("bar/single_32bit_integer"),
                       single_32bit_integer);

unit_tests/util/serialization.cpp

@@ -1,65 +0,0 @@
-#include "storage/serialization.hpp"
-
-#include <boost/filesystem.hpp>
-#include <boost/test/test_case_template.hpp>
-#include <boost/test/unit_test.hpp>
-
-#include <vector>
-
-BOOST_AUTO_TEST_SUITE(serialization_test)
-
-using namespace osrm;
-using namespace osrm::util;
-using namespace osrm::storage::io;
-using namespace osrm::storage::serialization;
-
-BOOST_AUTO_TEST_CASE(pack_test)
-{
-    std::vector<bool> v = {0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1};
-
-    BOOST_CHECK_EQUAL(osrm::storage::serialization::packBits(v, 0, 8), 0x2e);
-    BOOST_CHECK_EQUAL(osrm::storage::serialization::packBits(v, 5, 7), 0x65);
-    BOOST_CHECK_EQUAL(osrm::storage::serialization::packBits(v, 6, 8), 0x95);
-    BOOST_CHECK_EQUAL(osrm::storage::serialization::packBits(v, 11, 1), 0x01);
-}
-
-BOOST_AUTO_TEST_CASE(unpack_test)
-{
-    std::vector<bool> v(14), expected = {0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1};
-
-    osrm::storage::serialization::unpackBits(v, 0, 8, 0x2e);
-    osrm::storage::serialization::unpackBits(v, 5, 7, 0x65);
-    osrm::storage::serialization::unpackBits(v, 6, 8, 0x95);
-    osrm::storage::serialization::unpackBits(v, 11, 1, 0x01);
-    BOOST_CHECK_EQUAL_COLLECTIONS(v.begin(), v.end(), expected.begin(), expected.end());
-}
-
-struct SerializationFixture
-{
-    SerializationFixture() : temporary_file(boost::filesystem::unique_path()) {}
-    ~SerializationFixture() { remove(temporary_file); }
-
-    boost::filesystem::path temporary_file;
-};
-
-BOOST_AUTO_TEST_CASE(serialize_bool_vector)
-{
-    SerializationFixture fixture;
-    {
-        std::vector<std::vector<bool>> data = {
-            {}, {0}, {1, 1, 1}, {1, 1, 0, 0, 1, 1, 0, 0}, {1, 1, 0, 0, 1, 1, 0, 0, 1}};
-        for (const auto &v : data)
-        {
-            {
-                FileWriter writer(fixture.temporary_file, FileWriter::GenerateFingerprint);
-                write(writer, v);
-            }
-            std::vector<bool> result;
-            FileReader reader(fixture.temporary_file, FileReader::VerifyFingerprint);
-            read(reader, result);
-            BOOST_CHECK_EQUAL_COLLECTIONS(v.begin(), v.end(), result.begin(), result.end());
-        }
-    }
-}
-
-BOOST_AUTO_TEST_SUITE_END()