Consolidate vector deserialization.

include/util/io.hpp

@@ -53,30 +53,6 @@ bool serializeVector(std::ostream &stream, const std::vector<simple_type> &data)
     return static_cast<bool>(stream);
 }
 
-template <typename simple_type>
-bool deserializeVector(const std::string &filename, std::vector<simple_type> &data)
-{
-
-    storage::io::FileReader file(filename, true);
-
-    const auto count = file.ReadElementCount64();
-    data.resize(count);
-    if (count)
-        file.ReadInto(data.data(), count);
-    return true;
-}
-
-template <typename simple_type>
-bool deserializeVector(std::istream &stream, std::vector<simple_type> &data)
-{
-    std::uint64_t count = 0;
-    stream.read(reinterpret_cast<char *>(&count), sizeof(count));
-    data.resize(count);
-    if (count)
-        stream.read(reinterpret_cast<char *>(&data[0]), sizeof(simple_type) * count);
-    return static_cast<bool>(stream);
-}
-
 // serializes a vector of vectors into an adjacency array (creates a copy of the data internally)
 template <typename simple_type>
 bool serializeVectorIntoAdjacencyArray(const std::string &filename,
@@ -136,23 +112,20 @@ bool serializeVector(std::ofstream &out_stream, const stxxl::vector<simple_type>
 }
 
 template <typename simple_type>
-bool deserializeAdjacencyArray(const std::string &filename,
+void deserializeAdjacencyArray(const std::string &filename,
                                std::vector<std::uint32_t> &offsets,
                                std::vector<simple_type> &data)
 {
-    std::ifstream in_stream(filename, std::ios::binary);
+    storage::io::FileReader file(filename);
 
-    if (!deserializeVector(in_stream, offsets))
+    file.DeserializeVector(offsets);
-        return false;
+    file.DeserializeVector(data);
-
-    if (!deserializeVector(in_stream, data))
-        return false;
 
     // offsets have to match up with the size of the data
     if (offsets.empty() || (offsets.back() != boost::numeric_cast<std::uint32_t>(data.size())))
-        return false;
+        throw util::exception("Error in " + filename + (offsets.empty()
-
+                                                            ? "Offsets are empty"
-    return static_cast<bool>(in_stream);
+                                                            : "Offset and data size do not match"));
 }
 
 inline bool serializeFlags(const boost::filesystem::path &path, const std::vector<bool> &flags)
@@ -183,28 +156,6 @@ inline bool serializeFlags(const boost::filesystem::path &path, const std::vecto
     return static_cast<bool>(flag_stream);
 }
 
-inline bool deserializeFlags(const boost::filesystem::path &path, std::vector<bool> &flags)
-{
-    SimpleLogger().Write() << "Reading flags from " << path;
-    storage::io::FileReader flag_file(path, true);
-
-    const auto number_of_bits = flag_file.ReadOne<std::uint32_t>();
-    flags.resize(number_of_bits);
-    // putting bits in ints
-    std::uint32_t chunks = (number_of_bits + 31) / 32;
-    std::size_t bit_position = 0;
-    std::uint32_t chunk;
-    for (std::size_t chunk_id = 0; chunk_id < chunks; ++chunk_id)
-    {
-        flag_file.ReadInto(chunk);
-        std::bitset<32> chunk_bits(chunk);
-        for (std::size_t bit = 0; bit < 32 && bit_position < number_of_bits; ++bit, ++bit_position)
-            flags[bit_position] = chunk_bits[bit];
-    }
-    SimpleLogger().Write() << "Read " << number_of_bits << " bits in " << chunks
-                           << " Chunks from disk.";
-    return true;
-}
 } // namespace util
 } // namespace osrm
 

src/contractor/contractor.cpp

@@ -10,6 +10,7 @@
 #include "util/graph_loader.hpp"
 #include "util/integer_range.hpp"
 #include "util/io.hpp"
+#include "storage/io.hpp"
 #include "util/simple_logger.hpp"
 #include "util/static_graph.hpp"
 #include "util/static_rtree.hpp"
@@ -164,14 +165,12 @@ int Contractor::Run()
     util::SimpleLogger().Write() << "Reading node weights.";
     std::vector<EdgeWeight> node_weights;
     std::string node_file_name = config.osrm_input_path.string() + ".enw";
-    if (util::deserializeVector(node_file_name, node_weights))
+
     {
-        util::SimpleLogger().Write() << "Done reading node weights.";
+        storage::io::FileReader node_file(node_file_name, true);
-    }
+        node_file.DeserializeVector(node_weights);
-    else
-    {
-        throw util::exception("Failed reading node weights.");
     }
+    util::SimpleLogger().Write() << "Done reading node weights.";
 
     util::DeallocatingVector<QueryEdge> contracted_edge_list;
     ContractGraph(max_edge_id,

src/storage/storage.cpp

@@ -276,11 +276,9 @@ void Storage::PopulateLayout(DataLayout &layout)
     {
         std::vector<std::uint32_t> lane_description_offsets;
         std::vector<extractor::guidance::TurnLaneType::Mask> lane_description_masks;
-        if (!util::deserializeAdjacencyArray(config.turn_lane_description_path.string(),
+        util::deserializeAdjacencyArray(config.turn_lane_description_path.string(),
-                                             lane_description_offsets,
+                                        lane_description_offsets,
-                                             lane_description_masks))
+                                        lane_description_masks);
-            throw util::exception("Failed to read lane descriptions from: " +
-                                  config.turn_lane_description_path.string());
         layout.SetBlockSize<std::uint32_t>(DataLayout::LANE_DESCRIPTION_OFFSETS,
                                            lane_description_offsets.size());
         layout.SetBlockSize<extractor::guidance::TurnLaneType::Mask>(
@@ -545,15 +543,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Turn lane descriptions
     {
-        /* NOTE: file io - refactor this in the future */
         std::vector<std::uint32_t> lane_description_offsets;
         std::vector<extractor::guidance::TurnLaneType::Mask> lane_description_masks;
-        if (!util::deserializeAdjacencyArray(config.turn_lane_description_path.string(),
+        util::deserializeAdjacencyArray(config.turn_lane_description_path.string(),
-                                             lane_description_offsets,
+                                        lane_description_offsets,
-                                             lane_description_masks))
+                                        lane_description_masks);
-            throw util::exception("Failed to read lane descriptions from: " +
+
-                                  config.turn_lane_description_path.string());
-        /* END NOTE */
         const auto turn_lane_offset_ptr = layout.GetBlockPtr<std::uint32_t, true>(
             memory_ptr, DataLayout::LANE_DESCRIPTION_OFFSETS);
         if (!lane_description_offsets.empty())

unit_tests/util/io.cpp

@@ -1,4 +1,5 @@
 #include "util/io.hpp"
+#include "storage/io.hpp"
 #include "util/typedefs.hpp"
 
 #include <boost/test/test_case_template.hpp>
@@ -10,21 +11,6 @@ const static std::string IO_TMP_FILE = "test_io.tmp";
 
 BOOST_AUTO_TEST_SUITE(osrm_io)
 
-BOOST_AUTO_TEST_CASE(io_flags)
-{
-    std::vector<bool> flags_in, flags_out;
-    flags_in.resize(53);
-    for (std::size_t i = 0; i < flags_in.size(); ++i)
-        flags_in[i] = ((i % 2) == 1);
-
-    osrm::util::serializeFlags(IO_TMP_FILE, flags_in);
-    osrm::util::deserializeFlags(IO_TMP_FILE, flags_out);
-
-    BOOST_REQUIRE_EQUAL(flags_in.size(), flags_out.size());
-    BOOST_CHECK_EQUAL_COLLECTIONS(
-        flags_out.begin(), flags_out.end(), flags_in.begin(), flags_in.end());
-}
-
 BOOST_AUTO_TEST_CASE(io_data)
 {
     std::vector<int> data_in, data_out;
@@ -33,7 +19,9 @@ BOOST_AUTO_TEST_CASE(io_data)
         data_in[i] = i;
 
     osrm::util::serializeVector(IO_TMP_FILE, data_in);
-    osrm::util::deserializeVector(IO_TMP_FILE, data_out);
+
+    osrm::storage::io::FileReader f(IO_TMP_FILE);
+    f.DeserializeVector(data_out);
 
     BOOST_REQUIRE_EQUAL(data_in.size(), data_out.size());
     BOOST_CHECK_EQUAL_COLLECTIONS(data_out.begin(), data_out.end(), data_in.begin(), data_in.end());