diff --git a/include/util/io.hpp b/include/util/io.hpp index dfda8625a..f0fc1e457 100644 --- a/include/util/io.hpp +++ b/include/util/io.hpp @@ -53,30 +53,6 @@ bool serializeVector(std::ostream &stream, const std::vector &data) return static_cast(stream); } -template -bool deserializeVector(const std::string &filename, std::vector &data) -{ - - storage::io::FileReader file(filename, true); - - const auto count = file.ReadElementCount64(); - data.resize(count); - if (count) - file.ReadInto(data.data(), count); - return true; -} - -template -bool deserializeVector(std::istream &stream, std::vector &data) -{ - std::uint64_t count = 0; - stream.read(reinterpret_cast(&count), sizeof(count)); - data.resize(count); - if (count) - stream.read(reinterpret_cast(&data[0]), sizeof(simple_type) * count); - return static_cast(stream); -} - // serializes a vector of vectors into an adjacency array (creates a copy of the data internally) template bool serializeVectorIntoAdjacencyArray(const std::string &filename, @@ -136,23 +112,20 @@ bool serializeVector(std::ofstream &out_stream, const stxxl::vector } template -bool deserializeAdjacencyArray(const std::string &filename, +void deserializeAdjacencyArray(const std::string &filename, std::vector &offsets, std::vector &data) { - std::ifstream in_stream(filename, std::ios::binary); + storage::io::FileReader file(filename); - if (!deserializeVector(in_stream, offsets)) - return false; - - if (!deserializeVector(in_stream, data)) - return false; + file.DeserializeVector(offsets); + file.DeserializeVector(data); // offsets have to match up with the size of the data if (offsets.empty() || (offsets.back() != boost::numeric_cast(data.size()))) - return false; - - return static_cast(in_stream); + throw util::exception("Error in " + filename + (offsets.empty() + ? "Offsets are empty" + : "Offset and data size do not match")); } inline bool serializeFlags(const boost::filesystem::path &path, const std::vector &flags) @@ -183,28 +156,6 @@ inline bool serializeFlags(const boost::filesystem::path &path, const std::vecto return static_cast(flag_stream); } -inline bool deserializeFlags(const boost::filesystem::path &path, std::vector &flags) -{ - SimpleLogger().Write() << "Reading flags from " << path; - storage::io::FileReader flag_file(path, true); - - const auto number_of_bits = flag_file.ReadOne(); - flags.resize(number_of_bits); - // putting bits in ints - std::uint32_t chunks = (number_of_bits + 31) / 32; - std::size_t bit_position = 0; - std::uint32_t chunk; - for (std::size_t chunk_id = 0; chunk_id < chunks; ++chunk_id) - { - flag_file.ReadInto(chunk); - std::bitset<32> chunk_bits(chunk); - for (std::size_t bit = 0; bit < 32 && bit_position < number_of_bits; ++bit, ++bit_position) - flags[bit_position] = chunk_bits[bit]; - } - SimpleLogger().Write() << "Read " << number_of_bits << " bits in " << chunks - << " Chunks from disk."; - return true; -} } // namespace util } // namespace osrm diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp index 58aa997a9..fb5dfdf53 100644 --- a/src/contractor/contractor.cpp +++ b/src/contractor/contractor.cpp @@ -10,6 +10,7 @@ #include "util/graph_loader.hpp" #include "util/integer_range.hpp" #include "util/io.hpp" +#include "storage/io.hpp" #include "util/simple_logger.hpp" #include "util/static_graph.hpp" #include "util/static_rtree.hpp" @@ -164,14 +165,12 @@ int Contractor::Run() util::SimpleLogger().Write() << "Reading node weights."; std::vector node_weights; std::string node_file_name = config.osrm_input_path.string() + ".enw"; - if (util::deserializeVector(node_file_name, node_weights)) + { - util::SimpleLogger().Write() << "Done reading node weights."; - } - else - { - throw util::exception("Failed reading node weights."); + storage::io::FileReader node_file(node_file_name, true); + node_file.DeserializeVector(node_weights); } + util::SimpleLogger().Write() << "Done reading node weights."; util::DeallocatingVector contracted_edge_list; ContractGraph(max_edge_id, diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index a2003eefd..4571ddf8f 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -276,11 +276,9 @@ void Storage::PopulateLayout(DataLayout &layout) { std::vector lane_description_offsets; std::vector lane_description_masks; - if (!util::deserializeAdjacencyArray(config.turn_lane_description_path.string(), - lane_description_offsets, - lane_description_masks)) - throw util::exception("Failed to read lane descriptions from: " + - config.turn_lane_description_path.string()); + util::deserializeAdjacencyArray(config.turn_lane_description_path.string(), + lane_description_offsets, + lane_description_masks); layout.SetBlockSize(DataLayout::LANE_DESCRIPTION_OFFSETS, lane_description_offsets.size()); layout.SetBlockSize( @@ -545,15 +543,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // Turn lane descriptions { - /* NOTE: file io - refactor this in the future */ std::vector lane_description_offsets; std::vector lane_description_masks; - if (!util::deserializeAdjacencyArray(config.turn_lane_description_path.string(), - lane_description_offsets, - lane_description_masks)) - throw util::exception("Failed to read lane descriptions from: " + - config.turn_lane_description_path.string()); - /* END NOTE */ + util::deserializeAdjacencyArray(config.turn_lane_description_path.string(), + lane_description_offsets, + lane_description_masks); + const auto turn_lane_offset_ptr = layout.GetBlockPtr( memory_ptr, DataLayout::LANE_DESCRIPTION_OFFSETS); if (!lane_description_offsets.empty()) diff --git a/unit_tests/util/io.cpp b/unit_tests/util/io.cpp index 925e3ad18..adb1a624d 100644 --- a/unit_tests/util/io.cpp +++ b/unit_tests/util/io.cpp @@ -1,4 +1,5 @@ #include "util/io.hpp" +#include "storage/io.hpp" #include "util/typedefs.hpp" #include @@ -10,21 +11,6 @@ const static std::string IO_TMP_FILE = "test_io.tmp"; BOOST_AUTO_TEST_SUITE(osrm_io) -BOOST_AUTO_TEST_CASE(io_flags) -{ - std::vector flags_in, flags_out; - flags_in.resize(53); - for (std::size_t i = 0; i < flags_in.size(); ++i) - flags_in[i] = ((i % 2) == 1); - - osrm::util::serializeFlags(IO_TMP_FILE, flags_in); - osrm::util::deserializeFlags(IO_TMP_FILE, flags_out); - - BOOST_REQUIRE_EQUAL(flags_in.size(), flags_out.size()); - BOOST_CHECK_EQUAL_COLLECTIONS( - flags_out.begin(), flags_out.end(), flags_in.begin(), flags_in.end()); -} - BOOST_AUTO_TEST_CASE(io_data) { std::vector data_in, data_out; @@ -33,7 +19,9 @@ BOOST_AUTO_TEST_CASE(io_data) data_in[i] = i; osrm::util::serializeVector(IO_TMP_FILE, data_in); - osrm::util::deserializeVector(IO_TMP_FILE, data_out); + + osrm::storage::io::FileReader f(IO_TMP_FILE); + f.DeserializeVector(data_out); BOOST_REQUIRE_EQUAL(data_in.size(), data_out.size()); BOOST_CHECK_EQUAL_COLLECTIONS(data_out.begin(), data_out.end(), data_in.begin(), data_in.end());