Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Consolidate vector deserialization.

Browse Source
This commit is contained in:
Daniel Patterson 2016-11-15 00:08:45 -08:00
parent 1fc969e6c8
commit 7b1131b982
4 changed files with 23 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
include/util/io.hpp
View File

@ -53,30 +53,6 @@ bool serializeVector(std::ostream &stream, const std::vector<simple_type> &data)
return static_cast<bool>(stream);
}
template <typename simple_type>
bool deserializeVector(const std::string &filename, std::vector<simple_type> &data)
{
storage::io::FileReader file(filename, true);
const auto count = file.ReadElementCount64();
data.resize(count);
if (count)
file.ReadInto(data.data(), count);
return true;
}
template <typename simple_type>
bool deserializeVector(std::istream &stream, std::vector<simple_type> &data)
{
std::uint64_t count = 0;
stream.read(reinterpret_cast<char *>(&count), sizeof(count));
data.resize(count);
if (count)
stream.read(reinterpret_cast<char *>(&data[0]), sizeof(simple_type) * count);
return static_cast<bool>(stream);
}
// serializes a vector of vectors into an adjacency array (creates a copy of the data internally)
template <typename simple_type>
bool serializeVectorIntoAdjacencyArray(const std::string &filename,
@ -136,23 +112,20 @@ bool serializeVector(std::ofstream &out_stream, const stxxl::vector<simple_type>
}
template <typename simple_type>
bool deserializeAdjacencyArray(const std::string &filename,
void deserializeAdjacencyArray(const std::string &filename,
std::vector<std::uint32_t> &offsets,
std::vector<simple_type> &data)
{
std::ifstream in_stream(filename, std::ios::binary);
storage::io::FileReader file(filename);
if (!deserializeVector(in_stream, offsets))
return false;
if (!deserializeVector(in_stream, data))
return false;
file.DeserializeVector(offsets);
file.DeserializeVector(data);
// offsets have to match up with the size of the data
if (offsets.empty() || (offsets.back() != boost::numeric_cast<std::uint32_t>(data.size())))
return false;
return static_cast<bool>(in_stream);
throw util::exception("Error in " + filename + (offsets.empty()
? "Offsets are empty"
: "Offset and data size do not match"));
}
inline bool serializeFlags(const boost::filesystem::path &path, const std::vector<bool> &flags)
@ -183,28 +156,6 @@ inline bool serializeFlags(const boost::filesystem::path &path, const std::vecto
return static_cast<bool>(flag_stream);
}
inline bool deserializeFlags(const boost::filesystem::path &path, std::vector<bool> &flags)
{
SimpleLogger().Write() << "Reading flags from " << path;
storage::io::FileReader flag_file(path, true);
const auto number_of_bits = flag_file.ReadOne<std::uint32_t>();
flags.resize(number_of_bits);
// putting bits in ints
std::uint32_t chunks = (number_of_bits + 31) / 32;
std::size_t bit_position = 0;
std::uint32_t chunk;
for (std::size_t chunk_id = 0; chunk_id < chunks; ++chunk_id)
{
flag_file.ReadInto(chunk);
std::bitset<32> chunk_bits(chunk);
for (std::size_t bit = 0; bit < 32 && bit_position < number_of_bits; ++bit, ++bit_position)
flags[bit_position] = chunk_bits[bit];
}
SimpleLogger().Write() << "Read " << number_of_bits << " bits in " << chunks
<< " Chunks from disk.";
return true;
}
} // namespace util
} // namespace osrm

11
src/contractor/contractor.cpp
View File

@ -10,6 +10,7 @@
#include "util/graph_loader.hpp"
#include "util/integer_range.hpp"
#include "util/io.hpp"
#include "storage/io.hpp"
#include "util/simple_logger.hpp"
#include "util/static_graph.hpp"
#include "util/static_rtree.hpp"
@ -164,14 +165,12 @@ int Contractor::Run()
util::SimpleLogger().Write() << "Reading node weights.";
std::vector<EdgeWeight> node_weights;
std::string node_file_name = config.osrm_input_path.string() + ".enw";
if (util::deserializeVector(node_file_name, node_weights))
{
util::SimpleLogger().Write() << "Done reading node weights.";
}
else
{
throw util::exception("Failed reading node weights.");
storage::io::FileReader node_file(node_file_name, true);
node_file.DeserializeVector(node_weights);
}
util::SimpleLogger().Write() << "Done reading node weights.";
util::DeallocatingVector<QueryEdge> contracted_edge_list;
ContractGraph(max_edge_id,

19
src/storage/storage.cpp
View File

@ -276,11 +276,9 @@ void Storage::PopulateLayout(DataLayout &layout)
{
std::vector<std::uint32_t> lane_description_offsets;
std::vector<extractor::guidance::TurnLaneType::Mask> lane_description_masks;
if (!util::deserializeAdjacencyArray(config.turn_lane_description_path.string(),
lane_description_offsets,
lane_description_masks))
throw util::exception("Failed to read lane descriptions from: " +
config.turn_lane_description_path.string());
util::deserializeAdjacencyArray(config.turn_lane_description_path.string(),
lane_description_offsets,
lane_description_masks);
layout.SetBlockSize<std::uint32_t>(DataLayout::LANE_DESCRIPTION_OFFSETS,
lane_description_offsets.size());
layout.SetBlockSize<extractor::guidance::TurnLaneType::Mask>(
@ -545,15 +543,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Turn lane descriptions
{
/* NOTE: file io - refactor this in the future */
std::vector<std::uint32_t> lane_description_offsets;
std::vector<extractor::guidance::TurnLaneType::Mask> lane_description_masks;
if (!util::deserializeAdjacencyArray(config.turn_lane_description_path.string(),
lane_description_offsets,
lane_description_masks))
throw util::exception("Failed to read lane descriptions from: " +
config.turn_lane_description_path.string());
/* END NOTE */
util::deserializeAdjacencyArray(config.turn_lane_description_path.string(),
lane_description_offsets,
lane_description_masks);
const auto turn_lane_offset_ptr = layout.GetBlockPtr<std::uint32_t, true>(
memory_ptr, DataLayout::LANE_DESCRIPTION_OFFSETS);
if (!lane_description_offsets.empty())

20
unit_tests/util/io.cpp
View File

@ -1,4 +1,5 @@
#include "util/io.hpp"
#include "storage/io.hpp"
#include "util/typedefs.hpp"
#include <boost/test/test_case_template.hpp>
@ -10,21 +11,6 @@ const static std::string IO_TMP_FILE = "test_io.tmp";
BOOST_AUTO_TEST_SUITE(osrm_io)
BOOST_AUTO_TEST_CASE(io_flags)
{
std::vector<bool> flags_in, flags_out;
flags_in.resize(53);
for (std::size_t i = 0; i < flags_in.size(); ++i)
flags_in[i] = ((i % 2) == 1);
osrm::util::serializeFlags(IO_TMP_FILE, flags_in);
osrm::util::deserializeFlags(IO_TMP_FILE, flags_out);
BOOST_REQUIRE_EQUAL(flags_in.size(), flags_out.size());
BOOST_CHECK_EQUAL_COLLECTIONS(
flags_out.begin(), flags_out.end(), flags_in.begin(), flags_in.end());
}
BOOST_AUTO_TEST_CASE(io_data)
{
std::vector<int> data_in, data_out;
@ -33,7 +19,9 @@ BOOST_AUTO_TEST_CASE(io_data)
data_in[i] = i;
osrm::util::serializeVector(IO_TMP_FILE, data_in);
osrm::util::deserializeVector(IO_TMP_FILE, data_out);
osrm::storage::io::FileReader f(IO_TMP_FILE);
f.DeserializeVector(data_out);
BOOST_REQUIRE_EQUAL(data_in.size(), data_out.size());
BOOST_CHECK_EQUAL_COLLECTIONS(data_out.begin(), data_out.end(), data_in.begin(), data_in.end());