Consolidate file reading through the new FileReader class/interface. (#3321)
This commit is contained in:
Daniel Patterson
GitHub
@@ -10,6 +10,7 @@
|
||||
#include <boost/filesystem/fstream.hpp>
|
||||
#include <boost/spirit/include/qi.hpp>
|
||||
#include <boost/spirit/include/qi_int.hpp>
|
||||
#include <storage/io.hpp>
|
||||
|
||||
#include <iterator>
|
||||
#include <unordered_map>
|
||||
@@ -43,20 +44,14 @@ class RasterGrid
|
||||
ydim = _ydim;
|
||||
_data.reserve(ydim * xdim);
|
||||
|
||||
boost::filesystem::ifstream stream(filepath, std::ios::binary);
|
||||
if (!stream)
|
||||
{
|
||||
throw util::exception("Unable to open raster file.");
|
||||
}
|
||||
storage::io::FileReader file_reader(filepath, storage::io::FileReader::HasNoFingerprint);
|
||||
|
||||
stream.seekg(0, std::ios_base::end);
|
||||
std::string buffer;
|
||||
buffer.resize(static_cast<std::size_t>(stream.tellg()));
|
||||
|
||||
stream.seekg(0, std::ios_base::beg);
|
||||
buffer.resize(file_reader.Size());
|
||||
|
||||
BOOST_ASSERT(buffer.size() > 1);
|
||||
stream.read(&buffer[0], static_cast<std::streamsize>(buffer.size()));
|
||||
|
||||
file_reader.ReadInto(&buffer[0], buffer.size());
|
||||
|
||||
boost::algorithm::trim(buffer);
|
||||
|
||||
|
||||
+32
-142
@@ -1,14 +1,9 @@
|
||||
#ifndef OSRM_STORAGE_IO_HPP_
|
||||
#define OSRM_STORAGE_IO_HPP_
|
||||
|
||||
#include "contractor/query_edge.hpp"
|
||||
#include "extractor/extractor.hpp"
|
||||
#include "extractor/original_edge_data.hpp"
|
||||
#include "extractor/query_node.hpp"
|
||||
#include "util/exception.hpp"
|
||||
#include "util/fingerprint.hpp"
|
||||
#include "util/simple_logger.hpp"
|
||||
#include "util/static_graph.hpp"
|
||||
|
||||
#include <boost/filesystem/fstream.hpp>
|
||||
#include <boost/iostreams/seek.hpp>
|
||||
@@ -32,6 +27,16 @@ class FileReader
|
||||
boost::filesystem::ifstream input_stream;
|
||||
|
||||
public:
|
||||
class LineWrapper : public std::string
|
||||
{
|
||||
friend std::istream &operator>>(std::istream &is, LineWrapper &line)
|
||||
{
|
||||
return std::getline(is, line);
|
||||
}
|
||||
};
|
||||
auto GetLineIteratorBegin() { return std::istream_iterator<LineWrapper>(input_stream); }
|
||||
auto GetLineIteratorEnd() { return std::istream_iterator<LineWrapper>(); }
|
||||
|
||||
enum FingerprintFlag
|
||||
{
|
||||
VerifyFingerprint,
|
||||
@@ -67,6 +72,7 @@ class FileReader
|
||||
return;
|
||||
|
||||
const auto &result = input_stream.read(reinterpret_cast<char *>(dest), count * sizeof(T));
|
||||
|
||||
if (!result)
|
||||
{
|
||||
if (result.eof())
|
||||
@@ -78,6 +84,11 @@ class FileReader
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T> void ReadInto(std::vector<T> &target)
|
||||
{
|
||||
ReadInto(target.data(), target.size());
|
||||
}
|
||||
|
||||
template <typename T> void ReadInto(T &target) { ReadInto(&target, 1); }
|
||||
|
||||
template <typename T> T ReadOne()
|
||||
@@ -142,144 +153,23 @@ class FileReader
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string ReadLine()
|
||||
{
|
||||
std::string thisline;
|
||||
try
|
||||
{
|
||||
std::getline(input_stream, thisline);
|
||||
}
|
||||
catch (const std::ios_base::failure &e)
|
||||
{
|
||||
// EOF is OK here, everything else, re-throw
|
||||
if (!input_stream.eof())
|
||||
throw;
|
||||
}
|
||||
return thisline;
|
||||
}
|
||||
};
|
||||
|
||||
// To make function calls consistent, this function returns the fixed number of properties
|
||||
inline std::size_t readPropertiesCount() { return 1; }
|
||||
|
||||
struct HSGRHeader
|
||||
{
|
||||
std::uint32_t checksum;
|
||||
std::uint64_t number_of_nodes;
|
||||
std::uint64_t number_of_edges;
|
||||
};
|
||||
|
||||
// Reads the checksum, number of nodes and number of edges written in the header file of a `.hsgr`
|
||||
// file and returns them in a HSGRHeader struct
|
||||
inline HSGRHeader readHSGRHeader(io::FileReader &input_file)
|
||||
{
|
||||
const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid();
|
||||
const auto fingerprint_loaded = input_file.ReadOne<util::FingerPrint>();
|
||||
if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid))
|
||||
{
|
||||
util::SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build.\n"
|
||||
"Reprocess to get rid of this warning.";
|
||||
}
|
||||
|
||||
HSGRHeader header;
|
||||
input_file.ReadInto(header.checksum);
|
||||
input_file.ReadInto(header.number_of_nodes);
|
||||
input_file.ReadInto(header.number_of_edges);
|
||||
|
||||
BOOST_ASSERT_MSG(0 != header.number_of_nodes, "number of nodes is zero");
|
||||
// number of edges can be zero, this is the case in a few test fixtures
|
||||
|
||||
return header;
|
||||
}
|
||||
|
||||
// Reads the graph data of a `.hsgr` file into memory
|
||||
// Needs to be called after readHSGRHeader() to get the correct offset in the stream
|
||||
using NodeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::NodeArrayEntry;
|
||||
using EdgeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::EdgeArrayEntry;
|
||||
inline void readHSGR(io::FileReader &input_file,
|
||||
NodeT *node_buffer,
|
||||
const std::uint64_t number_of_nodes,
|
||||
EdgeT *edge_buffer,
|
||||
const std::uint64_t number_of_edges)
|
||||
{
|
||||
BOOST_ASSERT(node_buffer);
|
||||
BOOST_ASSERT(edge_buffer);
|
||||
input_file.ReadInto(node_buffer, number_of_nodes);
|
||||
input_file.ReadInto(edge_buffer, number_of_edges);
|
||||
}
|
||||
|
||||
// Loads datasource_indexes from .datasource_indexes into memory
|
||||
// Needs to be called after readElementCount() to get the correct offset in the stream
|
||||
inline void readDatasourceIndexes(io::FileReader &datasource_indexes_file,
|
||||
uint8_t *datasource_buffer,
|
||||
const std::uint64_t number_of_datasource_indexes)
|
||||
{
|
||||
BOOST_ASSERT(datasource_buffer);
|
||||
datasource_indexes_file.ReadInto(datasource_buffer, number_of_datasource_indexes);
|
||||
}
|
||||
|
||||
// Loads edge data from .edge files into memory which includes its
|
||||
// geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID
|
||||
// Needs to be called after readElementCount() to get the correct offset in the stream
|
||||
inline void readEdges(io::FileReader &edges_input_file,
|
||||
GeometryID *geometry_list,
|
||||
NameID *name_id_list,
|
||||
extractor::guidance::TurnInstruction *turn_instruction_list,
|
||||
LaneDataID *lane_data_id_list,
|
||||
extractor::TravelMode *travel_mode_list,
|
||||
EntryClassID *entry_class_id_list,
|
||||
util::guidance::TurnBearing *pre_turn_bearing_list,
|
||||
util::guidance::TurnBearing *post_turn_bearing_list,
|
||||
const std::uint64_t number_of_edges)
|
||||
{
|
||||
BOOST_ASSERT(geometry_list);
|
||||
BOOST_ASSERT(name_id_list);
|
||||
BOOST_ASSERT(turn_instruction_list);
|
||||
BOOST_ASSERT(lane_data_id_list);
|
||||
BOOST_ASSERT(travel_mode_list);
|
||||
BOOST_ASSERT(entry_class_id_list);
|
||||
extractor::OriginalEdgeData current_edge_data;
|
||||
for (std::uint64_t i = 0; i < number_of_edges; ++i)
|
||||
{
|
||||
edges_input_file.ReadInto(current_edge_data);
|
||||
|
||||
geometry_list[i] = current_edge_data.via_geometry;
|
||||
name_id_list[i] = current_edge_data.name_id;
|
||||
turn_instruction_list[i] = current_edge_data.turn_instruction;
|
||||
lane_data_id_list[i] = current_edge_data.lane_data_id;
|
||||
travel_mode_list[i] = current_edge_data.travel_mode;
|
||||
entry_class_id_list[i] = current_edge_data.entry_classid;
|
||||
pre_turn_bearing_list[i] = current_edge_data.pre_turn_bearing;
|
||||
post_turn_bearing_list[i] = current_edge_data.post_turn_bearing;
|
||||
}
|
||||
}
|
||||
|
||||
// Loads coordinates and OSM node IDs from .nodes files into memory
|
||||
// Needs to be called after readElementCount() to get the correct offset in the stream
|
||||
template <typename OSMNodeIDVectorT>
|
||||
void readNodes(io::FileReader &nodes_file,
|
||||
util::Coordinate *coordinate_list,
|
||||
OSMNodeIDVectorT &osmnodeid_list,
|
||||
const std::uint64_t number_of_coordinates)
|
||||
{
|
||||
BOOST_ASSERT(coordinate_list);
|
||||
extractor::QueryNode current_node;
|
||||
for (std::uint64_t i = 0; i < number_of_coordinates; ++i)
|
||||
{
|
||||
nodes_file.ReadInto(current_node);
|
||||
coordinate_list[i] = util::Coordinate(current_node.lon, current_node.lat);
|
||||
osmnodeid_list.push_back(current_node.node_id);
|
||||
BOOST_ASSERT(coordinate_list[i].IsValid());
|
||||
}
|
||||
}
|
||||
|
||||
// Reads datasource names out of .datasource_names files and metadata such as
|
||||
// the length and offset of each name
|
||||
struct DatasourceNamesData
|
||||
{
|
||||
std::vector<char> names;
|
||||
std::vector<std::size_t> offsets;
|
||||
std::vector<std::size_t> lengths;
|
||||
};
|
||||
inline DatasourceNamesData readDatasourceNames(io::FileReader &datasource_names_file)
|
||||
{
|
||||
DatasourceNamesData datasource_names_data;
|
||||
std::vector<std::string> lines = datasource_names_file.ReadLines();
|
||||
for (const auto &name : lines)
|
||||
{
|
||||
datasource_names_data.offsets.push_back(datasource_names_data.names.size());
|
||||
datasource_names_data.lengths.push_back(name.size());
|
||||
std::copy(name.c_str(),
|
||||
name.c_str() + name.size(),
|
||||
std::back_inserter(datasource_names_data.names));
|
||||
}
|
||||
return datasource_names_data;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
#ifndef OSRM_STORAGE_SERIALIZATION_HPP_
|
||||
#define OSRM_STORAGE_SERIALIZATION_HPP_
|
||||
|
||||
#include "contractor/query_edge.hpp"
|
||||
#include "extractor/extractor.hpp"
|
||||
#include "extractor/original_edge_data.hpp"
|
||||
#include "extractor/query_node.hpp"
|
||||
#include "storage/io.hpp"
|
||||
#include "util/exception.hpp"
|
||||
#include "util/fingerprint.hpp"
|
||||
#include "util/simple_logger.hpp"
|
||||
#include "util/static_graph.hpp"
|
||||
|
||||
#include <boost/filesystem/fstream.hpp>
|
||||
#include <boost/iostreams/seek.hpp>
|
||||
|
||||
#include <cerrno>
|
||||
#include <cstring>
|
||||
#include <tuple>
|
||||
#include <type_traits>
|
||||
|
||||
namespace osrm
|
||||
{
|
||||
namespace storage
|
||||
{
|
||||
namespace serialization
|
||||
{
|
||||
|
||||
// To make function calls consistent, this function returns the fixed number of properties
|
||||
inline std::size_t readPropertiesCount() { return 1; }
|
||||
|
||||
struct HSGRHeader
|
||||
{
|
||||
std::uint32_t checksum;
|
||||
std::uint64_t number_of_nodes;
|
||||
std::uint64_t number_of_edges;
|
||||
};
|
||||
|
||||
// Reads the checksum, number of nodes and number of edges written in the header file of a `.hsgr`
|
||||
// file and returns them in a HSGRHeader struct
|
||||
inline HSGRHeader readHSGRHeader(io::FileReader &input_file)
|
||||
{
|
||||
const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid();
|
||||
const auto fingerprint_loaded = input_file.ReadOne<util::FingerPrint>();
|
||||
if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid))
|
||||
{
|
||||
util::SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build.\n"
|
||||
"Reprocess to get rid of this warning.";
|
||||
}
|
||||
|
||||
HSGRHeader header;
|
||||
input_file.ReadInto(header.checksum);
|
||||
input_file.ReadInto(header.number_of_nodes);
|
||||
input_file.ReadInto(header.number_of_edges);
|
||||
|
||||
// If we have edges, then we must have nodes.
|
||||
// However, there can be nodes with no edges (some test cases create this)
|
||||
BOOST_ASSERT_MSG(header.number_of_edges == 0 || header.number_of_nodes > 0,
|
||||
"edges exist, but there are no nodes");
|
||||
|
||||
return header;
|
||||
}
|
||||
|
||||
// Reads the graph data of a `.hsgr` file into memory
|
||||
// Needs to be called after readHSGRHeader() to get the correct offset in the stream
|
||||
using NodeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::NodeArrayEntry;
|
||||
using EdgeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::EdgeArrayEntry;
|
||||
inline void readHSGR(io::FileReader &input_file,
|
||||
NodeT *node_buffer,
|
||||
const std::uint64_t number_of_nodes,
|
||||
EdgeT *edge_buffer,
|
||||
const std::uint64_t number_of_edges)
|
||||
{
|
||||
BOOST_ASSERT(node_buffer);
|
||||
BOOST_ASSERT(edge_buffer);
|
||||
input_file.ReadInto(node_buffer, number_of_nodes);
|
||||
input_file.ReadInto(edge_buffer, number_of_edges);
|
||||
}
|
||||
|
||||
// Loads datasource_indexes from .datasource_indexes into memory
|
||||
// Needs to be called after readElementCount() to get the correct offset in the stream
|
||||
inline void readDatasourceIndexes(io::FileReader &datasource_indexes_file,
|
||||
uint8_t *datasource_buffer,
|
||||
const std::uint64_t number_of_datasource_indexes)
|
||||
{
|
||||
BOOST_ASSERT(datasource_buffer);
|
||||
datasource_indexes_file.ReadInto(datasource_buffer, number_of_datasource_indexes);
|
||||
}
|
||||
|
||||
// Loads edge data from .edge files into memory which includes its
|
||||
// geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID
|
||||
// Needs to be called after readElementCount() to get the correct offset in the stream
|
||||
inline void readEdges(io::FileReader &edges_input_file,
|
||||
GeometryID *geometry_list,
|
||||
NameID *name_id_list,
|
||||
extractor::guidance::TurnInstruction *turn_instruction_list,
|
||||
LaneDataID *lane_data_id_list,
|
||||
extractor::TravelMode *travel_mode_list,
|
||||
EntryClassID *entry_class_id_list,
|
||||
util::guidance::TurnBearing *pre_turn_bearing_list,
|
||||
util::guidance::TurnBearing *post_turn_bearing_list,
|
||||
const std::uint64_t number_of_edges)
|
||||
{
|
||||
BOOST_ASSERT(geometry_list);
|
||||
BOOST_ASSERT(name_id_list);
|
||||
BOOST_ASSERT(turn_instruction_list);
|
||||
BOOST_ASSERT(lane_data_id_list);
|
||||
BOOST_ASSERT(travel_mode_list);
|
||||
BOOST_ASSERT(entry_class_id_list);
|
||||
extractor::OriginalEdgeData current_edge_data;
|
||||
for (std::uint64_t i = 0; i < number_of_edges; ++i)
|
||||
{
|
||||
edges_input_file.ReadInto(current_edge_data);
|
||||
|
||||
geometry_list[i] = current_edge_data.via_geometry;
|
||||
name_id_list[i] = current_edge_data.name_id;
|
||||
turn_instruction_list[i] = current_edge_data.turn_instruction;
|
||||
lane_data_id_list[i] = current_edge_data.lane_data_id;
|
||||
travel_mode_list[i] = current_edge_data.travel_mode;
|
||||
entry_class_id_list[i] = current_edge_data.entry_classid;
|
||||
pre_turn_bearing_list[i] = current_edge_data.pre_turn_bearing;
|
||||
post_turn_bearing_list[i] = current_edge_data.post_turn_bearing;
|
||||
}
|
||||
}
|
||||
|
||||
// Loads coordinates and OSM node IDs from .nodes files into memory
|
||||
// Needs to be called after readElementCount() to get the correct offset in the stream
|
||||
template <typename OSMNodeIDVectorT>
|
||||
void readNodes(io::FileReader &nodes_file,
|
||||
util::Coordinate *coordinate_list,
|
||||
OSMNodeIDVectorT &osmnodeid_list,
|
||||
const std::uint64_t number_of_coordinates)
|
||||
{
|
||||
BOOST_ASSERT(coordinate_list);
|
||||
extractor::QueryNode current_node;
|
||||
for (std::uint64_t i = 0; i < number_of_coordinates; ++i)
|
||||
{
|
||||
nodes_file.ReadInto(current_node);
|
||||
coordinate_list[i] = util::Coordinate(current_node.lon, current_node.lat);
|
||||
osmnodeid_list.push_back(current_node.node_id);
|
||||
BOOST_ASSERT(coordinate_list[i].IsValid());
|
||||
}
|
||||
}
|
||||
|
||||
// Reads datasource names out of .datasource_names files and metadata such as
|
||||
// the length and offset of each name
|
||||
struct DatasourceNamesData
|
||||
{
|
||||
std::vector<char> names;
|
||||
std::vector<std::size_t> offsets;
|
||||
std::vector<std::size_t> lengths;
|
||||
};
|
||||
inline DatasourceNamesData readDatasourceNames(io::FileReader &datasource_names_file)
|
||||
{
|
||||
DatasourceNamesData datasource_names_data;
|
||||
std::vector<std::string> lines = datasource_names_file.ReadLines();
|
||||
for (const auto &name : lines)
|
||||
{
|
||||
datasource_names_data.offsets.push_back(datasource_names_data.names.size());
|
||||
datasource_names_data.lengths.push_back(name.size());
|
||||
std::copy(name.c_str(),
|
||||
name.c_str() + name.size(),
|
||||
std::back_inserter(datasource_names_data.names));
|
||||
}
|
||||
return datasource_names_data;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -18,7 +18,7 @@ class NameTable
|
||||
{
|
||||
private:
|
||||
// FIXME should this use shared memory
|
||||
RangeTable<16, false> m_name_table;
|
||||
util::RangeTable<16, false> m_name_table;
|
||||
ShM<char, false>::vector m_names_char_list;
|
||||
|
||||
public:
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#ifndef RANGE_TABLE_HPP
|
||||
#define RANGE_TABLE_HPP
|
||||
|
||||
#include "storage/io.hpp"
|
||||
#include "util/integer_range.hpp"
|
||||
#include "util/shared_memory_vector_wrapper.hpp"
|
||||
|
||||
@@ -138,6 +139,21 @@ template <unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY> class RangeTable
|
||||
sum_lengths = lengths_prefix_sum;
|
||||
}
|
||||
|
||||
void ReadARangeTable(osrm::storage::io::FileReader &filereader)
|
||||
{
|
||||
unsigned number_of_blocks = filereader.ReadElementCount32();
|
||||
// read total length
|
||||
filereader.ReadInto(&sum_lengths, 1);
|
||||
|
||||
block_offsets.resize(number_of_blocks);
|
||||
diff_blocks.resize(number_of_blocks);
|
||||
|
||||
// read block offsets
|
||||
filereader.ReadInto(block_offsets.data(), number_of_blocks);
|
||||
// read blocks
|
||||
filereader.ReadInto(diff_blocks.data(), number_of_blocks);
|
||||
}
|
||||
|
||||
inline RangeT GetRange(const unsigned id) const
|
||||
{
|
||||
BOOST_ASSERT(id < block_offsets.size() + diff_blocks.size() * BLOCK_SIZE);
|
||||
|
||||
Reference in New Issue
Block a user