Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Destructor should fail hard if an exception is raised.

Browse Source
This commit is contained in:
Daniel Patterson 2016-11-14 14:34:50 -08:00
parent be496eb4e3
commit 2f9b5788d0
5 changed files with 127 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/engine/datafacade/shared_memory_datafacade.hpp
View File

@ -35,8 +35,8 @@ class SharedDataFacade : public BigRAMBlockDataFacadeBase
public: public:
// this function handle the deallocation of the shared memory it we can prove it will not be // this function handle the deallocation of the shared memory it we can prove it will not be
// used anymore // used anymore. We crash hard here if something goes wrong (noexcept).
virtual ~SharedDataFacade() virtual ~SharedDataFacade() noexcept
{ {
boost::interprocess::scoped_lock<boost::interprocess::named_sharable_mutex> exclusive_lock( boost::interprocess::scoped_lock<boost::interprocess::named_sharable_mutex> exclusive_lock(
data_region == storage::DATA_1 ? shared_barriers->regions_1_mutex data_region == storage::DATA_1 ? shared_barriers->regions_1_mutex

105
include/storage/io.hpp
View File

@ -24,33 +24,33 @@ namespace storage
namespace io namespace io
{ {
class File class FileReader
{ {
private: private:
std::string filename; const std::string filename;
boost::filesystem::ifstream input_stream; boost::filesystem::ifstream input_stream;
public: public:
File(const std::string &filename, const bool check_fingerprint = false) FileReader(const std::string &filename, const bool check_fingerprint = false)
: File(boost::filesystem::path(filename), check_fingerprint) : FileReader(boost::filesystem::path(filename), check_fingerprint)
{ {
} }
File(const boost::filesystem::path &filename_, const bool check_fingerprint = false) FileReader(const boost::filesystem::path &filename_, const bool check_fingerprint = false)
: filename(filename_.string())
{ {
filename = filename_.string();
input_stream.open(filename_, std::ios::binary); input_stream.open(filename_, std::ios::binary);
if (!input_stream) if (!input_stream)
throw util::exception("Error opening " + filename + ":" + std::strerror(errno)); throw util::exception("Error opening " + filename + ":" + std::strerror(errno));
if (check_fingerprint && !readAndCheckFingerprint()) if (check_fingerprint && !ReadAndCheckFingerprint())
{ {
throw util::exception("Fingerprint mismatch in " + filename); throw util::exception("Fingerprint mismatch in " + filename);
} }
} }
/* Read count objects of type T into pointer dest */ /* Read count objects of type T into pointer dest */
template <typename T> void readInto(T *dest, const std::size_t count) template <typename T> void ReadInto(T *dest, const std::size_t count)
{ {
static_assert(std::is_trivially_copyable<T>::value, static_assert(std::is_trivially_copyable<T>::value,
"bytewise reading requires trivially copyable type"); "bytewise reading requires trivially copyable type");
@ -73,35 +73,35 @@ class File
} }
} }
template <typename T> void readInto(T &target) { readInto(&target, 1); } template <typename T> void ReadInto(T &target) { ReadInto(&target, 1); }
template <typename T> T readOne() template <typename T> T ReadOne()
{ {
T tmp; T tmp;
readInto(tmp); ReadInto(tmp);
return tmp; return tmp;
} }
template <typename T> void skip(const std::size_t element_count) template <typename T> void Skip(const std::size_t element_count)
{ {
boost::iostreams::seek(input_stream, element_count * sizeof(T), BOOST_IOS::cur); boost::iostreams::seek(input_stream, element_count * sizeof(T), BOOST_IOS::cur);
} }
/*******************************************/ /*******************************************/
std::uint32_t readElementCount32() { return readOne<std::uint32_t>(); } std::uint32_t ReadElementCount32() { return ReadOne<std::uint32_t>(); }
std::uint64_t readElementCount64() { return readOne<std::uint64_t>(); } std::uint64_t ReadElementCount64() { return ReadOne<std::uint64_t>(); }
template <typename T> void deserializeVector(std::vector<T> &data) template <typename T> void DeserializeVector(std::vector<T> &data)
{ {
const auto count = readElementCount64(); const auto count = ReadElementCount64();
data.resize(count); data.resize(count);
readInto(data.data(), count); ReadInto(data.data(), count);
} }
bool readAndCheckFingerprint() bool ReadAndCheckFingerprint()
{ {
auto fingerprint = readOne<util::FingerPrint>(); auto fingerprint = ReadOne<util::FingerPrint>();
const auto valid = util::FingerPrint::GetValid(); const auto valid = util::FingerPrint::GetValid();
// compare the compilation state stored in the fingerprint // compare the compilation state stored in the fingerprint
return valid.IsMagicNumberOK(fingerprint) && valid.TestContractor(fingerprint) && return valid.IsMagicNumberOK(fingerprint) && valid.TestContractor(fingerprint) &&
@ -109,7 +109,7 @@ class File
valid.TestQueryObjects(fingerprint); valid.TestQueryObjects(fingerprint);
} }
std::size_t size() std::size_t Size()
{ {
auto current_pos = input_stream.tellg(); auto current_pos = input_stream.tellg();
input_stream.seekg(0, input_stream.end); input_stream.seekg(0, input_stream.end);
@ -118,7 +118,7 @@ class File
return length; return length;
} }
std::vector<std::string> readLines() std::vector<std::string> ReadLines()
{ {
std::vector<std::string> result; std::vector<std::string> result;
std::string thisline; std::string thisline;
@ -140,21 +140,6 @@ class File
} }
}; };
// Reads the count of elements that is written in the file header and returns the number
inline std::uint64_t readElementCount64(boost::filesystem::ifstream &input_stream)
{
std::uint64_t number_of_elements = 0;
input_stream.read(reinterpret_cast<char *>(&number_of_elements), sizeof(number_of_elements));
return number_of_elements;
}
inline std::uint32_t readElementCount32(boost::filesystem::ifstream &input_stream)
{
std::uint32_t number_of_elements = 0;
input_stream.read(reinterpret_cast<char *>(&number_of_elements), sizeof(number_of_elements));
return number_of_elements;
}
// To make function calls consistent, this function returns the fixed number of properties // To make function calls consistent, this function returns the fixed number of properties
inline std::size_t readPropertiesCount() { return 1; } inline std::size_t readPropertiesCount() { return 1; }
@ -179,10 +164,10 @@ static_assert(sizeof(HSGRHeader) == 20, "HSGRHeader is not packed");
// Reads the checksum, number of nodes and number of edges written in the header file of a `.hsgr` // Reads the checksum, number of nodes and number of edges written in the header file of a `.hsgr`
// file and returns them in a HSGRHeader struct // file and returns them in a HSGRHeader struct
inline HSGRHeader readHSGRHeader(io::File &input_file) inline HSGRHeader readHSGRHeader(io::FileReader &input_file)
{ {
const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid(); const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid();
const auto fingerprint_loaded = input_file.readOne<util::FingerPrint>(); const auto fingerprint_loaded = input_file.ReadOne<util::FingerPrint>();
if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid)) if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid))
{ {
util::SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build.\n" util::SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build.\n"
@ -190,9 +175,9 @@ inline HSGRHeader readHSGRHeader(io::File &input_file)
} }
HSGRHeader header; HSGRHeader header;
input_file.readInto(header.checksum); input_file.ReadInto(header.checksum);
input_file.readInto(header.number_of_nodes); input_file.ReadInto(header.number_of_nodes);
input_file.readInto(header.number_of_edges); input_file.ReadInto(header.number_of_edges);
BOOST_ASSERT_MSG(0 != header.number_of_nodes, "number of nodes is zero"); BOOST_ASSERT_MSG(0 != header.number_of_nodes, "number of nodes is zero");
// number of edges can be zero, this is the case in a few test fixtures // number of edges can be zero, this is the case in a few test fixtures
@ -204,7 +189,7 @@ inline HSGRHeader readHSGRHeader(io::File &input_file)
// Needs to be called after readHSGRHeader() to get the correct offset in the stream // Needs to be called after readHSGRHeader() to get the correct offset in the stream
using NodeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::NodeArrayEntry; using NodeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::NodeArrayEntry;
using EdgeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::EdgeArrayEntry; using EdgeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::EdgeArrayEntry;
inline void readHSGR(File &input_file, inline void readHSGR(io::FileReader &input_file,
NodeT *node_buffer, NodeT *node_buffer,
const std::uint64_t number_of_nodes, const std::uint64_t number_of_nodes,
EdgeT *edge_buffer, EdgeT *edge_buffer,
@ -212,17 +197,17 @@ inline void readHSGR(File &input_file,
{ {
BOOST_ASSERT(node_buffer); BOOST_ASSERT(node_buffer);
BOOST_ASSERT(edge_buffer); BOOST_ASSERT(edge_buffer);
input_file.readInto(node_buffer, number_of_nodes); input_file.ReadInto(node_buffer, number_of_nodes);
input_file.readInto(edge_buffer, number_of_edges); input_file.ReadInto(edge_buffer, number_of_edges);
} }
// Loads properties from a `.properties` file into memory // Loads properties from a `.properties` file into memory
inline void readProperties(File &properties_file, inline void readProperties(io::FileReader &properties_file,
extractor::ProfileProperties *properties, extractor::ProfileProperties *properties,
const std::size_t properties_size) const std::size_t properties_size)
{ {
BOOST_ASSERT(properties); BOOST_ASSERT(properties);
properties_file.readInto(properties, properties_size); properties_file.ReadInto(properties, properties_size);
} }
// Reads the timestamp in a `.timestamp` file // Reads the timestamp in a `.timestamp` file
@ -237,18 +222,18 @@ inline void readTimestamp(boost::filesystem::ifstream &timestamp_input_stream,
// Loads datasource_indexes from .datasource_indexes into memory // Loads datasource_indexes from .datasource_indexes into memory
// Needs to be called after readElementCount() to get the correct offset in the stream // Needs to be called after readElementCount() to get the correct offset in the stream
inline void readDatasourceIndexes(File &datasource_indexes_file, inline void readDatasourceIndexes(io::FileReader &datasource_indexes_file,
uint8_t *datasource_buffer, uint8_t *datasource_buffer,
const std::uint64_t number_of_datasource_indexes) const std::uint64_t number_of_datasource_indexes)
{ {
BOOST_ASSERT(datasource_buffer); BOOST_ASSERT(datasource_buffer);
datasource_indexes_file.readInto(datasource_buffer, number_of_datasource_indexes); datasource_indexes_file.ReadInto(datasource_buffer, number_of_datasource_indexes);
} }
// Loads edge data from .edge files into memory which includes its // Loads edge data from .edge files into memory which includes its
// geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID // geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID
// Needs to be called after readElementCount() to get the correct offset in the stream // Needs to be called after readElementCount() to get the correct offset in the stream
inline void readEdges(File &edges_input_file, inline void readEdges(io::FileReader &edges_input_file,
GeometryID *geometry_list, GeometryID *geometry_list,
NameID *name_id_list, NameID *name_id_list,
extractor::guidance::TurnInstruction *turn_instruction_list, extractor::guidance::TurnInstruction *turn_instruction_list,
@ -268,7 +253,7 @@ inline void readEdges(File &edges_input_file,
extractor::OriginalEdgeData current_edge_data; extractor::OriginalEdgeData current_edge_data;
for (std::uint64_t i = 0; i < number_of_edges; ++i) for (std::uint64_t i = 0; i < number_of_edges; ++i)
{ {
edges_input_file.readInto(current_edge_data); edges_input_file.ReadInto(current_edge_data);
geometry_list[i] = current_edge_data.via_geometry; geometry_list[i] = current_edge_data.via_geometry;
name_id_list[i] = current_edge_data.name_id; name_id_list[i] = current_edge_data.name_id;
@ -284,7 +269,7 @@ inline void readEdges(File &edges_input_file,
// Loads coordinates and OSM node IDs from .nodes files into memory // Loads coordinates and OSM node IDs from .nodes files into memory
// Needs to be called after readElementCount() to get the correct offset in the stream // Needs to be called after readElementCount() to get the correct offset in the stream
template <typename OSMNodeIDVectorT> template <typename OSMNodeIDVectorT>
void readNodes(io::File &nodes_file, void readNodes(io::FileReader &nodes_file,
util::Coordinate *coordinate_list, util::Coordinate *coordinate_list,
OSMNodeIDVectorT &osmnodeid_list, OSMNodeIDVectorT &osmnodeid_list,
const std::uint64_t number_of_coordinates) const std::uint64_t number_of_coordinates)
@ -293,7 +278,7 @@ void readNodes(io::File &nodes_file,
extractor::QueryNode current_node; extractor::QueryNode current_node;
for (std::uint64_t i = 0; i < number_of_coordinates; ++i) for (std::uint64_t i = 0; i < number_of_coordinates; ++i)
{ {
nodes_file.readInto(current_node); nodes_file.ReadInto(current_node);
coordinate_list[i] = util::Coordinate(current_node.lon, current_node.lat); coordinate_list[i] = util::Coordinate(current_node.lon, current_node.lat);
osmnodeid_list.push_back(current_node.node_id); osmnodeid_list.push_back(current_node.node_id);
BOOST_ASSERT(coordinate_list[i].IsValid()); BOOST_ASSERT(coordinate_list[i].IsValid());
@ -308,10 +293,10 @@ struct DatasourceNamesData
std::vector<std::size_t> offsets; std::vector<std::size_t> offsets;
std::vector<std::size_t> lengths; std::vector<std::size_t> lengths;
}; };
inline DatasourceNamesData readDatasourceNames(io::File &datasource_names_file) inline DatasourceNamesData readDatasourceNames(io::FileReader &datasource_names_file)
{ {
DatasourceNamesData datasource_names_data; DatasourceNamesData datasource_names_data;
std::vector<std::string> lines = datasource_names_file.readLines(); std::vector<std::string> lines = datasource_names_file.ReadLines();
for (const auto &name : lines) for (const auto &name : lines)
{ {
datasource_names_data.offsets.push_back(datasource_names_data.names.size()); datasource_names_data.offsets.push_back(datasource_names_data.names.size());
@ -322,18 +307,6 @@ inline DatasourceNamesData readDatasourceNames(io::File &datasource_names_file)
} }
return datasource_names_data; return datasource_names_data;
} }
// Loads ram indexes of R-Trees from `.ramIndex` files into memory
// Needs to be called after readElementCount() to get the correct offset in the stream
// template <bool UseSharedMemory>
// NB Cannot be written without templated type because of cyclic depencies between
// `static_rtree.hpp` and `io.hpp`
template <typename RTreeNodeT>
void readRamIndex(File &ram_index_file, RTreeNodeT *rtree_buffer, const std::uint64_t tree_size)
{
BOOST_ASSERT(rtree_buffer);
ram_index_file.readInto(rtree_buffer, tree_size);
}
} }
} }
} }

12
include/util/io.hpp
View File

@ -57,12 +57,12 @@ template <typename simple_type>
bool deserializeVector(const std::string &filename, std::vector<simple_type> &data) bool deserializeVector(const std::string &filename, std::vector<simple_type> &data)
{ {
storage::io::File file(filename, true); storage::io::FileReader file(filename, true);
const auto count = file.readElementCount64(); const auto count = file.ReadElementCount64();
data.resize(count); data.resize(count);
if (count) if (count)
file.readInto(data.data(), count); file.ReadInto(data.data(), count);
return true; return true;
} }
@ -186,9 +186,9 @@ inline bool serializeFlags(const boost::filesystem::path &path, const std::vecto
inline bool deserializeFlags(const boost::filesystem::path &path, std::vector<bool> &flags) inline bool deserializeFlags(const boost::filesystem::path &path, std::vector<bool> &flags)
{ {
SimpleLogger().Write() << "Reading flags from " << path; SimpleLogger().Write() << "Reading flags from " << path;
storage::io::File flag_file(path, true); storage::io::FileReader flag_file(path, true);
const auto number_of_bits = flag_file.readOne<std::uint32_t>(); const auto number_of_bits = flag_file.ReadOne<std::uint32_t>();
flags.resize(number_of_bits); flags.resize(number_of_bits);
// putting bits in ints // putting bits in ints
std::uint32_t chunks = (number_of_bits + 31) / 32; std::uint32_t chunks = (number_of_bits + 31) / 32;
@ -196,7 +196,7 @@ inline bool deserializeFlags(const boost::filesystem::path &path, std::vector<bo
std::uint32_t chunk; std::uint32_t chunk;
for (std::size_t chunk_id = 0; chunk_id < chunks; ++chunk_id) for (std::size_t chunk_id = 0; chunk_id < chunks; ++chunk_id)
{ {
flag_file.readInto(chunk); flag_file.ReadInto(chunk);
std::bitset<32> chunk_bits(chunk); std::bitset<32> chunk_bits(chunk);
for (std::size_t bit = 0; bit < 32 && bit_position < number_of_bits; ++bit, ++bit_position) for (std::size_t bit = 0; bit < 32 && bit_position < number_of_bits; ++bit, ++bit_position)
flags[bit_position] = chunk_bits[bit]; flags[bit_position] = chunk_bits[bit];

6
include/util/static_rtree.hpp
View File

@ -346,12 +346,12 @@ class StaticRTree
const CoordinateListT &coordinate_list) const CoordinateListT &coordinate_list)
: m_coordinate_list(coordinate_list) : m_coordinate_list(coordinate_list)
{ {
storage::io::File tree_node_file(node_file); storage::io::FileReader tree_node_file(node_file);
const auto tree_size = tree_node_file.readElementCount64(); const auto tree_size = tree_node_file.ReadElementCount64();
m_search_tree.resize(tree_size); m_search_tree.resize(tree_size);
storage::io::readRamIndex(tree_node_file, &m_search_tree[0], tree_size); tree_node_file.ReadInto(&m_search_tree[0], tree_size);
MapLeafNodesFile(leaf_file); MapLeafNodesFile(leaf_file);
} }

154
src/storage/storage.cpp
View File

@ -261,15 +261,15 @@ void Storage::LoadLayout(DataLayout *layout_ptr)
// collect number of elements to store in shared memory object // collect number of elements to store in shared memory object
util::SimpleLogger().Write() << "load names from: " << config.names_data_path; util::SimpleLogger().Write() << "load names from: " << config.names_data_path;
// number of entries in name index // number of entries in name index
io::File name_file(config.names_data_path); io::FileReader name_file(config.names_data_path);
const auto name_blocks = name_file.readElementCount32(); const auto name_blocks = name_file.ReadElementCount32();
layout_ptr->SetBlockSize<unsigned>(DataLayout::NAME_OFFSETS, name_blocks); layout_ptr->SetBlockSize<unsigned>(DataLayout::NAME_OFFSETS, name_blocks);
layout_ptr->SetBlockSize<typename util::RangeTable<16, true>::BlockT>( layout_ptr->SetBlockSize<typename util::RangeTable<16, true>::BlockT>(
DataLayout::NAME_BLOCKS, name_blocks); DataLayout::NAME_BLOCKS, name_blocks);
BOOST_ASSERT_MSG(0 != name_blocks, "name file broken"); BOOST_ASSERT_MSG(0 != name_blocks, "name file broken");
const auto number_of_chars = name_file.readElementCount32(); const auto number_of_chars = name_file.ReadElementCount32();
layout_ptr->SetBlockSize<char>(DataLayout::NAME_CHAR_LIST, number_of_chars); layout_ptr->SetBlockSize<char>(DataLayout::NAME_CHAR_LIST, number_of_chars);
} }
@ -289,8 +289,8 @@ void Storage::LoadLayout(DataLayout *layout_ptr)
// Loading information for original edges // Loading information for original edges
{ {
io::File edges_file(config.edges_data_path); io::FileReader edges_file(config.edges_data_path);
const auto number_of_original_edges = edges_file.readElementCount64(); const auto number_of_original_edges = edges_file.ReadElementCount64();
// note: settings this all to the same size is correct, we extract them from the same struct // note: settings this all to the same size is correct, we extract them from the same struct
layout_ptr->SetBlockSize<NodeID>(DataLayout::VIA_NODE_LIST, number_of_original_edges); layout_ptr->SetBlockSize<NodeID>(DataLayout::VIA_NODE_LIST, number_of_original_edges);
@ -308,7 +308,7 @@ void Storage::LoadLayout(DataLayout *layout_ptr)
} }
{ {
io::File hsgr_file(config.hsgr_data_path); io::FileReader hsgr_file(config.hsgr_data_path);
const auto hsgr_header = io::readHSGRHeader(hsgr_file); const auto hsgr_header = io::readHSGRHeader(hsgr_file);
layout_ptr->SetBlockSize<unsigned>(DataLayout::HSGR_CHECKSUM, 1); layout_ptr->SetBlockSize<unsigned>(DataLayout::HSGR_CHECKSUM, 1);
@ -320,9 +320,9 @@ void Storage::LoadLayout(DataLayout *layout_ptr)
// load rsearch tree size // load rsearch tree size
{ {
io::File tree_node_file(config.ram_index_path); io::FileReader tree_node_file(config.ram_index_path);
const auto tree_size = tree_node_file.readElementCount64(); const auto tree_size = tree_node_file.ReadElementCount64();
layout_ptr->SetBlockSize<RTreeNode>(DataLayout::R_SEARCH_TREE, tree_size); layout_ptr->SetBlockSize<RTreeNode>(DataLayout::R_SEARCH_TREE, tree_size);
} }
@ -335,22 +335,22 @@ void Storage::LoadLayout(DataLayout *layout_ptr)
// read timestampsize // read timestampsize
{ {
io::File timestamp_file(config.timestamp_path); io::FileReader timestamp_file(config.timestamp_path);
const auto timestamp_size = timestamp_file.size(); const auto timestamp_size = timestamp_file.Size();
layout_ptr->SetBlockSize<char>(DataLayout::TIMESTAMP, timestamp_size); layout_ptr->SetBlockSize<char>(DataLayout::TIMESTAMP, timestamp_size);
} }
// load core marker size // load core marker size
{ {
io::File core_marker_file(config.core_data_path); io::FileReader core_marker_file(config.core_data_path);
const auto number_of_core_markers = core_marker_file.readElementCount32(); const auto number_of_core_markers = core_marker_file.ReadElementCount32();
layout_ptr->SetBlockSize<unsigned>(DataLayout::CORE_MARKER, number_of_core_markers); layout_ptr->SetBlockSize<unsigned>(DataLayout::CORE_MARKER, number_of_core_markers);
} }
// load coordinate size // load coordinate size
{ {
io::File node_file(config.nodes_data_path); io::FileReader node_file(config.nodes_data_path);
const auto coordinate_list_size = node_file.readElementCount64(); const auto coordinate_list_size = node_file.ReadElementCount64();
layout_ptr->SetBlockSize<util::Coordinate>(DataLayout::COORDINATE_LIST, layout_ptr->SetBlockSize<util::Coordinate>(DataLayout::COORDINATE_LIST,
coordinate_list_size); coordinate_list_size);
// we'll read a list of OSM node IDs from the same data, so set the block size for the same // we'll read a list of OSM node IDs from the same data, so set the block size for the same
@ -362,15 +362,15 @@ void Storage::LoadLayout(DataLayout *layout_ptr)
// load geometries sizes // load geometries sizes
{ {
io::File geometry_file(config.geometries_path); io::FileReader geometry_file(config.geometries_path);
const auto number_of_geometries_indices = geometry_file.readElementCount32(); const auto number_of_geometries_indices = geometry_file.ReadElementCount32();
layout_ptr->SetBlockSize<unsigned>(DataLayout::GEOMETRIES_INDEX, layout_ptr->SetBlockSize<unsigned>(DataLayout::GEOMETRIES_INDEX,
number_of_geometries_indices); number_of_geometries_indices);
geometry_file.skip<unsigned>(number_of_geometries_indices); geometry_file.Skip<unsigned>(number_of_geometries_indices);
const auto number_of_compressed_geometries = geometry_file.readElementCount32(); const auto number_of_compressed_geometries = geometry_file.ReadElementCount32();
layout_ptr->SetBlockSize<NodeID>(DataLayout::GEOMETRIES_NODE_LIST, layout_ptr->SetBlockSize<NodeID>(DataLayout::GEOMETRIES_NODE_LIST,
number_of_compressed_geometries); number_of_compressed_geometries);
layout_ptr->SetBlockSize<EdgeWeight>(DataLayout::GEOMETRIES_FWD_WEIGHT_LIST, layout_ptr->SetBlockSize<EdgeWeight>(DataLayout::GEOMETRIES_FWD_WEIGHT_LIST,
@ -382,8 +382,8 @@ void Storage::LoadLayout(DataLayout *layout_ptr)
// load datasource sizes. This file is optional, and it's non-fatal if it doesn't // load datasource sizes. This file is optional, and it's non-fatal if it doesn't
// exist. // exist.
{ {
io::File geometry_datasource_file(config.datasource_indexes_path); io::FileReader geometry_datasource_file(config.datasource_indexes_path);
const auto number_of_compressed_datasources = geometry_datasource_file.readElementCount64(); const auto number_of_compressed_datasources = geometry_datasource_file.ReadElementCount64();
layout_ptr->SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST, layout_ptr->SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST,
number_of_compressed_datasources); number_of_compressed_datasources);
} }
@ -391,7 +391,7 @@ void Storage::LoadLayout(DataLayout *layout_ptr)
// Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't // Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't
// exist // exist
{ {
io::File datasource_names_file(config.datasource_names_path); io::FileReader datasource_names_file(config.datasource_names_path);
const io::DatasourceNamesData datasource_names_data = const io::DatasourceNamesData datasource_names_data =
io::readDatasourceNames(datasource_names_file); io::readDatasourceNames(datasource_names_file);
@ -405,34 +405,34 @@ void Storage::LoadLayout(DataLayout *layout_ptr)
} }
{ {
io::File intersection_file(config.intersection_class_path, true); io::FileReader intersection_file(config.intersection_class_path, true);
std::vector<BearingClassID> bearing_class_id_table; std::vector<BearingClassID> bearing_class_id_table;
intersection_file.deserializeVector(bearing_class_id_table); intersection_file.DeserializeVector(bearing_class_id_table);
layout_ptr->SetBlockSize<BearingClassID>(DataLayout::BEARING_CLASSID, layout_ptr->SetBlockSize<BearingClassID>(DataLayout::BEARING_CLASSID,
bearing_class_id_table.size()); bearing_class_id_table.size());
const auto bearing_blocks = intersection_file.readElementCount32(); const auto bearing_blocks = intersection_file.ReadElementCount32();
intersection_file.skip<std::uint32_t>(1); // sum_lengths intersection_file.Skip<std::uint32_t>(1); // sum_lengths
layout_ptr->SetBlockSize<unsigned>(DataLayout::BEARING_OFFSETS, bearing_blocks); layout_ptr->SetBlockSize<unsigned>(DataLayout::BEARING_OFFSETS, bearing_blocks);
layout_ptr->SetBlockSize<typename util::RangeTable<16, true>::BlockT>( layout_ptr->SetBlockSize<typename util::RangeTable<16, true>::BlockT>(
DataLayout::BEARING_BLOCKS, bearing_blocks); DataLayout::BEARING_BLOCKS, bearing_blocks);
// No need to read the data // No need to read the data
intersection_file.skip<unsigned>(bearing_blocks); intersection_file.Skip<unsigned>(bearing_blocks);
intersection_file.skip<typename util::RangeTable<16, true>::BlockT>(bearing_blocks); intersection_file.Skip<typename util::RangeTable<16, true>::BlockT>(bearing_blocks);
const auto num_bearings = intersection_file.readElementCount64(); const auto num_bearings = intersection_file.ReadElementCount64();
// Skip over the actual data // Skip over the actual data
intersection_file.skip<DiscreteBearing>(num_bearings); intersection_file.Skip<DiscreteBearing>(num_bearings);
layout_ptr->SetBlockSize<DiscreteBearing>(DataLayout::BEARING_VALUES, num_bearings); layout_ptr->SetBlockSize<DiscreteBearing>(DataLayout::BEARING_VALUES, num_bearings);
std::vector<util::guidance::EntryClass> entry_class_table; std::vector<util::guidance::EntryClass> entry_class_table;
intersection_file.deserializeVector(entry_class_table); intersection_file.DeserializeVector(entry_class_table);
layout_ptr->SetBlockSize<util::guidance::EntryClass>(DataLayout::ENTRY_CLASS, layout_ptr->SetBlockSize<util::guidance::EntryClass>(DataLayout::ENTRY_CLASS,
entry_class_table.size()); entry_class_table.size());
@ -440,8 +440,8 @@ void Storage::LoadLayout(DataLayout *layout_ptr)
{ {
// Loading turn lane data // Loading turn lane data
io::File lane_data_file(config.turn_lane_data_path); io::FileReader lane_data_file(config.turn_lane_data_path);
const auto lane_tuple_count = lane_data_file.readElementCount64(); const auto lane_tuple_count = lane_data_file.ReadElementCount64();
layout_ptr->SetBlockSize<util::guidance::LaneTupleIdPair>(DataLayout::TURN_LANE_DATA, layout_ptr->SetBlockSize<util::guidance::LaneTupleIdPair>(DataLayout::TURN_LANE_DATA,
lane_tuple_count); lane_tuple_count);
} }
@ -454,7 +454,7 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr)
// Load the HSGR file // Load the HSGR file
{ {
io::File hsgr_file(config.hsgr_data_path); io::FileReader hsgr_file(config.hsgr_data_path);
auto hsgr_header = io::readHSGRHeader(hsgr_file); auto hsgr_header = io::readHSGRHeader(hsgr_file);
unsigned *checksum_ptr = unsigned *checksum_ptr =
layout_ptr->GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM); layout_ptr->GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
@ -493,9 +493,9 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr)
// Name data // Name data
{ {
io::File name_file(config.names_data_path); io::FileReader name_file(config.names_data_path);
const auto name_blocks_count = name_file.readElementCount32(); const auto name_blocks_count = name_file.ReadElementCount32();
const auto name_char_list_count = name_file.readElementCount32(); const auto name_char_list_count = name_file.ReadElementCount32();
using NameRangeTable = util::RangeTable<16, true>; using NameRangeTable = util::RangeTable<16, true>;
@ -507,16 +507,16 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr)
// Loading street names // Loading street names
const auto name_offsets_ptr = const auto name_offsets_ptr =
layout_ptr->GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::NAME_OFFSETS); layout_ptr->GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::NAME_OFFSETS);
name_file.readInto(name_offsets_ptr, name_blocks_count); name_file.ReadInto(name_offsets_ptr, name_blocks_count);
const auto name_blocks_ptr = const auto name_blocks_ptr =
layout_ptr->GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::NAME_BLOCKS); layout_ptr->GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::NAME_BLOCKS);
name_file.readInto(reinterpret_cast<char *>(name_blocks_ptr), name_file.ReadInto(reinterpret_cast<char *>(name_blocks_ptr),
layout_ptr->GetBlockSize(DataLayout::NAME_BLOCKS)); layout_ptr->GetBlockSize(DataLayout::NAME_BLOCKS));
// The file format contains the element count a second time. Don't know why, // The file format contains the element count a second time. Don't know why,
// but we need to read it here to progress the file pointer to the correct spot // but we need to read it here to progress the file pointer to the correct spot
const auto temp_count = name_file.readElementCount32(); const auto temp_count = name_file.ReadElementCount32();
const auto name_char_ptr = const auto name_char_ptr =
layout_ptr->GetBlockPtr<char, true>(memory_ptr, DataLayout::NAME_CHAR_LIST); layout_ptr->GetBlockPtr<char, true>(memory_ptr, DataLayout::NAME_CHAR_LIST);
@ -525,14 +525,14 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr)
layout_ptr->GetBlockSize(DataLayout::NAME_CHAR_LIST), layout_ptr->GetBlockSize(DataLayout::NAME_CHAR_LIST),
"Name file corrupted!"); "Name file corrupted!");
name_file.readInto(name_char_ptr, temp_count); name_file.ReadInto(name_char_ptr, temp_count);
} }
// Turn lane data // Turn lane data
{ {
io::File lane_data_file(config.turn_lane_data_path); io::FileReader lane_data_file(config.turn_lane_data_path);
const auto lane_tuple_count = lane_data_file.readElementCount64(); const auto lane_tuple_count = lane_data_file.ReadElementCount64();
// Need to call GetBlockPtr -> it write the memory canary, even if no data needs to be // Need to call GetBlockPtr -> it write the memory canary, even if no data needs to be
// loaded. // loaded.
@ -541,7 +541,7 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr)
memory_ptr, DataLayout::TURN_LANE_DATA); memory_ptr, DataLayout::TURN_LANE_DATA);
BOOST_ASSERT(lane_tuple_count * sizeof(util::guidance::LaneTupleIdPair) == BOOST_ASSERT(lane_tuple_count * sizeof(util::guidance::LaneTupleIdPair) ==
layout_ptr->GetBlockSize(DataLayout::TURN_LANE_DATA)); layout_ptr->GetBlockSize(DataLayout::TURN_LANE_DATA));
lane_data_file.readInto(turn_lane_data_ptr, lane_tuple_count); lane_data_file.ReadInto(turn_lane_data_ptr, lane_tuple_count);
} }
// Turn lane descriptions // Turn lane descriptions
@ -580,9 +580,9 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr)
// Load original edge data // Load original edge data
{ {
io::File edges_input_file(config.edges_data_path); io::FileReader edges_input_file(config.edges_data_path);
const auto number_of_original_edges = edges_input_file.readElementCount64(); const auto number_of_original_edges = edges_input_file.ReadElementCount64();
const auto via_geometry_ptr = const auto via_geometry_ptr =
layout_ptr->GetBlockPtr<GeometryID, true>(memory_ptr, DataLayout::VIA_NODE_LIST); layout_ptr->GetBlockPtr<GeometryID, true>(memory_ptr, DataLayout::VIA_NODE_LIST);
@ -623,37 +623,37 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr)
// load compressed geometry // load compressed geometry
{ {
io::File geometry_input_file(config.geometries_path); io::FileReader geometry_input_file(config.geometries_path);
const auto geometry_index_count = geometry_input_file.readElementCount32(); const auto geometry_index_count = geometry_input_file.ReadElementCount32();
const auto geometries_index_ptr = const auto geometries_index_ptr =
layout_ptr->GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::GEOMETRIES_INDEX); layout_ptr->GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::GEOMETRIES_INDEX);
BOOST_ASSERT(geometry_index_count == layout_ptr->num_entries[DataLayout::GEOMETRIES_INDEX]); BOOST_ASSERT(geometry_index_count == layout_ptr->num_entries[DataLayout::GEOMETRIES_INDEX]);
geometry_input_file.readInto(geometries_index_ptr, geometry_index_count); geometry_input_file.ReadInto(geometries_index_ptr, geometry_index_count);
const auto geometries_node_id_list_ptr = const auto geometries_node_id_list_ptr =
layout_ptr->GetBlockPtr<NodeID, true>(memory_ptr, DataLayout::GEOMETRIES_NODE_LIST); layout_ptr->GetBlockPtr<NodeID, true>(memory_ptr, DataLayout::GEOMETRIES_NODE_LIST);
const auto geometry_node_lists_count = geometry_input_file.readElementCount32(); const auto geometry_node_lists_count = geometry_input_file.ReadElementCount32();
BOOST_ASSERT(geometry_node_lists_count == BOOST_ASSERT(geometry_node_lists_count ==
layout_ptr->num_entries[DataLayout::GEOMETRIES_NODE_LIST]); layout_ptr->num_entries[DataLayout::GEOMETRIES_NODE_LIST]);
geometry_input_file.readInto(geometries_node_id_list_ptr, geometry_node_lists_count); geometry_input_file.ReadInto(geometries_node_id_list_ptr, geometry_node_lists_count);
const auto geometries_fwd_weight_list_ptr = layout_ptr->GetBlockPtr<EdgeWeight, true>( const auto geometries_fwd_weight_list_ptr = layout_ptr->GetBlockPtr<EdgeWeight, true>(
memory_ptr, DataLayout::GEOMETRIES_FWD_WEIGHT_LIST); memory_ptr, DataLayout::GEOMETRIES_FWD_WEIGHT_LIST);
BOOST_ASSERT(geometry_node_lists_count == BOOST_ASSERT(geometry_node_lists_count ==
layout_ptr->num_entries[DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]); layout_ptr->num_entries[DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]);
geometry_input_file.readInto(geometries_fwd_weight_list_ptr, geometry_node_lists_count); geometry_input_file.ReadInto(geometries_fwd_weight_list_ptr, geometry_node_lists_count);
const auto geometries_rev_weight_list_ptr = layout_ptr->GetBlockPtr<EdgeWeight, true>( const auto geometries_rev_weight_list_ptr = layout_ptr->GetBlockPtr<EdgeWeight, true>(
memory_ptr, DataLayout::GEOMETRIES_REV_WEIGHT_LIST); memory_ptr, DataLayout::GEOMETRIES_REV_WEIGHT_LIST);
BOOST_ASSERT(geometry_node_lists_count == BOOST_ASSERT(geometry_node_lists_count ==
layout_ptr->num_entries[DataLayout::GEOMETRIES_REV_WEIGHT_LIST]); layout_ptr->num_entries[DataLayout::GEOMETRIES_REV_WEIGHT_LIST]);
geometry_input_file.readInto(geometries_rev_weight_list_ptr, geometry_node_lists_count); geometry_input_file.ReadInto(geometries_rev_weight_list_ptr, geometry_node_lists_count);
} }
{ {
io::File geometry_datasource_file(config.datasource_indexes_path); io::FileReader geometry_datasource_file(config.datasource_indexes_path);
const auto number_of_compressed_datasources = geometry_datasource_file.readElementCount64(); const auto number_of_compressed_datasources = geometry_datasource_file.ReadElementCount64();
// load datasource information (if it exists) // load datasource information (if it exists)
const auto datasources_list_ptr = const auto datasources_list_ptr =
@ -667,7 +667,7 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr)
{ {
/* Load names */ /* Load names */
io::File datasource_names_file(config.datasource_names_path); io::FileReader datasource_names_file(config.datasource_names_path);
const auto datasource_names_data = io::readDatasourceNames(datasource_names_file); const auto datasource_names_data = io::readDatasourceNames(datasource_names_file);
@ -702,8 +702,8 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr)
// Loading list of coordinates // Loading list of coordinates
{ {
io::File nodes_file(config.nodes_data_path); io::FileReader nodes_file(config.nodes_data_path);
nodes_file.skip<std::uint64_t>(1); // node_count nodes_file.Skip<std::uint64_t>(1); // node_count
const auto coordinates_ptr = layout_ptr->GetBlockPtr<util::Coordinate, true>( const auto coordinates_ptr = layout_ptr->GetBlockPtr<util::Coordinate, true>(
memory_ptr, DataLayout::COORDINATE_LIST); memory_ptr, DataLayout::COORDINATE_LIST);
const auto osmnodeid_ptr = const auto osmnodeid_ptr =
@ -720,34 +720,34 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr)
// store timestamp // store timestamp
{ {
io::File timestamp_file(config.timestamp_path); io::FileReader timestamp_file(config.timestamp_path);
const auto timestamp_size = timestamp_file.size(); const auto timestamp_size = timestamp_file.Size();
const auto timestamp_ptr = const auto timestamp_ptr =
layout_ptr->GetBlockPtr<char, true>(memory_ptr, DataLayout::TIMESTAMP); layout_ptr->GetBlockPtr<char, true>(memory_ptr, DataLayout::TIMESTAMP);
BOOST_ASSERT(timestamp_size == layout_ptr->num_entries[DataLayout::TIMESTAMP]); BOOST_ASSERT(timestamp_size == layout_ptr->num_entries[DataLayout::TIMESTAMP]);
timestamp_file.readInto(timestamp_ptr, timestamp_size); timestamp_file.ReadInto(timestamp_ptr, timestamp_size);
} }
// store search tree portion of rtree // store search tree portion of rtree
{ {
io::File tree_node_file(config.ram_index_path); io::FileReader tree_node_file(config.ram_index_path);
// perform this read so that we're at the right stream position for the next // perform this read so that we're at the right stream position for the next
// read. // read.
tree_node_file.skip<std::uint64_t>(1); tree_node_file.Skip<std::uint64_t>(1);
const auto rtree_ptr = const auto rtree_ptr =
layout_ptr->GetBlockPtr<RTreeNode, true>(memory_ptr, DataLayout::R_SEARCH_TREE); layout_ptr->GetBlockPtr<RTreeNode, true>(memory_ptr, DataLayout::R_SEARCH_TREE);
io::readRamIndex(
tree_node_file, rtree_ptr, layout_ptr->num_entries[DataLayout::R_SEARCH_TREE]); tree_node_file.ReadInto(rtree_ptr, layout_ptr->num_entries[DataLayout::R_SEARCH_TREE]);
} }
{ {
io::File core_marker_file(config.core_data_path); io::FileReader core_marker_file(config.core_data_path);
const auto number_of_core_markers = core_marker_file.readElementCount32(); const auto number_of_core_markers = core_marker_file.ReadElementCount32();
// load core markers // load core markers
std::vector<char> unpacked_core_markers(number_of_core_markers); std::vector<char> unpacked_core_markers(number_of_core_markers);
core_marker_file.readInto(unpacked_core_markers.data(), number_of_core_markers); core_marker_file.ReadInto(unpacked_core_markers.data(), number_of_core_markers);
const auto core_marker_ptr = const auto core_marker_ptr =
layout_ptr->GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::CORE_MARKER); layout_ptr->GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::CORE_MARKER);
@ -776,7 +776,7 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr)
// load profile properties // load profile properties
{ {
io::File profile_properties_file(config.properties_path); io::FileReader profile_properties_file(config.properties_path);
const auto profile_properties_ptr = const auto profile_properties_ptr =
layout_ptr->GetBlockPtr<extractor::ProfileProperties, true>(memory_ptr, layout_ptr->GetBlockPtr<extractor::ProfileProperties, true>(memory_ptr,
DataLayout::PROPERTIES); DataLayout::PROPERTIES);
@ -786,28 +786,28 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr)
// Load intersection data // Load intersection data
{ {
io::File intersection_file(config.intersection_class_path, true); io::FileReader intersection_file(config.intersection_class_path, true);
std::vector<BearingClassID> bearing_class_id_table; std::vector<BearingClassID> bearing_class_id_table;
intersection_file.deserializeVector(bearing_class_id_table); intersection_file.DeserializeVector(bearing_class_id_table);
const auto bearing_blocks = intersection_file.readElementCount32(); const auto bearing_blocks = intersection_file.ReadElementCount32();
intersection_file.skip<std::uint32_t>(1); // sum_lengths intersection_file.Skip<std::uint32_t>(1); // sum_lengths
std::vector<unsigned> bearing_offsets_data(bearing_blocks); std::vector<unsigned> bearing_offsets_data(bearing_blocks);
std::vector<typename util::RangeTable<16, true>::BlockT> bearing_blocks_data( std::vector<typename util::RangeTable<16, true>::BlockT> bearing_blocks_data(
bearing_blocks); bearing_blocks);
intersection_file.readInto(bearing_offsets_data.data(), bearing_blocks); intersection_file.ReadInto(bearing_offsets_data.data(), bearing_blocks);
intersection_file.readInto(bearing_blocks_data.data(), bearing_blocks); intersection_file.ReadInto(bearing_blocks_data.data(), bearing_blocks);
const auto num_bearings = intersection_file.readElementCount64(); const auto num_bearings = intersection_file.ReadElementCount64();
std::vector<DiscreteBearing> bearing_class_table(num_bearings); std::vector<DiscreteBearing> bearing_class_table(num_bearings);
intersection_file.readInto(bearing_class_table.data(), num_bearings); intersection_file.ReadInto(bearing_class_table.data(), num_bearings);
std::vector<util::guidance::EntryClass> entry_class_table; std::vector<util::guidance::EntryClass> entry_class_table;
intersection_file.deserializeVector(entry_class_table); intersection_file.DeserializeVector(entry_class_table);
// load intersection classes // load intersection classes
if (!bearing_class_id_table.empty()) if (!bearing_class_id_table.empty())