diff --git a/include/storage/io.hpp b/include/storage/io.hpp index 44702fbee..0119023a9 100644 --- a/include/storage/io.hpp +++ b/include/storage/io.hpp @@ -8,10 +8,14 @@ #include "util/fingerprint.hpp" #include "util/simple_logger.hpp" #include "util/static_graph.hpp" +#include "util/exception.hpp" #include +#include #include +#include +#include namespace osrm { @@ -20,6 +24,122 @@ namespace storage namespace io { +class File +{ + private: + std::string filename; + boost::filesystem::ifstream input_stream; + + public: + File(const std::string &filename, const bool check_fingerprint = false) + : File(boost::filesystem::path(filename), check_fingerprint) + { + } + + File(const boost::filesystem::path &filename_, const bool check_fingerprint = false) + { + filename = filename_.string(); + input_stream.open(filename_, std::ios::binary); + if (!input_stream) + throw util::exception("Error opening " + filename + ":" + std::strerror(errno)); + + if (check_fingerprint && !readAndCheckFingerprint()) + { + throw util::exception("Fingerprint mismatch in " + filename); + } + } + + /* Read count objects of type T into pointer dest */ + template void readInto(T *dest, const std::size_t count) + { + static_assert(std::is_trivially_copyable::value, + "bytewise reading requires trivially copyable type"); + if (count == 0) + return; + input_stream.read(reinterpret_cast(dest), count * sizeof(T)); + + // safe to cast here, according to CPP docs, negative values for gcount + // are never used. + const unsigned long bytes_read = static_cast(input_stream.gcount()); + const auto expected_bytes = count * sizeof(T); + + if (bytes_read == 0) + { + throw util::exception("Error reading from " + filename + ": " + std::strerror(errno)); + } + else if (bytes_read < expected_bytes) + { + throw util::exception("Error reading from " + filename + ": Unexpected end of file"); + } + } + + template void readInto(T &target) { readInto(&target, 1); } + + template T readOne() + { + T tmp; + readInto(tmp); + return tmp; + } + + template void skip(const std::size_t element_count) + { + boost::iostreams::seek(input_stream, element_count * sizeof(T), BOOST_IOS::cur); + } + + /*******************************************/ + + std::uint32_t readElementCount32() { return readOne(); } + std::uint64_t readElementCount64() { return readOne(); } + + template void deserializeVector(std::vector &data) + { + const auto count = readElementCount64(); + data.resize(count); + readInto(data.data(), count); + } + + bool readAndCheckFingerprint() + { + auto fingerprint = readOne(); + const auto valid = util::FingerPrint::GetValid(); + // compare the compilation state stored in the fingerprint + return valid.IsMagicNumberOK(fingerprint) && valid.TestContractor(fingerprint) && + valid.TestGraphUtil(fingerprint) && valid.TestRTree(fingerprint) && + valid.TestQueryObjects(fingerprint); + } + + std::size_t size() + { + auto current_pos = input_stream.tellg(); + input_stream.seekg(0, input_stream.end); + auto length = input_stream.tellg(); + input_stream.seekg(current_pos, input_stream.beg); + return length; + } + + std::vector readLines() + { + std::vector result; + std::string thisline; + try + { + while (std::getline(input_stream, thisline)) + { + std::clog << "Read " << thisline << std::endl; + result.push_back(thisline); + } + } + catch (const std::ios_base::failure &e) + { + // EOF is OK here, everything else, re-throw + if (!input_stream.eof()) + throw; + } + return result; + } +}; + // Reads the count of elements that is written in the file header and returns the number inline std::uint64_t readElementCount64(boost::filesystem::ifstream &input_stream) { @@ -59,11 +179,10 @@ static_assert(sizeof(HSGRHeader) == 20, "HSGRHeader is not packed"); // Reads the checksum, number of nodes and number of edges written in the header file of a `.hsgr` // file and returns them in a HSGRHeader struct -inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream) +inline HSGRHeader readHSGRHeader(io::File &input_file) { const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid(); - util::FingerPrint fingerprint_loaded; - input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(util::FingerPrint)); + const auto fingerprint_loaded = input_file.readOne(); if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid)) { util::SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build.\n" @@ -71,11 +190,9 @@ inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream) } HSGRHeader header; - input_stream.read(reinterpret_cast(&header.checksum), sizeof(header.checksum)); - input_stream.read(reinterpret_cast(&header.number_of_nodes), - sizeof(header.number_of_nodes)); - input_stream.read(reinterpret_cast(&header.number_of_edges), - sizeof(header.number_of_edges)); + input_file.readInto(header.checksum); + input_file.readInto(header.number_of_nodes); + input_file.readInto(header.number_of_edges); BOOST_ASSERT_MSG(0 != header.number_of_nodes, "number of nodes is zero"); // number of edges can be zero, this is the case in a few test fixtures @@ -87,7 +204,7 @@ inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream) // Needs to be called after readHSGRHeader() to get the correct offset in the stream using NodeT = typename util::StaticGraph::NodeArrayEntry; using EdgeT = typename util::StaticGraph::EdgeArrayEntry; -inline void readHSGR(boost::filesystem::ifstream &input_stream, +inline void readHSGR(File &input_file, NodeT *node_buffer, const std::uint64_t number_of_nodes, EdgeT *edge_buffer, @@ -95,17 +212,17 @@ inline void readHSGR(boost::filesystem::ifstream &input_stream, { BOOST_ASSERT(node_buffer); BOOST_ASSERT(edge_buffer); - input_stream.read(reinterpret_cast(node_buffer), number_of_nodes * sizeof(NodeT)); - input_stream.read(reinterpret_cast(edge_buffer), number_of_edges * sizeof(EdgeT)); + input_file.readInto(node_buffer, number_of_nodes); + input_file.readInto(edge_buffer, number_of_edges); } // Loads properties from a `.properties` file into memory -inline void readProperties(boost::filesystem::ifstream &properties_stream, +inline void readProperties(File &properties_file, extractor::ProfileProperties *properties, const std::size_t properties_size) { BOOST_ASSERT(properties); - properties_stream.read(reinterpret_cast(properties), properties_size); + properties_file.readInto(properties, properties_size); } // Reads the timestamp in a `.timestamp` file @@ -120,19 +237,18 @@ inline void readTimestamp(boost::filesystem::ifstream ×tamp_input_stream, // Loads datasource_indexes from .datasource_indexes into memory // Needs to be called after readElementCount() to get the correct offset in the stream -inline void readDatasourceIndexes(boost::filesystem::ifstream &datasource_indexes_input_stream, +inline void readDatasourceIndexes(File &datasource_indexes_file, uint8_t *datasource_buffer, const std::uint64_t number_of_datasource_indexes) { BOOST_ASSERT(datasource_buffer); - datasource_indexes_input_stream.read(reinterpret_cast(datasource_buffer), - number_of_datasource_indexes * sizeof(std::uint8_t)); + datasource_indexes_file.readInto(datasource_buffer, number_of_datasource_indexes); } // Loads edge data from .edge files into memory which includes its // geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID // Needs to be called after readElementCount() to get the correct offset in the stream -inline void readEdges(boost::filesystem::ifstream &edges_input_stream, +inline void readEdges(File &edges_input_file, GeometryID *geometry_list, NameID *name_id_list, extractor::guidance::TurnInstruction *turn_instruction_list, @@ -152,7 +268,7 @@ inline void readEdges(boost::filesystem::ifstream &edges_input_stream, extractor::OriginalEdgeData current_edge_data; for (std::uint64_t i = 0; i < number_of_edges; ++i) { - edges_input_stream.read((char *)&(current_edge_data), sizeof(extractor::OriginalEdgeData)); + edges_input_file.readInto(current_edge_data); geometry_list[i] = current_edge_data.via_geometry; name_id_list[i] = current_edge_data.name_id; @@ -168,7 +284,7 @@ inline void readEdges(boost::filesystem::ifstream &edges_input_stream, // Loads coordinates and OSM node IDs from .nodes files into memory // Needs to be called after readElementCount() to get the correct offset in the stream template -void readNodes(boost::filesystem::ifstream &nodes_input_stream, +void readNodes(io::File &nodes_file, util::Coordinate *coordinate_list, OSMNodeIDVectorT &osmnodeid_list, const std::uint64_t number_of_coordinates) @@ -177,7 +293,7 @@ void readNodes(boost::filesystem::ifstream &nodes_input_stream, extractor::QueryNode current_node; for (std::uint64_t i = 0; i < number_of_coordinates; ++i) { - nodes_input_stream.read((char *)¤t_node, sizeof(extractor::QueryNode)); + nodes_file.readInto(current_node); coordinate_list[i] = util::Coordinate(current_node.lon, current_node.lat); osmnodeid_list.push_back(current_node.node_id); BOOST_ASSERT(coordinate_list[i].IsValid()); @@ -192,12 +308,11 @@ struct DatasourceNamesData std::vector offsets; std::vector lengths; }; -inline DatasourceNamesData -readDatasourceNames(boost::filesystem::ifstream &datasource_names_input_stream) +inline DatasourceNamesData readDatasourceNames(io::File &datasource_names_file) { DatasourceNamesData datasource_names_data; - std::string name; - while (std::getline(datasource_names_input_stream, name)) + std::vector lines = datasource_names_file.readLines(); + for (const auto &name : lines) { datasource_names_data.offsets.push_back(datasource_names_data.names.size()); datasource_names_data.lengths.push_back(name.size()); @@ -214,13 +329,10 @@ readDatasourceNames(boost::filesystem::ifstream &datasource_names_input_stream) // NB Cannot be written without templated type because of cyclic depencies between // `static_rtree.hpp` and `io.hpp` template -void readRamIndex(boost::filesystem::ifstream &ram_index_input_stream, - RTreeNodeT *rtree_buffer, - const std::uint64_t tree_size) +void readRamIndex(File &ram_index_file, RTreeNodeT *rtree_buffer, const std::uint64_t tree_size) { BOOST_ASSERT(rtree_buffer); - ram_index_input_stream.read(reinterpret_cast(rtree_buffer), - sizeof(RTreeNodeT) * tree_size); + ram_index_file.readInto(rtree_buffer, tree_size); } } } diff --git a/include/util/guidance/turn_lanes.hpp b/include/util/guidance/turn_lanes.hpp index 69d2c5926..34f047f2f 100644 --- a/include/util/guidance/turn_lanes.hpp +++ b/include/util/guidance/turn_lanes.hpp @@ -17,6 +17,7 @@ namespace util namespace guidance { class LaneTuple; +class LaneTupleIdPair; } // namespace guidance } // namespace util } // namespace osrm @@ -27,6 +28,11 @@ template <> struct hash<::osrm::util::guidance::LaneTuple> { inline std::size_t operator()(const ::osrm::util::guidance::LaneTuple &bearing_class) const; }; +template <> struct hash<::osrm::util::guidance::LaneTupleIdPair> +{ + inline std::size_t + operator()(const ::osrm::util::guidance::LaneTupleIdPair &bearing_class) const; +}; } // namespace std namespace osrm @@ -73,7 +79,23 @@ class LaneTuple } }; -using LaneTupleIdPair = std::pair; +class LaneTupleIdPair +{ + public: + util::guidance::LaneTuple first; + LaneDescriptionID second; + + bool operator==(const LaneTupleIdPair &other) const; + + friend std::size_t hash_value(const LaneTupleIdPair &pair) + { + std::size_t seed{0}; + boost::hash_combine(seed, pair.first); + boost::hash_combine(seed, pair.second); + return seed; + } +}; + } // namespace guidance } // namespace util } // namespace osrm diff --git a/include/util/io.hpp b/include/util/io.hpp index e208f7c25..6436748fe 100644 --- a/include/util/io.hpp +++ b/include/util/io.hpp @@ -15,6 +15,7 @@ #include #include "util/fingerprint.hpp" +#include "storage/io.hpp" namespace osrm { @@ -28,17 +29,6 @@ inline bool writeFingerprint(std::ostream &stream) return static_cast(stream); } -inline bool readAndCheckFingerprint(std::istream &stream) -{ - FingerPrint fingerprint; - const auto valid = FingerPrint::GetValid(); - stream.read(reinterpret_cast(&fingerprint), sizeof(fingerprint)); - // compare the compilation state stored in the fingerprint - return static_cast(stream) && valid.IsMagicNumberOK(fingerprint) && - valid.TestContractor(fingerprint) && valid.TestGraphUtil(fingerprint) && - valid.TestRTree(fingerprint) && valid.TestQueryObjects(fingerprint); -} - template bool serializeVector(const std::string &filename, const std::vector &data) { @@ -66,17 +56,14 @@ bool serializeVector(std::ostream &stream, const std::vector &data) template bool deserializeVector(const std::string &filename, std::vector &data) { - std::ifstream stream(filename, std::ios::binary); - if (!readAndCheckFingerprint(stream)) - return false; + storage::io::File file(filename, true); - std::uint64_t count = 0; - stream.read(reinterpret_cast(&count), sizeof(count)); + const auto count = file.readElementCount64(); data.resize(count); if (count) - stream.read(reinterpret_cast(&data[0]), sizeof(simple_type) * count); - return static_cast(stream); + file.readInto(data.data(), count); + return true; } template @@ -199,13 +186,9 @@ inline bool serializeFlags(const boost::filesystem::path &path, const std::vecto inline bool deserializeFlags(const boost::filesystem::path &path, std::vector &flags) { SimpleLogger().Write() << "Reading flags from " << path; - std::ifstream flag_stream(path.string(), std::ios::binary); + storage::io::File flag_file(path, true); - if (!readAndCheckFingerprint(flag_stream)) - return false; - - std::uint32_t number_of_bits; - flag_stream.read(reinterpret_cast(&number_of_bits), sizeof(number_of_bits)); + const auto number_of_bits = flag_file.readOne(); flags.resize(number_of_bits); // putting bits in ints std::uint32_t chunks = (number_of_bits + 31) / 32; @@ -213,14 +196,14 @@ inline bool deserializeFlags(const boost::filesystem::path &path, std::vector(&chunk), sizeof(chunk)); + flag_file.readInto(chunk); std::bitset<32> chunk_bits(chunk); for (std::size_t bit = 0; bit < 32 && bit_position < number_of_bits; ++bit, ++bit_position) flags[bit_position] = chunk_bits[bit]; } SimpleLogger().Write() << "Read " << number_of_bits << " bits in " << chunks << " Chunks from disk."; - return static_cast(flag_stream); + return true; } } // namespace util } // namespace osrm diff --git a/include/util/static_rtree.hpp b/include/util/static_rtree.hpp index ef6145df8..19a0c69c5 100644 --- a/include/util/static_rtree.hpp +++ b/include/util/static_rtree.hpp @@ -346,18 +346,9 @@ class StaticRTree const CoordinateListT &coordinate_list) : m_coordinate_list(coordinate_list) { - // open tree node file and load into RAM. - if (!boost::filesystem::exists(node_file)) - { - throw exception("ram index file does not exist"); - } - if (boost::filesystem::file_size(node_file) == 0) - { - throw exception("ram index file is empty"); - } - boost::filesystem::ifstream tree_node_file(node_file, std::ios::binary); + storage::io::File tree_node_file(node_file); - const auto tree_size = storage::io::readElementCount64(tree_node_file); + const auto tree_size = tree_node_file.readElementCount64(); m_search_tree.resize(tree_size); storage::io::readRamIndex(tree_node_file, &m_search_tree[0], tree_size); diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index fc70bfcb5..8fb639987 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -28,13 +28,11 @@ #endif #include -#include #include #include #include #include #include -#include #include @@ -263,19 +261,15 @@ void Storage::LoadLayout(DataLayout *layout_ptr) // collect number of elements to store in shared memory object util::SimpleLogger().Write() << "load names from: " << config.names_data_path; // number of entries in name index - boost::filesystem::ifstream name_stream(config.names_data_path, std::ios::binary); - if (!name_stream) - { - throw util::exception("Could not open " + config.names_data_path.string() + - " for reading."); - } - const auto name_blocks = io::readElementCount32(name_stream); + io::File name_file(config.names_data_path); + + const auto name_blocks = name_file.readElementCount32(); layout_ptr->SetBlockSize(DataLayout::NAME_OFFSETS, name_blocks); layout_ptr->SetBlockSize::BlockT>( DataLayout::NAME_BLOCKS, name_blocks); BOOST_ASSERT_MSG(0 != name_blocks, "name file broken"); - const auto number_of_chars = io::readElementCount32(name_stream); + const auto number_of_chars = name_file.readElementCount32(); layout_ptr->SetBlockSize(DataLayout::NAME_CHAR_LIST, number_of_chars); } @@ -295,13 +289,8 @@ void Storage::LoadLayout(DataLayout *layout_ptr) // Loading information for original edges { - boost::filesystem::ifstream edges_input_stream(config.edges_data_path, std::ios::binary); - if (!edges_input_stream) - { - throw util::exception("Could not open " + config.edges_data_path.string() + - " for reading."); - } - const auto number_of_original_edges = io::readElementCount64(edges_input_stream); + io::File edges_file(config.edges_data_path); + const auto number_of_original_edges = edges_file.readElementCount64(); // note: settings this all to the same size is correct, we extract them from the same struct layout_ptr->SetBlockSize(DataLayout::VIA_NODE_LIST, number_of_original_edges); @@ -319,14 +308,9 @@ void Storage::LoadLayout(DataLayout *layout_ptr) } { - boost::filesystem::ifstream hsgr_input_stream(config.hsgr_data_path, std::ios::binary); - if (!hsgr_input_stream) - { - throw util::exception("Could not open " + config.hsgr_data_path.string() + - " for reading."); - } + io::File hsgr_file(config.hsgr_data_path); - const auto hsgr_header = io::readHSGRHeader(hsgr_input_stream); + const auto hsgr_header = io::readHSGRHeader(hsgr_file); layout_ptr->SetBlockSize(DataLayout::HSGR_CHECKSUM, 1); layout_ptr->SetBlockSize(DataLayout::GRAPH_NODE_LIST, hsgr_header.number_of_nodes); @@ -336,11 +320,13 @@ void Storage::LoadLayout(DataLayout *layout_ptr) // load rsearch tree size { - boost::filesystem::ifstream tree_node_file(config.ram_index_path, std::ios::binary); + io::File tree_node_file(config.ram_index_path); - const auto tree_size = io::readElementCount64(tree_node_file); + const auto tree_size = tree_node_file.readElementCount64(); layout_ptr->SetBlockSize(DataLayout::R_SEARCH_TREE, tree_size); + } + { // allocate space in shared memory for profile properties const auto properties_size = io::readPropertiesCount(); layout_ptr->SetBlockSize(DataLayout::PROPERTIES, @@ -349,38 +335,22 @@ void Storage::LoadLayout(DataLayout *layout_ptr) // read timestampsize { - boost::filesystem::ifstream timestamp_stream(config.timestamp_path); - if (!timestamp_stream) - { - throw util::exception("Could not open " + config.timestamp_path.string() + - " for reading."); - } - const auto timestamp_size = io::readNumberOfBytes(timestamp_stream); + io::File timestamp_file(config.timestamp_path); + const auto timestamp_size = timestamp_file.size(); layout_ptr->SetBlockSize(DataLayout::TIMESTAMP, timestamp_size); } // load core marker size { - boost::filesystem::ifstream core_marker_file(config.core_data_path, std::ios::binary); - if (!core_marker_file) - { - throw util::exception("Could not open " + config.core_data_path.string() + - " for reading."); - } - - const auto number_of_core_markers = io::readElementCount32(core_marker_file); + io::File core_marker_file(config.core_data_path); + const auto number_of_core_markers = core_marker_file.readElementCount32(); layout_ptr->SetBlockSize(DataLayout::CORE_MARKER, number_of_core_markers); } // load coordinate size { - boost::filesystem::ifstream nodes_input_stream(config.nodes_data_path, std::ios::binary); - if (!nodes_input_stream) - { - throw util::exception("Could not open " + config.core_data_path.string() + - " for reading."); - } - const auto coordinate_list_size = io::readElementCount64(nodes_input_stream); + io::File node_file(config.nodes_data_path); + const auto coordinate_list_size = node_file.readElementCount64(); layout_ptr->SetBlockSize(DataLayout::COORDINATE_LIST, coordinate_list_size); // we'll read a list of OSM node IDs from the same data, so set the block size for the same @@ -392,20 +362,15 @@ void Storage::LoadLayout(DataLayout *layout_ptr) // load geometries sizes { - boost::filesystem::ifstream geometry_input_stream(config.geometries_path, std::ios::binary); - if (!geometry_input_stream) - { - throw util::exception("Could not open " + config.geometries_path.string() + - " for reading."); - } + io::File geometry_file(config.geometries_path); - const auto number_of_geometries_indices = io::readElementCount32(geometry_input_stream); + const auto number_of_geometries_indices = geometry_file.readElementCount32(); layout_ptr->SetBlockSize(DataLayout::GEOMETRIES_INDEX, number_of_geometries_indices); - boost::iostreams::seek( - geometry_input_stream, number_of_geometries_indices * sizeof(unsigned), BOOST_IOS::cur); - const auto number_of_compressed_geometries = io::readElementCount32(geometry_input_stream); + geometry_file.skip(number_of_geometries_indices); + + const auto number_of_compressed_geometries = geometry_file.readElementCount32(); layout_ptr->SetBlockSize(DataLayout::GEOMETRIES_NODE_LIST, number_of_compressed_geometries); layout_ptr->SetBlockSize(DataLayout::GEOMETRIES_FWD_WEIGHT_LIST, @@ -417,15 +382,8 @@ void Storage::LoadLayout(DataLayout *layout_ptr) // load datasource sizes. This file is optional, and it's non-fatal if it doesn't // exist. { - boost::filesystem::ifstream geometry_datasource_input_stream(config.datasource_indexes_path, - std::ios::binary); - if (!geometry_datasource_input_stream) - { - throw util::exception("Could not open " + config.datasource_indexes_path.string() + - " for reading."); - } - const auto number_of_compressed_datasources = - io::readElementCount64(geometry_datasource_input_stream); + io::File geometry_datasource_file(config.datasource_indexes_path); + const auto number_of_compressed_datasources = geometry_datasource_file.readElementCount64(); layout_ptr->SetBlockSize(DataLayout::DATASOURCES_LIST, number_of_compressed_datasources); } @@ -433,16 +391,10 @@ void Storage::LoadLayout(DataLayout *layout_ptr) // Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't // exist { - boost::filesystem::ifstream datasource_names_input_stream(config.datasource_names_path, - std::ios::binary); - if (!datasource_names_input_stream) - { - throw util::exception("Could not open " + config.datasource_names_path.string() + - " for reading."); - } + io::File datasource_names_file(config.datasource_names_path); const io::DatasourceNamesData datasource_names_data = - io::readDatasourceNames(datasource_names_input_stream); + io::readDatasourceNames(datasource_names_file); layout_ptr->SetBlockSize(DataLayout::DATASOURCE_NAME_DATA, datasource_names_data.names.size()); @@ -453,54 +405,34 @@ void Storage::LoadLayout(DataLayout *layout_ptr) } { - boost::filesystem::ifstream intersection_stream(config.intersection_class_path, - std::ios::binary); - - if (!static_cast(intersection_stream)) - throw util::exception("Could not open " + config.intersection_class_path.string() + - " for reading."); - - if (!util::readAndCheckFingerprint(intersection_stream)) - throw util::exception("Fingerprint of " + config.intersection_class_path.string() + - " does not match or could not read from file"); + io::File intersection_file(config.intersection_class_path, true); std::vector bearing_class_id_table; - if (!util::deserializeVector(intersection_stream, bearing_class_id_table)) - throw util::exception("Failed to bearing class ids read from " + - config.names_data_path.string()); + intersection_file.deserializeVector(bearing_class_id_table); layout_ptr->SetBlockSize(DataLayout::BEARING_CLASSID, bearing_class_id_table.size()); - const auto bearing_blocks = io::readElementCount32(intersection_stream); - const auto sum_lengths = io::readElementCount32(intersection_stream); + const auto bearing_blocks = intersection_file.readElementCount32(); + intersection_file.skip(1); // sum_lengths layout_ptr->SetBlockSize(DataLayout::BEARING_OFFSETS, bearing_blocks); layout_ptr->SetBlockSize::BlockT>( DataLayout::BEARING_BLOCKS, bearing_blocks); - // Skip the actual data - boost::iostreams::seek(intersection_stream, - bearing_blocks * sizeof(unsigned) + - bearing_blocks * - sizeof(typename util::RangeTable<16, true>::BlockT), - BOOST_IOS::cur); + // No need to read the data + intersection_file.skip(bearing_blocks); + intersection_file.skip::BlockT>(bearing_blocks); - const auto num_bearings = io::readElementCount64(intersection_stream); + const auto num_bearings = intersection_file.readElementCount64(); // Skip over the actual data - boost::iostreams::seek( - intersection_stream, num_bearings * sizeof(DiscreteBearing), BOOST_IOS::cur); + intersection_file.skip(num_bearings); + layout_ptr->SetBlockSize(DataLayout::BEARING_VALUES, num_bearings); - if (!static_cast(intersection_stream)) - throw util::exception("Failed to read bearing values from " + - config.intersection_class_path.string()); - std::vector entry_class_table; - if (!util::deserializeVector(intersection_stream, entry_class_table)) - throw util::exception("Failed to read entry classes from " + - config.intersection_class_path.string()); + intersection_file.deserializeVector(entry_class_table); layout_ptr->SetBlockSize(DataLayout::ENTRY_CLASS, entry_class_table.size()); @@ -508,10 +440,10 @@ void Storage::LoadLayout(DataLayout *layout_ptr) { // Loading turn lane data - boost::filesystem::ifstream lane_data_stream(config.turn_lane_data_path, std::ios::binary); - const auto lane_tupel_count = io::readElementCount64(lane_data_stream); + io::File lane_data_file(config.turn_lane_data_path); + const auto lane_tuple_count = lane_data_file.readElementCount64(); layout_ptr->SetBlockSize(DataLayout::TURN_LANE_DATA, - lane_tupel_count); + lane_tuple_count); } } @@ -522,8 +454,8 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr) // Load the HSGR file { - boost::filesystem::ifstream hsgr_input_stream(config.hsgr_data_path, std::ios::binary); - auto hsgr_header = io::readHSGRHeader(hsgr_input_stream); + io::File hsgr_file(config.hsgr_data_path); + auto hsgr_header = io::readHSGRHeader(hsgr_file); unsigned *checksum_ptr = layout_ptr->GetBlockPtr(memory_ptr, DataLayout::HSGR_CHECKSUM); *checksum_ptr = hsgr_header.checksum; @@ -538,12 +470,11 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr) layout_ptr->GetBlockPtr(memory_ptr, DataLayout::GRAPH_EDGE_LIST); - io::readHSGR(hsgr_input_stream, + io::readHSGR(hsgr_file, graph_node_list_ptr, hsgr_header.number_of_nodes, graph_edge_list_ptr, hsgr_header.number_of_edges); - hsgr_input_stream.close(); } // store the filename of the on-disk portion of the RTree @@ -562,61 +493,54 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr) // Name data { - /* To be replaced by io loading code */ - boost::filesystem::ifstream name_stream(config.names_data_path, std::ios::binary); - const auto name_blocks = io::readElementCount32(name_stream); - const auto number_of_chars = io::readElementCount32(name_stream); - /* END to be replaced */ + io::File name_file(config.names_data_path); + const auto name_blocks_count = name_file.readElementCount32(); + const auto name_char_list_count = name_file.readElementCount32(); + + using NameRangeTable = util::RangeTable<16, true>; + + BOOST_ASSERT(name_blocks_count * sizeof(unsigned) == + layout_ptr->GetBlockSize(DataLayout::NAME_OFFSETS)); + BOOST_ASSERT(name_blocks_count * sizeof(typename NameRangeTable::BlockT) == + layout_ptr->GetBlockSize(DataLayout::NAME_BLOCKS)); // Loading street names const auto name_offsets_ptr = layout_ptr->GetBlockPtr(memory_ptr, DataLayout::NAME_OFFSETS); - if (layout_ptr->GetBlockSize(DataLayout::NAME_OFFSETS) > 0) - { - name_stream.read(reinterpret_cast(name_offsets_ptr), - layout_ptr->GetBlockSize(DataLayout::NAME_OFFSETS)); - } + name_file.readInto(name_offsets_ptr, name_blocks_count); const auto name_blocks_ptr = layout_ptr->GetBlockPtr(memory_ptr, DataLayout::NAME_BLOCKS); - if (layout_ptr->GetBlockSize(DataLayout::NAME_BLOCKS) > 0) - { - name_stream.read(reinterpret_cast(name_blocks_ptr), - layout_ptr->GetBlockSize(DataLayout::NAME_BLOCKS)); - } + name_file.readInto(reinterpret_cast(name_blocks_ptr), + layout_ptr->GetBlockSize(DataLayout::NAME_BLOCKS)); + + // For some reason, this value (the same as ) + const auto temp_count = name_file.readElementCount32(); const auto name_char_ptr = layout_ptr->GetBlockPtr(memory_ptr, DataLayout::NAME_CHAR_LIST); - const auto name_char_list_count = io::readElementCount32(name_stream); - - BOOST_ASSERT_MSG(layout_ptr->AlignBlockSize(name_char_list_count) == + BOOST_ASSERT_MSG(layout_ptr->AlignBlockSize(temp_count) == layout_ptr->GetBlockSize(DataLayout::NAME_CHAR_LIST), "Name file corrupted!"); - if (layout_ptr->GetBlockSize(DataLayout::NAME_CHAR_LIST) > 0) - { - name_stream.read(name_char_ptr, layout_ptr->GetBlockSize(DataLayout::NAME_CHAR_LIST)); - } + name_file.readInto(name_char_ptr, temp_count); } // Turn lane data { - /* NOTE: file io - refactor this in the future */ - boost::filesystem::ifstream lane_data_stream(config.turn_lane_data_path, std::ios::binary); - const auto lane_tupel_count = io::readElementCount64(lane_data_stream); - /* END NOTE */ + io::File lane_data_file(config.turn_lane_data_path); + + const auto lane_tuple_count = lane_data_file.readElementCount64(); // Need to call GetBlockPtr -> it write the memory canary, even if no data needs to be // loaded. const auto turn_lane_data_ptr = layout_ptr->GetBlockPtr( memory_ptr, DataLayout::TURN_LANE_DATA); - if (layout_ptr->GetBlockSize(DataLayout::TURN_LANE_DATA) > 0) - { - lane_data_stream.read(reinterpret_cast(turn_lane_data_ptr), - layout_ptr->GetBlockSize(DataLayout::TURN_LANE_DATA)); - } + BOOST_ASSERT(lane_tuple_count * sizeof(util::guidance::LaneTupleIdPair) == + layout_ptr->GetBlockSize(DataLayout::TURN_LANE_DATA)); + lane_data_file.readInto(turn_lane_data_ptr, lane_tuple_count); } // Turn lane descriptions @@ -655,16 +579,9 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr) // Load original edge data { - /* NOTE: file io - refactor this in the future */ - boost::filesystem::ifstream edges_input_stream(config.edges_data_path, std::ios::binary); - if (!edges_input_stream) - { - throw util::exception("Could not open " + config.edges_data_path.string() + - " for reading."); - } + io::File edges_input_file(config.edges_data_path); - const auto number_of_original_edges = io::readElementCount64(edges_input_stream); - /* END NOTE */ + const auto number_of_original_edges = edges_input_file.readElementCount64(); const auto via_geometry_ptr = layout_ptr->GetBlockPtr(memory_ptr, DataLayout::VIA_NODE_LIST); @@ -691,7 +608,7 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr) const auto entry_class_id_ptr = layout_ptr->GetBlockPtr(memory_ptr, DataLayout::ENTRY_CLASSID); - io::readEdges(edges_input_stream, + io::readEdges(edges_input_file, via_geometry_ptr, name_id_ptr, turn_instructions_ptr, @@ -705,91 +622,53 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr) // load compressed geometry { - /* NOTE: file io - refactor this in the future */ - boost::filesystem::ifstream geometry_input_stream(config.geometries_path, std::ios::binary); - /* END NOTE */ + io::File geometry_input_file(config.geometries_path); - const auto geometry_index_count = io::readElementCount32(geometry_input_stream); + const auto geometry_index_count = geometry_input_file.readElementCount32(); const auto geometries_index_ptr = layout_ptr->GetBlockPtr(memory_ptr, DataLayout::GEOMETRIES_INDEX); BOOST_ASSERT(geometry_index_count == layout_ptr->num_entries[DataLayout::GEOMETRIES_INDEX]); + geometry_input_file.readInto(geometries_index_ptr, geometry_index_count); - if (layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_INDEX) > 0) - { - geometry_input_stream.read(reinterpret_cast(geometries_index_ptr), - layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_INDEX)); - } const auto geometries_node_id_list_ptr = layout_ptr->GetBlockPtr(memory_ptr, DataLayout::GEOMETRIES_NODE_LIST); - - const auto geometry_node_lists_count = io::readElementCount32(geometry_input_stream); + const auto geometry_node_lists_count = geometry_input_file.readElementCount32(); BOOST_ASSERT(geometry_node_lists_count == layout_ptr->num_entries[DataLayout::GEOMETRIES_NODE_LIST]); + geometry_input_file.readInto(geometries_node_id_list_ptr, geometry_node_lists_count); - if (layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_NODE_LIST) > 0) - { - geometry_input_stream.read(reinterpret_cast(geometries_node_id_list_ptr), - layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_NODE_LIST)); - } const auto geometries_fwd_weight_list_ptr = layout_ptr->GetBlockPtr( memory_ptr, DataLayout::GEOMETRIES_FWD_WEIGHT_LIST); - BOOST_ASSERT(geometry_node_lists_count == layout_ptr->num_entries[DataLayout::GEOMETRIES_FWD_WEIGHT_LIST]); + geometry_input_file.readInto(geometries_fwd_weight_list_ptr, geometry_node_lists_count); - if (layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_FWD_WEIGHT_LIST) > 0) - { - geometry_input_stream.read( - reinterpret_cast(geometries_fwd_weight_list_ptr), - layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_FWD_WEIGHT_LIST)); - } const auto geometries_rev_weight_list_ptr = layout_ptr->GetBlockPtr( memory_ptr, DataLayout::GEOMETRIES_REV_WEIGHT_LIST); - BOOST_ASSERT(geometry_node_lists_count == layout_ptr->num_entries[DataLayout::GEOMETRIES_REV_WEIGHT_LIST]); - - if (layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_REV_WEIGHT_LIST) > 0) - { - geometry_input_stream.read( - reinterpret_cast(geometries_rev_weight_list_ptr), - layout_ptr->GetBlockSize(DataLayout::GEOMETRIES_REV_WEIGHT_LIST)); - } + geometry_input_file.readInto(geometries_rev_weight_list_ptr, geometry_node_lists_count); } { - boost::filesystem::ifstream geometry_datasource_input_stream(config.datasource_indexes_path, - std::ios::binary); - if (!geometry_datasource_input_stream) - { - throw util::exception("Could not open " + config.datasource_indexes_path.string() + - " for reading."); - } - const auto number_of_compressed_datasources = - io::readElementCount64(geometry_datasource_input_stream); + io::File geometry_datasource_file(config.datasource_indexes_path); + const auto number_of_compressed_datasources = geometry_datasource_file.readElementCount64(); // load datasource information (if it exists) const auto datasources_list_ptr = layout_ptr->GetBlockPtr(memory_ptr, DataLayout::DATASOURCES_LIST); if (number_of_compressed_datasources > 0) { - io::readDatasourceIndexes(geometry_datasource_input_stream, - datasources_list_ptr, - number_of_compressed_datasources); + io::readDatasourceIndexes( + geometry_datasource_file, datasources_list_ptr, number_of_compressed_datasources); } } { /* Load names */ - boost::filesystem::ifstream datasource_names_input_stream(config.datasource_names_path, - std::ios::binary); - if (!datasource_names_input_stream) - { - throw util::exception("Could not open " + config.datasource_names_path.string() + - " for reading."); - } + io::File datasource_names_file(config.datasource_names_path); - const auto datasource_names_data = io::readDatasourceNames(datasource_names_input_stream); + const auto datasource_names_data = io::readDatasourceNames(datasource_names_file); // load datasource name information (if it exists) const auto datasource_name_data_ptr = @@ -822,8 +701,8 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr) // Loading list of coordinates { - boost::filesystem::ifstream nodes_input_stream(config.nodes_data_path, std::ios::binary); - io::readElementCount64(nodes_input_stream); + io::File nodes_file(config.nodes_data_path); + nodes_file.skip(1); // node_count const auto coordinates_ptr = layout_ptr->GetBlockPtr( memory_ptr, DataLayout::COORDINATE_LIST); const auto osmnodeid_ptr = @@ -832,7 +711,7 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr) osmnodeid_list.reset(osmnodeid_ptr, layout_ptr->num_entries[DataLayout::OSM_NODE_ID_LIST]); - io::readNodes(nodes_input_stream, + io::readNodes(nodes_file, coordinates_ptr, osmnodeid_list, layout_ptr->num_entries[DataLayout::COORDINATE_LIST]); @@ -840,22 +719,21 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr) // store timestamp { - /* NOTE: file io - refactor this in the future */ - boost::filesystem::ifstream timestamp_stream(config.timestamp_path); - const auto timestamp_size = io::readNumberOfBytes(timestamp_stream); - /* END NOTE */ + io::File timestamp_file(config.timestamp_path); + const auto timestamp_size = timestamp_file.size(); const auto timestamp_ptr = layout_ptr->GetBlockPtr(memory_ptr, DataLayout::TIMESTAMP); - io::readTimestamp(timestamp_stream, timestamp_ptr, timestamp_size); + BOOST_ASSERT(timestamp_size == layout_ptr->num_entries[DataLayout::TIMESTAMP]); + timestamp_file.readInto(timestamp_ptr, timestamp_size); } // store search tree portion of rtree { - boost::filesystem::ifstream tree_node_file(config.ram_index_path, std::ios::binary); + io::File tree_node_file(config.ram_index_path); // perform this read so that we're at the right stream position for the next // read. - io::readElementCount64(tree_node_file); + tree_node_file.skip(1); const auto rtree_ptr = layout_ptr->GetBlockPtr(memory_ptr, DataLayout::R_SEARCH_TREE); io::readRamIndex( @@ -863,21 +741,12 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr) } { - /* NOTE: file io - refactor this in the future */ - boost::filesystem::ifstream core_marker_file(config.core_data_path, std::ios::binary); - if (!core_marker_file) - { - throw util::exception("Could not open " + config.core_data_path.string() + - " for reading."); - } - - const auto number_of_core_markers = io::readElementCount32(core_marker_file); - /* END NOTE */ + io::File core_marker_file(config.core_data_path); + const auto number_of_core_markers = core_marker_file.readElementCount32(); // load core markers std::vector unpacked_core_markers(number_of_core_markers); - core_marker_file.read(reinterpret_cast(unpacked_core_markers.data()), - sizeof(char) * number_of_core_markers); + core_marker_file.readInto(unpacked_core_markers.data(), number_of_core_markers); const auto core_marker_ptr = layout_ptr->GetBlockPtr(memory_ptr, DataLayout::CORE_MARKER); @@ -906,68 +775,38 @@ void Storage::LoadData(DataLayout *layout_ptr, char *memory_ptr) // load profile properties { - boost::filesystem::ifstream profile_properties_stream(config.properties_path); - if (!profile_properties_stream) - { - util::exception("Could not open " + config.properties_path.string() + " for reading!"); - } - io::readPropertiesCount(); + io::File profile_properties_file(config.properties_path); const auto profile_properties_ptr = layout_ptr->GetBlockPtr(memory_ptr, DataLayout::PROPERTIES); - io::readProperties(profile_properties_stream, - profile_properties_ptr, - sizeof(extractor::ProfileProperties)); + const auto properties_size = io::readPropertiesCount(); + io::readProperties(profile_properties_file, profile_properties_ptr, properties_size); } // Load intersection data { - boost::filesystem::ifstream intersection_stream(config.intersection_class_path, - std::ios::binary); - if (!static_cast(intersection_stream)) - throw util::exception("Could not open " + config.intersection_class_path.string() + - " for reading."); - - if (!util::readAndCheckFingerprint(intersection_stream)) - throw util::exception("Fingerprint of " + config.intersection_class_path.string() + - " does not match or could not read from file"); + io::File intersection_file(config.intersection_class_path, true); std::vector bearing_class_id_table; - if (!util::deserializeVector(intersection_stream, bearing_class_id_table)) - throw util::exception("Failed to bearing class ids read from " + - config.names_data_path.string()); + intersection_file.deserializeVector(bearing_class_id_table); - const auto bearing_blocks = io::readElementCount32(intersection_stream); - const auto sum_lengths = io::readElementCount32(intersection_stream); + const auto bearing_blocks = intersection_file.readElementCount32(); + intersection_file.skip(1); // sum_lengths std::vector bearing_offsets_data(bearing_blocks); std::vector::BlockT> bearing_blocks_data( bearing_blocks); - if (bearing_blocks) - { - intersection_stream.read(reinterpret_cast(bearing_offsets_data.data()), - bearing_blocks * - sizeof(decltype(bearing_offsets_data)::value_type)); - } + intersection_file.readInto(bearing_offsets_data.data(), bearing_blocks); + intersection_file.readInto(bearing_blocks_data.data(), bearing_blocks); - if (bearing_blocks) - { - intersection_stream.read(reinterpret_cast(bearing_blocks_data.data()), - bearing_blocks * - sizeof(decltype(bearing_blocks_data)::value_type)); - } - - const auto num_bearings = io::readElementCount64(intersection_stream); + const auto num_bearings = intersection_file.readElementCount64(); std::vector bearing_class_table(num_bearings); - intersection_stream.read(reinterpret_cast(bearing_class_table.data()), - sizeof(decltype(bearing_class_table)::value_type) * num_bearings); + intersection_file.readInto(bearing_class_table.data(), num_bearings); std::vector entry_class_table; - if (!util::deserializeVector(intersection_stream, entry_class_table)) - throw util::exception("Failed to read entry classes from " + - config.intersection_class_path.string()); + intersection_file.deserializeVector(entry_class_table); // load intersection classes if (!bearing_class_id_table.empty()) diff --git a/src/util/guidance/turn_lanes.cpp b/src/util/guidance/turn_lanes.cpp index 941be8129..cad86c059 100644 --- a/src/util/guidance/turn_lanes.cpp +++ b/src/util/guidance/turn_lanes.cpp @@ -31,6 +31,11 @@ bool LaneTuple::operator==(const LaneTuple other) const bool LaneTuple::operator!=(const LaneTuple other) const { return !(*this == other); } +bool LaneTupleIdPair::operator==(const LaneTupleIdPair &other) const +{ + return other.first == first && other.second == second; +} + } // namespace guidance } // namespace util } // namespace osrm