From cf35bbeb50824b38bad9154b44899000bc3a5cc6 Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Fri, 21 Oct 2016 15:24:55 -0700 Subject: [PATCH] refactor function names; consolidate readCount() functions; remove templated types as much as possible for type safety; add more comments; clean up code, add const if possible; --- .../engine/datafacade/internal_datafacade.hpp | 42 ++-- include/extractor/original_edge_data.hpp | 2 +- include/storage/io.hpp | 210 ++++++++---------- include/util/static_rtree.hpp | 9 +- src/contractor/contractor.cpp | 22 +- src/extractor/edge_based_graph_factory.cpp | 4 +- src/extractor/extractor.cpp | 7 +- src/storage/storage.cpp | 71 +++--- 8 files changed, 169 insertions(+), 198 deletions(-) diff --git a/include/engine/datafacade/internal_datafacade.hpp b/include/engine/datafacade/internal_datafacade.hpp index 62213eee5..c43d258e5 100644 --- a/include/engine/datafacade/internal_datafacade.hpp +++ b/include/engine/datafacade/internal_datafacade.hpp @@ -123,8 +123,8 @@ class InternalDataFacade final : public BaseDataFacade { throw util::exception("Could not open " + properties_path.string() + " for reading."); } - auto PropertiesSize = storage::io::readPropertiesSize(); - storage::io::readProperties(in_stream, &m_profile_properties, PropertiesSize); + const auto properties_size = storage::io::readPropertiesCount(); + storage::io::readProperties(in_stream, &m_profile_properties, properties_size); } void LoadLaneTupleIdPairs(const boost::filesystem::path &lane_data_path) @@ -151,7 +151,7 @@ class InternalDataFacade final : public BaseDataFacade throw util::exception("Could not open " + timestamp_path.string() + " for reading."); } - auto timestamp_size = storage::io::readTimestampSize(timestamp_stream); + const auto timestamp_size = storage::io::readNumberOfBytes(timestamp_stream); m_timestamp.resize(timestamp_size); storage::io::readTimestamp(timestamp_stream, &m_timestamp.front(), timestamp_size); } @@ -164,7 +164,7 @@ class InternalDataFacade final : public BaseDataFacade throw util::exception("Could not open " + hsgr_path.string() + " for reading."); } - auto header = storage::io::readHSGRHeader(hsgr_input_stream); + const auto header = storage::io::readHSGRHeader(hsgr_input_stream); m_check_sum = header.checksum; util::ShM::vector node_list(header.number_of_nodes); @@ -190,10 +190,10 @@ class InternalDataFacade final : public BaseDataFacade throw util::exception("Could not open " + nodes_file_path.string() + " for reading."); } - std::uint32_t number_of_coordinates = storage::io::readNodesSize(nodes_input_stream); + const auto number_of_coordinates = storage::io::readElementCount(nodes_input_stream); m_coordinate_list.resize(number_of_coordinates); m_osmnodeid_list.reserve(number_of_coordinates); - storage::io::readNodesData( + storage::io::readNodes( nodes_input_stream, m_coordinate_list.data(), m_osmnodeid_list, number_of_coordinates); boost::filesystem::ifstream edges_input_stream(edges_file_path, std::ios::binary); @@ -201,7 +201,7 @@ class InternalDataFacade final : public BaseDataFacade { throw util::exception("Could not open " + edges_file_path.string() + " for reading."); } - auto number_of_edges = storage::io::readEdgesSize(edges_input_stream); + const auto number_of_edges = storage::io::readElementCount(edges_input_stream); m_via_geometry_list.resize(number_of_edges); m_name_ID_list.resize(number_of_edges); m_turn_instruction_list.resize(number_of_edges); @@ -211,16 +211,16 @@ class InternalDataFacade final : public BaseDataFacade m_pre_turn_bearing.resize(number_of_edges); m_post_turn_bearing.resize(number_of_edges); - storage::io::readEdgesData(edges_input_stream, - m_via_geometry_list.data(), - m_name_ID_list.data(), - m_turn_instruction_list.data(), - m_lane_data_id.data(), - m_travel_mode_list.data(), - m_entry_class_id_list.data(), - m_pre_turn_bearing.data(), - m_post_turn_bearing.data(), - number_of_edges); + storage::io::readEdges(edges_input_stream, + m_via_geometry_list.data(), + m_name_ID_list.data(), + m_turn_instruction_list.data(), + m_lane_data_id.data(), + m_travel_mode_list.data(), + m_entry_class_id_list.data(), + m_pre_turn_bearing.data(), + m_post_turn_bearing.data(), + number_of_edges); } void LoadCoreInformation(const boost::filesystem::path &core_data_file) @@ -292,7 +292,7 @@ class InternalDataFacade final : public BaseDataFacade } BOOST_ASSERT(datasources_stream); - auto number_of_datasources = storage::io::readDatasourceIndexesSize(datasources_stream); + const auto number_of_datasources = storage::io::readElementCount(datasources_stream); if (number_of_datasources > 0) { m_datasource_list.resize(number_of_datasources); @@ -308,15 +308,15 @@ class InternalDataFacade final : public BaseDataFacade } BOOST_ASSERT(datasourcenames_stream); - auto datasource_names_data = storage::io::readDatasourceNamesData(datasourcenames_stream); + const auto datasource_names_data = storage::io::readDatasourceNames(datasourcenames_stream); m_datasource_names.resize(datasource_names_data.lengths.size()); - for (std::uint32_t i = 0; i < datasource_names_data.lengths.size(); ++i) + for (std::size_t i = 0; i < datasource_names_data.lengths.size(); ++i) { auto name_begin = datasource_names_data.names.begin() + datasource_names_data.offsets[i]; auto name_end = datasource_names_data.names.begin() + datasource_names_data.offsets[i] + datasource_names_data.lengths[i]; - m_datasource_names[i] = std::move(std::string(name_begin, name_end)); + m_datasource_names[i] = std::string(name_begin, name_end); } } diff --git a/include/extractor/original_edge_data.hpp b/include/extractor/original_edge_data.hpp index a8303a3bd..51c22deda 100644 --- a/include/extractor/original_edge_data.hpp +++ b/include/extractor/original_edge_data.hpp @@ -17,7 +17,7 @@ namespace extractor struct OriginalEdgeData { explicit OriginalEdgeData(GeometryID via_geometry, - unsigned name_id, + NameID name_id, LaneDataID lane_data_id, guidance::TurnInstruction turn_instruction, EntryClassID entry_classid, diff --git a/include/storage/io.hpp b/include/storage/io.hpp index 8d61831f4..f7407eb18 100644 --- a/include/storage/io.hpp +++ b/include/storage/io.hpp @@ -1,10 +1,13 @@ #ifndef OSRM_STORAGE_IO_HPP_ #define OSRM_STORAGE_IO_HPP_ +#include "contractor/query_edge.hpp" +#include "extractor/extractor.hpp" #include "extractor/original_edge_data.hpp" #include "extractor/query_node.hpp" #include "util/fingerprint.hpp" #include "util/simple_logger.hpp" +#include "util/static_graph.hpp" #include @@ -17,27 +20,38 @@ namespace storage namespace io { -inline std::size_t readPropertiesSize() { return 1; } - -template -inline void readProperties(boost::filesystem::ifstream &properties_stream, - PropertiesT *properties, - std::size_t properties_size) +// Reads the count of elements that is written in the file header and returns the number +inline std::uint64_t readElementCount(boost::filesystem::ifstream &input_stream) { - properties_stream.read(reinterpret_cast(properties), properties_size); + std::uint64_t number_of_elements = 0; + input_stream.read((char *)&number_of_elements, sizeof(std::uint64_t)); + return number_of_elements; +} + +// To make function calls consistent, this function returns the fixed number of properties +inline std::size_t readPropertiesCount() { return 1; } + +// Returns the number of bytes in a file +inline std::size_t readNumberOfBytes(boost::filesystem::ifstream &input_stream) +{ + input_stream.seekg(0, input_stream.end); + auto length = input_stream.tellg(); + input_stream.seekg(0, input_stream.beg); + return length; } #pragma pack(push, 1) struct HSGRHeader { std::uint32_t checksum; - std::uint32_t number_of_nodes; - std::uint32_t number_of_edges; + std::uint64_t number_of_nodes; + std::uint64_t number_of_edges; }; #pragma pack(pop) -static_assert(sizeof(HSGRHeader) == 12, "HSGRHeader is not packed"); +static_assert(sizeof(HSGRHeader) == 20, "HSGRHeader is not packed"); -// Returns the checksum and the number of nodes and number of edges +// Reads the checksum, number of nodes and number of edges written in the header file of a `.hsgr` +// file and returns them in a HSGRHeader struct inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream) { const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid(); @@ -62,66 +76,65 @@ inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream) return header; } +// Reads the graph data of a `.hsgr` file into memory // Needs to be called after readHSGRHeader() to get the correct offset in the stream -template -void readHSGR(boost::filesystem::ifstream &input_stream, - NodeT node_buffer[], - std::uint32_t number_of_nodes, - EdgeT edge_buffer[], - std::uint32_t number_of_edges) +using NodeT = typename util::StaticGraph::NodeArrayEntry; +using EdgeT = typename util::StaticGraph::EdgeArrayEntry; +inline void readHSGR(boost::filesystem::ifstream &input_stream, + NodeT *node_buffer, + const std::uint64_t number_of_nodes, + EdgeT *edge_buffer, + const std::uint64_t number_of_edges) { BOOST_ASSERT(node_buffer); BOOST_ASSERT(edge_buffer); input_stream.read(reinterpret_cast(node_buffer), number_of_nodes * sizeof(NodeT)); input_stream.read(reinterpret_cast(edge_buffer), number_of_edges * sizeof(EdgeT)); } -// Returns the size of the timestamp in a file -inline std::uint32_t readTimestampSize(boost::filesystem::ifstream ×tamp_input_stream) + +// Loads properties from a `.properties` file into memory +inline void readProperties(boost::filesystem::ifstream &properties_stream, + extractor::ProfileProperties *properties, + const std::size_t properties_size) { - timestamp_input_stream.seekg(0, timestamp_input_stream.end); - auto length = timestamp_input_stream.tellg(); - timestamp_input_stream.seekg(0, timestamp_input_stream.beg); - return length; + BOOST_ASSERT(properties); + properties_stream.read(reinterpret_cast(properties), properties_size); } -// Reads the timestamp in a file +// Reads the timestamp in a `.timestamp` file +// Use readNumberOfBytes() beforehand to get the length of the file inline void readTimestamp(boost::filesystem::ifstream ×tamp_input_stream, - char timestamp[], - std::size_t timestamp_length) + char *timestamp, + const std::size_t timestamp_length) { BOOST_ASSERT(timestamp); timestamp_input_stream.read(timestamp, timestamp_length * sizeof(char)); } -// Returns the number of edges in a .edges file -inline std::uint32_t readEdgesSize(boost::filesystem::ifstream &edges_input_stream) +// Loads datasource_indexes from .datasource_indexes into memory +// Needs to be called after readElementCount() to get the correct offset in the stream +inline void readDatasourceIndexes(boost::filesystem::ifstream &datasource_indexes_input_stream, + uint8_t *datasource_buffer, + const std::uint64_t number_of_datasource_indexes) { - std::uint32_t number_of_edges; - edges_input_stream.read((char *)&number_of_edges, sizeof(std::uint32_t)); - return number_of_edges; + BOOST_ASSERT(datasource_buffer); + datasource_indexes_input_stream.read(reinterpret_cast(datasource_buffer), + number_of_datasource_indexes * sizeof(std::uint8_t)); } -// Reads edge data from .edge files which includes its +// Loads edge data from .edge files into memory which includes its // geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID -// Needs to be called after readEdgesSize() to get the correct offset in the stream -template -void readEdgesData(boost::filesystem::ifstream &edges_input_stream, - GeometryIDT geometry_list[], - NameIDT name_id_list[], - TurnInstructionT turn_instruction_list[], - LaneDataIDT lane_data_id_list[], - TravelModeT travel_mode_list[], - EntryClassIDT entry_class_id_list[], - PreTurnBearingT pre_turn_bearing_list[], - PostTurnBearingT post_turn_bearing_list[], - std::uint32_t number_of_edges) +// Needs to be called after readElementCount() to get the correct offset in the stream +inline void readEdges(boost::filesystem::ifstream &edges_input_stream, + GeometryID *geometry_list, + NameID *name_id_list, + extractor::guidance::TurnInstruction *turn_instruction_list, + LaneDataID *lane_data_id_list, + extractor::TravelMode *travel_mode_list, + EntryClassID *entry_class_id_list, + util::guidance::TurnBearing *pre_turn_bearing_list, + util::guidance::TurnBearing *post_turn_bearing_list, + const std::uint64_t number_of_edges) { BOOST_ASSERT(geometry_list); BOOST_ASSERT(name_id_list); @@ -130,9 +143,10 @@ void readEdgesData(boost::filesystem::ifstream &edges_input_stream, BOOST_ASSERT(travel_mode_list); BOOST_ASSERT(entry_class_id_list); extractor::OriginalEdgeData current_edge_data; - for (std::uint32_t i = 0; i < number_of_edges; ++i) + for (std::uint64_t i = 0; i < number_of_edges; ++i) { edges_input_stream.read((char *)&(current_edge_data), sizeof(extractor::OriginalEdgeData)); + geometry_list[i] = current_edge_data.via_geometry; name_id_list[i] = current_edge_data.name_id; turn_instruction_list[i] = current_edge_data.turn_instruction; @@ -144,96 +158,62 @@ void readEdgesData(boost::filesystem::ifstream &edges_input_stream, } } -// Returns the number of nodes in a .nodes file -inline std::uint32_t readNodesSize(boost::filesystem::ifstream &nodes_input_stream) -{ - std::uint32_t number_of_coordinates; - nodes_input_stream.read((char *)&number_of_coordinates, sizeof(std::uint32_t)); - return number_of_coordinates; -} - -// Reads coordinates and OSM node IDs from .nodes files -// Needs to be called after readNodesSize() to get the correct offset in the stream -template -void readNodesData(boost::filesystem::ifstream &nodes_input_stream, - CoordinateT coordinate_list[], - OSMNodeIDVectorT &osmnodeid_list, - std::uint32_t number_of_coordinates) +// Loads coordinates and OSM node IDs from .nodes files into memory +// Needs to be called after readElementCount() to get the correct offset in the stream +template +void readNodes(boost::filesystem::ifstream &nodes_input_stream, + util::Coordinate *coordinate_list, + OSMNodeIDVectorT &osmnodeid_list, + const std::uint64_t number_of_coordinates) { BOOST_ASSERT(coordinate_list); extractor::QueryNode current_node; - for (std::uint32_t i = 0; i < number_of_coordinates; ++i) + for (std::uint64_t i = 0; i < number_of_coordinates; ++i) { nodes_input_stream.read((char *)¤t_node, sizeof(extractor::QueryNode)); - coordinate_list[i] = CoordinateT(current_node.lon, current_node.lat); + coordinate_list[i] = util::Coordinate(current_node.lon, current_node.lat); osmnodeid_list.push_back(current_node.node_id); BOOST_ASSERT(coordinate_list[i].IsValid()); } } -// Returns the number of indexes in a .datasource_indexes file -inline std::uint64_t -readDatasourceIndexesSize(boost::filesystem::ifstream &datasource_indexes_input_stream) -{ - std::uint64_t number_of_datasource_indexes; - datasource_indexes_input_stream.read(reinterpret_cast(&number_of_datasource_indexes), - sizeof(std::uint64_t)); - return number_of_datasource_indexes; -} - -// Reads datasource_indexes -// Needs to be called after readDatasourceIndexesSize() to get the correct offset in the stream -inline void readDatasourceIndexes(boost::filesystem::ifstream &datasource_indexes_input_stream, - uint8_t datasource_buffer[], - std::uint32_t number_of_datasource_indexes) -{ - BOOST_ASSERT(datasource_buffer); - datasource_indexes_input_stream.read(reinterpret_cast(datasource_buffer), - number_of_datasource_indexes * sizeof(std::uint8_t)); -} // Reads datasource names out of .datasource_names files and metadata such as // the length and offset of each name struct DatasourceNamesData { std::vector names; - std::vector offsets; - std::vector lengths; + std::vector offsets; + std::vector lengths; }; -inline DatasourceNamesData -readDatasourceNamesData(boost::filesystem::ifstream &datasource_names_input_stream) +inline DatasourceNamesData readDatasourceNames(boost::filesystem::ifstream &datasource_names_input_stream) { DatasourceNamesData datasource_names_data; - if (datasource_names_input_stream) + std::string name; + while (std::getline(datasource_names_input_stream, name)) { - std::string name; - while (std::getline(datasource_names_input_stream, name)) - { - datasource_names_data.offsets.push_back(datasource_names_data.names.size()); - datasource_names_data.lengths.push_back(name.size()); - std::copy(name.c_str(), - name.c_str() + name.size(), - std::back_inserter(datasource_names_data.names)); - } + datasource_names_data.offsets.push_back(datasource_names_data.names.size()); + datasource_names_data.lengths.push_back(name.size()); + std::copy(name.c_str(), + name.c_str() + name.size(), + std::back_inserter(datasource_names_data.names)); } return datasource_names_data; } -// Returns the number of ram indexes -inline std::uint32_t readRamIndexSize(boost::filesystem::ifstream &ram_index_input_stream) -{ - std::uint32_t tree_size = 0; - ram_index_input_stream.read((char *)&tree_size, sizeof(std::uint32_t)); - return tree_size; -} - +// Loads ram indexes of R-Trees from `.ramIndex` files into memory +// Needs to be called after readElementCount() to get the correct offset in the stream +// template +// NB Cannot be written without templated type because of cyclic depencies between +// `static_rtree.hpp` and `io.hpp` template -void readRamIndexData(boost::filesystem::ifstream &ram_index_input_stream, - RTreeNodeT rtree_buffer[], - std::uint32_t tree_size) +void readRamIndex(boost::filesystem::ifstream &ram_index_input_stream, + RTreeNodeT *rtree_buffer, + const std::uint64_t tree_size) { BOOST_ASSERT(rtree_buffer); - ram_index_input_stream.read(reinterpret_cast(rtree_buffer), sizeof(RTreeNodeT) * tree_size); + ram_index_input_stream.read(reinterpret_cast(rtree_buffer), + sizeof(RTreeNodeT) * tree_size); } } diff --git a/include/util/static_rtree.hpp b/include/util/static_rtree.hpp index 7be332eb6..ba1fccae2 100644 --- a/include/util/static_rtree.hpp +++ b/include/util/static_rtree.hpp @@ -333,9 +333,9 @@ class StaticRTree // open tree file boost::filesystem::ofstream tree_node_file(tree_node_filename, std::ios::binary); - std::uint32_t size_of_tree = m_search_tree.size(); + std::uint64_t size_of_tree = m_search_tree.size(); BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty"); - tree_node_file.write((char *)&size_of_tree, sizeof(std::uint32_t)); + tree_node_file.write((char *)&size_of_tree, sizeof(size_of_tree)); tree_node_file.write((char *)&m_search_tree[0], sizeof(TreeNode) * size_of_tree); MapLeafNodesFile(leaf_node_filename); @@ -357,11 +357,10 @@ class StaticRTree } boost::filesystem::ifstream tree_node_file(node_file, std::ios::binary); - - std::uint32_t tree_size = storage::io::readRamIndexSize(tree_node_file); + const auto tree_size = storage::io::readElementCount(tree_node_file); m_search_tree.resize(tree_size); - storage::io::readRamIndexData(tree_node_file, &m_search_tree[0], tree_size); + storage::io::readRamIndex(tree_node_file, &m_search_tree[0], tree_size); MapLeafNodesFile(leaf_file); } diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp index c4ea3641a..58aa997a9 100644 --- a/src/contractor/contractor.cpp +++ b/src/contractor/contractor.cpp @@ -420,8 +420,8 @@ parse_turn_penalty_lookup_from_csv_files(const std::vector &turn_pe local.push_back(std::move(val)); } - util::SimpleLogger().Write() << "Loaded penalty file " << filename << " with " << local.size() - << " turn penalties"; + util::SimpleLogger().Write() << "Loaded penalty file " << filename << " with " + << local.size() << " turn penalties"; { Mutex::scoped_lock _{flatten_mutex}; @@ -437,8 +437,10 @@ parse_turn_penalty_lookup_from_csv_files(const std::vector &turn_pe // With flattened map-ish view of all the files, sort and unique them on from,to,source // The greater '>' is used here since we want to give files later on higher precedence const auto sort_by = [](const TurnPenaltySource &lhs, const TurnPenaltySource &rhs) { - return std::tie(lhs.segment.from, lhs.segment.via, lhs.segment.to, lhs.penalty_source.source) > - std::tie(rhs.segment.from, rhs.segment.via, rhs.segment.to, rhs.penalty_source.source); + return std::tie( + lhs.segment.from, lhs.segment.via, lhs.segment.to, lhs.penalty_source.source) > + std::tie( + rhs.segment.from, rhs.segment.via, rhs.segment.to, rhs.penalty_source.source); }; std::stable_sort(begin(map), end(map), sort_by); @@ -569,8 +571,8 @@ EdgeID Contractor::LoadEdgeExpandedGraph( throw util::exception("Failed to open " + nodes_filename); } - unsigned number_of_nodes = 0; - nodes_input_stream.read((char *)&number_of_nodes, sizeof(unsigned)); + std::uint64_t number_of_nodes = 0; + nodes_input_stream.read((char *)&number_of_nodes, sizeof(std::uint64_t)); internal_to_external_node_map.resize(number_of_nodes); // Load all the query nodes into a vector @@ -976,7 +978,7 @@ Contractor::WriteContractedGraph(unsigned max_node_id, { // Sorting contracted edges in a way that the static query graph can read some in in-place. tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end()); - const unsigned contracted_edge_count = contracted_edge_list.size(); + const std::uint64_t contracted_edge_count = contracted_edge_list.size(); util::SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count << " edges"; @@ -1033,13 +1035,13 @@ Contractor::WriteContractedGraph(unsigned max_node_id, const unsigned edges_crc32 = crc32_calculator(contracted_edge_list); util::SimpleLogger().Write() << "Writing CRC32: " << edges_crc32; - const unsigned node_array_size = node_array.size(); + const std::uint64_t node_array_size = node_array.size(); // serialize crc32, aka checksum hsgr_output_stream.write((char *)&edges_crc32, sizeof(unsigned)); // serialize number of nodes - hsgr_output_stream.write((char *)&node_array_size, sizeof(unsigned)); + hsgr_output_stream.write((char *)&node_array_size, sizeof(std::uint64_t)); // serialize number of edges - hsgr_output_stream.write((char *)&contracted_edge_count, sizeof(unsigned)); + hsgr_output_stream.write((char *)&contracted_edge_count, sizeof(std::uint64_t)); // serialize all nodes if (node_array_size > 0) { diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index 534bfa594..1995f1052 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -331,7 +331,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( } // Writes a dummy value at the front that is updated later with the total length - const unsigned length_prefix_empty_space{0}; + const std::uint64_t length_prefix_empty_space{0}; edge_data_file.write(reinterpret_cast(&length_prefix_empty_space), sizeof(length_prefix_empty_space)); @@ -598,7 +598,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( // Finally jump back to the empty space at the beginning and write length prefix edge_data_file.seekp(std::ios::beg); - const auto length_prefix = boost::numeric_cast(original_edges_counter); + const auto length_prefix = boost::numeric_cast(original_edges_counter); static_assert(sizeof(length_prefix_empty_space) == sizeof(length_prefix), "type mismatch"); edge_data_file.write(reinterpret_cast(&length_prefix), sizeof(length_prefix)); diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 7b8d99e69..db451576d 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -112,8 +112,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) TIMER_START(extracting); const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); - const auto number_of_threads = - std::min(recommended_num_threads, config.requested_num_threads); + const auto number_of_threads = std::min(recommended_num_threads, config.requested_num_threads); tbb::task_scheduler_init init(number_of_threads); { @@ -527,8 +526,8 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment, void Extractor::WriteNodeMapping(const std::vector &internal_to_external_node_map) { boost::filesystem::ofstream node_stream(config.node_output_path, std::ios::binary); - const unsigned size_of_mapping = internal_to_external_node_map.size(); - node_stream.write((char *)&size_of_mapping, sizeof(unsigned)); + const std::uint64_t size_of_mapping = internal_to_external_node_map.size(); + node_stream.write((char *)&size_of_mapping, sizeof(std::uint64_t)); if (size_of_mapping > 0) { node_stream.write((char *)internal_to_external_node_map.data(), diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 4bde35b3b..28881e1be 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -230,7 +230,7 @@ Storage::ReturnCode Storage::Run(int max_wait) throw util::exception("Could not open " + config.edges_data_path.string() + " for reading."); } - auto number_of_original_edges = io::readEdgesSize(edges_input_stream); + const auto number_of_original_edges = io::readElementCount(edges_input_stream); // note: settings this all to the same size is correct, we extract them from the same struct shared_layout_ptr->SetBlockSize(SharedDataLayout::VIA_NODE_LIST, @@ -256,7 +256,7 @@ Storage::ReturnCode Storage::Run(int max_wait) throw util::exception("Could not open " + config.hsgr_data_path.string() + " for reading."); } - auto hsgr_header = io::readHSGRHeader(hsgr_input_stream); + const auto hsgr_header = io::readHSGRHeader(hsgr_input_stream); shared_layout_ptr->SetBlockSize(SharedDataLayout::HSGR_CHECKSUM, 1); shared_layout_ptr->SetBlockSize(SharedDataLayout::GRAPH_NODE_LIST, hsgr_header.number_of_nodes); @@ -266,13 +266,13 @@ Storage::ReturnCode Storage::Run(int max_wait) // load rsearch tree size boost::filesystem::ifstream tree_node_file(config.ram_index_path, std::ios::binary); - std::uint32_t tree_size = io::readRamIndexSize(tree_node_file); + const auto tree_size = io::readElementCount(tree_node_file); shared_layout_ptr->SetBlockSize(SharedDataLayout::R_SEARCH_TREE, tree_size); // allocate space in shared memory for profile properties - std::size_t PropertiesSize = io::readPropertiesSize(); + const auto properties_size = io::readPropertiesCount(); shared_layout_ptr->SetBlockSize(SharedDataLayout::PROPERTIES, - PropertiesSize); + properties_size); // read timestampsize boost::filesystem::ifstream timestamp_stream(config.timestamp_path); @@ -280,7 +280,7 @@ Storage::ReturnCode Storage::Run(int max_wait) { throw util::exception("Could not open " + config.timestamp_path.string() + " for reading."); } - std::size_t timestamp_size = io::readTimestampSize(timestamp_stream); + const auto timestamp_size = io::readNumberOfBytes(timestamp_stream); shared_layout_ptr->SetBlockSize(SharedDataLayout::TIMESTAMP, timestamp_size); // load core marker size @@ -301,7 +301,7 @@ Storage::ReturnCode Storage::Run(int max_wait) { throw util::exception("Could not open " + config.core_data_path.string() + " for reading."); } - auto coordinate_list_size = io::readNodesSize(nodes_input_stream); + const auto coordinate_list_size = io::readElementCount(nodes_input_stream); shared_layout_ptr->SetBlockSize(SharedDataLayout::COORDINATE_LIST, coordinate_list_size); // we'll read a list of OSM node IDs from the same data, so set the block size for the same @@ -342,12 +342,8 @@ Storage::ReturnCode Storage::Run(int max_wait) throw util::exception("Could not open " + config.datasource_indexes_path.string() + " for reading."); } - std::uint64_t number_of_compressed_datasources = 0; - if (geometry_datasource_input_stream) - { - number_of_compressed_datasources = - io::readDatasourceIndexesSize(geometry_datasource_input_stream); - } + const auto number_of_compressed_datasources = + io::readElementCount(geometry_datasource_input_stream); shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCES_LIST, number_of_compressed_datasources); @@ -361,19 +357,15 @@ Storage::ReturnCode Storage::Run(int max_wait) throw util::exception("Could not open " + config.datasource_names_path.string() + " for reading."); } - io::DatasourceNamesData datasource_names_data; - if (datasource_names_input_stream) - { - datasource_names_data = io::readDatasourceNamesData(datasource_names_input_stream); - } + const io::DatasourceNamesData datasource_names_data = + io::readDatasourceNames(datasource_names_input_stream); shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCE_NAME_DATA, datasource_names_data.names.size()); - shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCE_NAME_OFFSETS, - datasource_names_data.offsets.size()); - shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCE_NAME_LENGTHS, - datasource_names_data.lengths.size()); - + shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCE_NAME_OFFSETS, + datasource_names_data.offsets.size()); + shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCE_NAME_LENGTHS, + datasource_names_data.lengths.size()); boost::filesystem::ifstream intersection_stream(config.intersection_class_path, std::ios::binary); @@ -564,16 +556,16 @@ Storage::ReturnCode Storage::Run(int max_wait) EntryClassID *entry_class_id_ptr = shared_layout_ptr->GetBlockPtr( shared_memory_ptr, SharedDataLayout::ENTRY_CLASSID); - io::readEdgesData(edges_input_stream, - via_geometry_ptr, - name_id_ptr, - turn_instructions_ptr, - lane_data_id_ptr, - travel_mode_ptr, - entry_class_id_ptr, - pre_turn_bearing_ptr, - post_turn_bearing_ptr, - number_of_original_edges); + io::readEdges(edges_input_stream, + via_geometry_ptr, + name_id_ptr, + turn_instructions_ptr, + lane_data_id_ptr, + travel_mode_ptr, + entry_class_id_ptr, + pre_turn_bearing_ptr, + post_turn_bearing_ptr, + number_of_original_edges); edges_input_stream.close(); // load compressed geometry @@ -676,7 +668,7 @@ Storage::ReturnCode Storage::Run(int max_wait) util::PackedVector osmnodeid_list; osmnodeid_list.reset(osmnodeid_ptr, shared_layout_ptr->num_entries[SharedDataLayout::OSM_NODE_ID_LIST]); - io::readNodesData(nodes_input_stream, coordinates_ptr, osmnodeid_list, coordinate_list_size); + io::readNodes(nodes_input_stream, coordinates_ptr, osmnodeid_list, coordinate_list_size); nodes_input_stream.close(); // store timestamp @@ -685,9 +677,9 @@ Storage::ReturnCode Storage::Run(int max_wait) io::readTimestamp(timestamp_stream, timestamp_ptr, timestamp_size); // store search tree portion of rtree - RTreeNode *rtree_ptrtest = shared_layout_ptr->GetBlockPtr(shared_memory_ptr, - SharedDataLayout::R_SEARCH_TREE); - io::readRamIndexData(tree_node_file, rtree_ptrtest, tree_size); + RTreeNode *rtree_ptrtest = shared_layout_ptr->GetBlockPtr( + shared_memory_ptr, SharedDataLayout::R_SEARCH_TREE); + io::readRamIndex(tree_node_file, rtree_ptrtest, tree_size); // load core markers std::vector unpacked_core_markers(number_of_core_markers); @@ -744,9 +736,8 @@ Storage::ReturnCode Storage::Run(int max_wait) { util::exception("Could not open " + config.properties_path.string() + " for reading!"); } - io::readProperties(profile_properties_stream, - profile_properties_ptr, - sizeof(extractor::ProfileProperties)); + io::readProperties( + profile_properties_stream, profile_properties_ptr, sizeof(extractor::ProfileProperties)); // load intersection classes if (!bearing_class_id_table.empty())