From 44fdf8670295c3d5a0c6ae0d897b18c88bfea161 Mon Sep 17 00:00:00 2001 From: Lauren Budorick Date: Mon, 23 May 2016 20:13:32 -0400 Subject: [PATCH] IT WORKS --- .../engine/datafacade/internal_datafacade.hpp | 9 +- .../engine/datafacade/shared_datafacade.hpp | 5 +- include/util/packed_vector.hpp | 89 ++++++++++++++++--- include/util/shared_memory_vector_wrapper.hpp | 2 + src/storage/storage.cpp | 9 +- 5 files changed, 94 insertions(+), 20 deletions(-) diff --git a/include/engine/datafacade/internal_datafacade.hpp b/include/engine/datafacade/internal_datafacade.hpp index 00d7dbc42..38af1d607 100644 --- a/include/engine/datafacade/internal_datafacade.hpp +++ b/include/engine/datafacade/internal_datafacade.hpp @@ -24,6 +24,7 @@ #include "util/static_graph.hpp" #include "util/static_rtree.hpp" #include "util/typedefs.hpp" +#include "util/packed_vector.hpp" #include "osrm/coordinate.hpp" @@ -73,7 +74,7 @@ class InternalDataFacade final : public BaseDataFacade std::string m_timestamp; util::ShM::vector m_coordinate_list; - util::ShM::vector m_osmnodeid_list; + util::PackedVector m_osmnodeid_list; util::ShM::vector m_via_node_list; util::ShM::vector m_name_ID_list; util::ShM::vector m_turn_instruction_list; @@ -157,12 +158,12 @@ class InternalDataFacade final : public BaseDataFacade unsigned number_of_coordinates = 0; nodes_input_stream.read((char *)&number_of_coordinates, sizeof(unsigned)); m_coordinate_list.resize(number_of_coordinates); - m_osmnodeid_list.resize(number_of_coordinates); + m_osmnodeid_list.reserve(number_of_coordinates); for (unsigned i = 0; i < number_of_coordinates; ++i) { nodes_input_stream.read((char *)¤t_node, sizeof(extractor::QueryNode)); m_coordinate_list[i] = util::Coordinate(current_node.lon, current_node.lat); - m_osmnodeid_list[i] = current_node.node_id; + m_osmnodeid_list.push_back(current_node.node_id); BOOST_ASSERT(m_coordinate_list[i].IsValid()); } @@ -443,7 +444,7 @@ class InternalDataFacade final : public BaseDataFacade OSMNodeID GetOSMNodeIDOfNode(const unsigned id) const override final { - return m_osmnodeid_list[id]; + return m_osmnodeid_list.at(id); } extractor::guidance::TurnInstruction diff --git a/include/engine/datafacade/shared_datafacade.hpp b/include/engine/datafacade/shared_datafacade.hpp index ae5ac425d..f700cecfa 100644 --- a/include/engine/datafacade/shared_datafacade.hpp +++ b/include/engine/datafacade/shared_datafacade.hpp @@ -76,7 +76,7 @@ class SharedDataFacade final : public BaseDataFacade extractor::ProfileProperties *m_profile_properties; util::ShM::vector m_coordinate_list; - util::ShM::vector m_osmnodeid_list; + util::PackedVector m_osmnodeid_list; util::ShM::vector m_via_node_list; util::ShM::vector m_name_ID_list; util::ShM::vector m_turn_instruction_list; @@ -175,6 +175,7 @@ class SharedDataFacade final : public BaseDataFacade shared_memory, storage::SharedDataLayout::OSM_NODE_ID_LIST); m_osmnodeid_list.reset(osmnodeid_list_ptr, data_layout->num_entries[storage::SharedDataLayout::OSM_NODE_ID_LIST]); + m_osmnodeid_list.set_number_of_entries(util::PackedVectorCapacity(data_layout->num_entries[storage::SharedDataLayout::OSM_NODE_ID_LIST])); auto travel_mode_list_ptr = data_layout->GetBlockPtr( shared_memory, storage::SharedDataLayout::TRAVEL_MODE); @@ -479,7 +480,7 @@ class SharedDataFacade final : public BaseDataFacade OSMNodeID GetOSMNodeIDOfNode(const unsigned id) const override final { - return m_osmnodeid_list[id]; + return m_osmnodeid_list.at(id); } virtual void GetUncompressedGeometry(const EdgeID id, diff --git a/include/util/packed_vector.hpp b/include/util/packed_vector.hpp index d6c28e68d..954a70e30 100644 --- a/include/util/packed_vector.hpp +++ b/include/util/packed_vector.hpp @@ -15,12 +15,22 @@ const constexpr std::size_t BITSIZE = 33; const constexpr std::size_t ELEMSIZE = 64; const constexpr std::size_t PACKSIZE = BITSIZE * ELEMSIZE; +std::size_t PackedVectorSize(std::size_t elements) +{ + return ceil(float(elements) / ELEMSIZE) * BITSIZE; +}; + +std::size_t PackedVectorCapacity(std::size_t vec_size) +{ + return floor(float(vec_size) / BITSIZE) * ELEMSIZE; +} + /** * Since OSM node IDs are (at the time of writing) not quite yet overflowing 32 bits, and * will predictably be containable within 33 bits for a long time, the following packs * 64-bit OSM IDs as 33-bit numbers within a 64-bit vector. */ -class PackedVector +template class PackedVector { public: PackedVector() = default; @@ -38,24 +48,27 @@ class PackedVector { // insert ID at the left side of this element std::uint64_t at_left = node_id << (ELEMSIZE - BITSIZE); - vec.push_back(at_left); + + add_last_elem(at_left); } else if (available >= BITSIZE) { // insert ID somewhere in the middle of this element; ID can be contained // entirely within one element const std::uint64_t shifted = node_id << (available - BITSIZE); - vec.back() = vec.back() | shifted; + + replace_last_elem(vec_back() | shifted); } else { // ID will be split between the end of this element and the beginning // of the next element const std::uint64_t left = node_id >> (BITSIZE - available); - vec.back() = vec.back() | left; std::uint64_t right = node_id << (ELEMSIZE - (BITSIZE - available)); - vec.push_back(right); + + replace_last_elem(vec_back() | left); + add_last_elem(right); } num_elements++; @@ -74,7 +87,7 @@ class PackedVector trunc((pack_index - back_half) / 2); BOOST_ASSERT(index < vec.size()); - const std::uint64_t elem = vec.at(index); + const std::uint64_t elem = static_cast(vec.at(index)); if (left_index == 0) { @@ -95,7 +108,7 @@ class PackedVector const std::uint64_t left_side = (elem & left_mask) << (BITSIZE - left_index); BOOST_ASSERT(index < vec.size() - 1); - const std::uint64_t next_elem = vec.at(index + 1); + const std::uint64_t next_elem = static_cast(vec.at(index + 1)); const std::uint64_t right_side = next_elem >> (ELEMSIZE - (BITSIZE - left_index)); return static_cast(left_side | right_side); @@ -107,19 +120,73 @@ class PackedVector return num_elements; } - void reserve(std::size_t capacity) + template + void reserve(typename std::enable_if::type capacity) { - vec.reserve(ceil(float(capacity) / ELEMSIZE) * BITSIZE); + vec.reserve(PackedVectorSize(capacity)); + } + + template + void reset(typename std::enable_if::type *ptr, typename std::enable_if::type size) + { + vec.reset(reinterpret_cast(ptr), size); + } + + template + void set_number_of_entries(typename std::enable_if::type count) + { + num_elements = count; } std::size_t capacity() const { - return floor(float(vec.capacity()) / BITSIZE) * ELEMSIZE; + return PackedVectorCapacity(vec.capacity()); } private: - std::vector vec; + + typename util::ShM::vector vec; + std::size_t num_elements = 0; + + signed cursor = -1; + + template + void replace_last_elem(typename std::enable_if::type last_elem) + { + vec[cursor] = last_elem; + } + + template + void replace_last_elem(typename std::enable_if::type last_elem) + { + vec.back() = last_elem; + } + + template + void add_last_elem(typename std::enable_if::type last_elem) + { + vec[cursor + 1] = last_elem; + cursor++; + } + + template + void add_last_elem(typename std::enable_if::type last_elem) + { + vec.push_back(last_elem); + } + + template + std::uint64_t vec_back(typename std::enable_if::type* = nullptr) + { + return vec[cursor]; + } + + template + std::uint64_t vec_back(typename std::enable_if::type* = nullptr) + { + return vec.back(); + } }; } } diff --git a/include/util/shared_memory_vector_wrapper.hpp b/include/util/shared_memory_vector_wrapper.hpp index c3127e3ef..2fec536be 100644 --- a/include/util/shared_memory_vector_wrapper.hpp +++ b/include/util/shared_memory_vector_wrapper.hpp @@ -3,6 +3,8 @@ #include +#include "util/simple_logger.hpp" + #include #include diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index c655a629f..389957456 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -20,6 +20,7 @@ #include "util/static_graph.hpp" #include "util/static_rtree.hpp" #include "util/typedefs.hpp" +#include "util/packed_vector.hpp" #ifdef __linux__ #include @@ -244,9 +245,9 @@ int Storage::Run() nodes_input_stream.read((char *)&coordinate_list_size, sizeof(unsigned)); shared_layout_ptr->SetBlockSize(SharedDataLayout::COORDINATE_LIST, coordinate_list_size); - // we'll read a list of OSM node IDs from the same data, so set the same block size: + // we'll read a list of OSM node IDs from the same data, so set the block size for the same number of items: shared_layout_ptr->SetBlockSize(SharedDataLayout::OSM_NODE_ID_LIST, - coordinate_list_size); + util::PackedVectorSize(coordinate_list_size)); // load geometries sizes boost::filesystem::ifstream geometry_input_stream(config.geometries_path, std::ios::binary); @@ -540,13 +541,15 @@ int Storage::Run() shared_memory_ptr, SharedDataLayout::COORDINATE_LIST); OSMNodeID *osmnodeid_ptr = shared_layout_ptr->GetBlockPtr( shared_memory_ptr, SharedDataLayout::OSM_NODE_ID_LIST); + util::PackedVector osmnodeid_list; + osmnodeid_list.reset(osmnodeid_ptr, shared_layout_ptr->num_entries[storage::SharedDataLayout::OSM_NODE_ID_LIST]); extractor::QueryNode current_node; for (unsigned i = 0; i < coordinate_list_size; ++i) { nodes_input_stream.read((char *)¤t_node, sizeof(extractor::QueryNode)); coordinates_ptr[i] = util::Coordinate(current_node.lon, current_node.lat); - osmnodeid_ptr[i] = OSMNodeID(current_node.node_id); + osmnodeid_list.push_back(OSMNodeID(current_node.node_id)); } nodes_input_stream.close();