Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Make PackedVector generic

Browse Source
This commit is contained in:
Patrick Niklaus 2017-04-10 20:15:25 +00:00 committed by Patrick Niklaus
parent e85c4f87e9
commit a66918a303
12 changed files with 64 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View File

@ -16,6 +16,7 @@
#include "extractor/profile_properties.hpp"
#include "extractor/segment_data_container.hpp"
#include "extractor/turn_data_container.hpp"
#include "extractor/packed_osm_ids.hpp"
#include "contractor/query_graph.hpp"
@ -214,7 +215,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
unsigned m_check_sum;
util::vector_view<util::Coordinate> m_coordinate_list;
util::PackedVectorView<OSMNodeID> m_osmnodeid_list;
extractor::PackedOSMIDsView m_osmnodeid_list;
util::NameTable m_names_table;
util::vector_view<std::uint32_t> m_lane_description_offsets;
util::vector_view<extractor::guidance::TurnLaneType::Mask> m_lane_description_masks;

5
include/extractor/edge_based_graph_factory.hpp
View File

@ -15,6 +15,7 @@
#include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp"
#include "extractor/restriction_map.hpp"
#include "extractor/packed_osm_ids.hpp"
#include "util/deallocating_vector.hpp"
#include "util/guidance/bearing_class.hpp"
@ -74,7 +75,7 @@ class EdgeBasedGraphFactory
const std::unordered_set<NodeID> &traffic_lights,
std::shared_ptr<const RestrictionMap> restriction_map,
const std::vector<util::Coordinate> &coordinates,
const util::PackedVector<OSMNodeID> &osm_node_ids,
const extractor::PackedOSMIDs &osm_node_ids,
ProfileProperties profile_properties,
const util::NameTable &name_table,
std::vector<std::uint32_t> &turn_lane_offsets,
@ -131,7 +132,7 @@ class EdgeBasedGraphFactory
EdgeID m_max_edge_id;
const std::vector<util::Coordinate> &m_coordinates;
const util::PackedVector<OSMNodeID> &m_osm_node_ids;
const extractor::PackedOSMIDs &m_osm_node_ids;
std::shared_ptr<util::NodeBasedDynamicGraph> m_node_based_graph;
std::shared_ptr<RestrictionMap const> m_restriction_map;

4
include/extractor/extractor.hpp
View File

@ -59,7 +59,7 @@ class Extractor
std::pair<std::size_t, EdgeID>
BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
std::vector<util::Coordinate> &coordinates,
util::PackedVector<OSMNodeID> &osm_node_ids,
extractor::PackedOSMIDs &osm_node_ids,
std::vector<EdgeBasedNode> &node_based_edge_list,
std::vector<bool> &node_is_startpoint,
std::vector<EdgeWeight> &edge_based_node_weights,
@ -78,7 +78,7 @@ class Extractor
LoadNodeBasedGraph(std::unordered_set<NodeID> &barrier_nodes,
std::unordered_set<NodeID> &traffic_lights,
std::vector<util::Coordinate> &coordinates,
util::PackedVector<OSMNodeID> &osm_node_ids);
extractor::PackedOSMIDs &osm_node_ids);
void WriteEdgeBasedGraph(const std::string &output_file_filename,
const EdgeID max_edge_id,

3
include/util/graph_loader.hpp
View File

@ -5,6 +5,7 @@
#include "extractor/node_based_edge.hpp"
#include "extractor/query_node.hpp"
#include "extractor/restriction.hpp"
#include "extractor/packed_osm_ids.hpp"
#include "storage/io.hpp"
#include "util/exception.hpp"
#include "util/fingerprint.hpp"
@ -59,7 +60,7 @@ NodeID loadNodesFromFile(storage::io::FileReader &file_reader,
BarrierOutIter barriers,
TrafficSignalsOutIter traffic_signals,
std::vector<util::Coordinate> &coordinates,
util::PackedVector<OSMNodeID> &osm_node_ids)
extractor::PackedOSMIDs &osm_node_ids)
{
auto number_of_nodes = file_reader.ReadElementCount64();
Log() << "Importing number_of_nodes new = " << number_of_nodes << " nodes ";

83
include/util/packed_vector.hpp
View File

@ -16,33 +16,28 @@ namespace util
{
namespace detail
{
template <typename T, storage::Ownership Ownership> class PackedVector;
template <typename T, std::size_t Bits, storage::Ownership Ownership> class PackedVector;
}
namespace serialization
{
template <typename T, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Ownership> &vec);
template <typename T, std::size_t Bits, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Bits, Ownership> &vec);
template <typename T, storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, const detail::PackedVector<T, Ownership> &vec);
template <typename T, std::size_t Bits, storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::PackedVector<T, Bits, Ownership> &vec);
}
namespace detail
{
/**
* Since OSM node IDs are (at the time of writing) not quite yet overflowing 32 bits, and
* will predictably be containable within 33 bits for a long time, the following packs
* 64-bit OSM IDs as 33-bit numbers within a 64-bit vector.
*
* NOTE: this type is templated for future use, but will require a slight refactor to
* configure BITSIZE and ELEMSIZE
*/
template <typename T, storage::Ownership Ownership> class PackedVector
template <typename T, std::size_t Bits, storage::Ownership Ownership> class PackedVector
{
static const constexpr std::size_t BITSIZE = 33;
static const constexpr std::size_t ELEMSIZE = 64;
static const constexpr std::size_t PACKSIZE = BITSIZE * ELEMSIZE;
static_assert(sizeof(T) <= sizeof(std::uint64_t), "Maximum size of type T is 8 bytes");
static_assert(Bits <= sizeof(std::uint64_t)*CHAR_BIT, "Maximum number of bits it 64.");
static const constexpr std::size_t ELEMSIZE = sizeof(std::uint64_t)*CHAR_BIT;
static const constexpr std::size_t PACKSIZE = Bits * ELEMSIZE;
public:
using value_type = T;
@ -54,31 +49,31 @@ template <typename T, storage::Ownership Ownership> class PackedVector
*/
inline static std::size_t elements_to_blocks(std::size_t elements)
{
return std::ceil(static_cast<double>(elements) * BITSIZE / ELEMSIZE);
return std::ceil(static_cast<double>(elements) * Bits / ELEMSIZE);
}
void push_back(T incoming_node_id)
void push_back(T data)
{
std::uint64_t node_id = static_cast<std::uint64_t>(incoming_node_id);
std::uint64_t node_id = static_cast<std::uint64_t>(data);
// mask incoming values, just in case they are > bitsize
const std::uint64_t incoming_mask = static_cast<std::uint64_t>(pow(2, BITSIZE)) - 1;
const std::uint64_t incoming_mask = static_cast<std::uint64_t>(pow(2, Bits)) - 1;
node_id = node_id & incoming_mask;
const std::size_t available = (PACKSIZE - BITSIZE * num_elements) % ELEMSIZE;
const std::size_t available = (PACKSIZE - Bits * num_elements) % ELEMSIZE;
if (available == 0)
{
// insert ID at the left side of this element
std::uint64_t at_left = node_id << (ELEMSIZE - BITSIZE);
std::uint64_t at_left = node_id << (ELEMSIZE - Bits);
add_last_elem(at_left);
}
else if (available >= BITSIZE)
else if (available >= Bits)
{
// insert ID somewhere in the middle of this element; ID can be contained
// entirely within one element
const std::uint64_t shifted = node_id << (available - BITSIZE);
const std::uint64_t shifted = node_id << (available - Bits);
replace_last_elem(vec_back() | shifted);
}
@ -86,9 +81,9 @@ template <typename T, storage::Ownership Ownership> class PackedVector
{
// ID will be split between the end of this element and the beginning
// of the next element
const std::uint64_t left = node_id >> (BITSIZE - available);
const std::uint64_t left = node_id >> (Bits - available);
std::uint64_t right = node_id << (ELEMSIZE - (BITSIZE - available));
std::uint64_t right = node_id << (ELEMSIZE - (Bits - available));
replace_last_elem(vec_back() | left);
add_last_elem(right);
@ -105,11 +100,11 @@ template <typename T, storage::Ownership Ownership> class PackedVector
const std::size_t pack_group = trunc(a_index / ELEMSIZE);
const std::size_t pack_index = (a_index + ELEMSIZE) % ELEMSIZE;
const std::size_t left_index = (PACKSIZE - BITSIZE * pack_index) % ELEMSIZE;
const std::size_t left_index = (PACKSIZE - Bits * pack_index) % ELEMSIZE;
const bool back_half = pack_index >= BITSIZE;
const std::size_t index = pack_group * BITSIZE + trunc(pack_index / BITSIZE) +
trunc((pack_index - back_half) / 2);
const bool back_half = pack_index >= Bits;
const std::size_t index =
pack_group * Bits + trunc(pack_index / Bits) + trunc((pack_index - back_half) / 2);
BOOST_ASSERT(index < vec.size());
const std::uint64_t elem = static_cast<std::uint64_t>(vec.at(index));
@ -117,25 +112,25 @@ template <typename T, storage::Ownership Ownership> class PackedVector
if (left_index == 0)
{
// ID is at the far left side of this element
return T{elem >> (ELEMSIZE - BITSIZE)};
return T{elem >> (ELEMSIZE - Bits)};
}
else if (left_index >= BITSIZE)
else if (left_index >= Bits)
{
// ID is entirely contained within this element
const std::uint64_t at_right = elem >> (left_index - BITSIZE);
const std::uint64_t left_mask = static_cast<std::uint64_t>(pow(2, BITSIZE)) - 1;
const std::uint64_t at_right = elem >> (left_index - Bits);
const std::uint64_t left_mask = static_cast<std::uint64_t>(pow(2, Bits)) - 1;
return T{at_right & left_mask};
}
else
{
// ID is split between this and the next element
const std::uint64_t left_mask = static_cast<std::uint64_t>(pow(2, left_index)) - 1;
const std::uint64_t left_side = (elem & left_mask) << (BITSIZE - left_index);
const std::uint64_t left_side = (elem & left_mask) << (Bits - left_index);
BOOST_ASSERT(index < vec.size() - 1);
const std::uint64_t next_elem = static_cast<std::uint64_t>(vec.at(index + 1));
const std::uint64_t right_side = next_elem >> (ELEMSIZE - (BITSIZE - left_index));
const std::uint64_t right_side = next_elem >> (ELEMSIZE - (Bits - left_index));
return T{left_side | right_side};
}
}
@ -163,14 +158,14 @@ template <typename T, storage::Ownership Ownership> class PackedVector
std::size_t capacity() const
{
return std::floor(static_cast<double>(vec.capacity()) * ELEMSIZE / BITSIZE);
return std::floor(static_cast<double>(vec.capacity()) * ELEMSIZE / Bits);
}
friend void serialization::read<T, Ownership>(storage::io::FileReader &reader,
detail::PackedVector<T, Ownership> &vec);
friend void serialization::read<T, Bits, Ownership>(storage::io::FileReader &reader,
PackedVector &vec);
friend void serialization::write<T, Ownership>(storage::io::FileWriter &writer,
const detail::PackedVector<T, Ownership> &vec);
friend void serialization::write<T, Bits, Ownership>(storage::io::FileWriter &writer,
const PackedVector &vec);
private:
util::ViewOrVector<std::uint64_t, Ownership> vec;
@ -218,8 +213,8 @@ template <typename T, storage::Ownership Ownership> class PackedVector
};
}
template <typename T> using PackedVector = detail::PackedVector<T, storage::Ownership::Container>;
template <typename T> using PackedVectorView = detail::PackedVector<T, storage::Ownership::View>;
template <typename T, std::size_t Bits> using PackedVector = detail::PackedVector<T, Bits, storage::Ownership::Container>;
template <typename T, std::size_t Bits> using PackedVectorView = detail::PackedVector<T, Bits, storage::Ownership::View>;
}
}

8
include/util/serialization.hpp
View File

@ -14,15 +14,15 @@ namespace util
{
namespace serialization
{
template <typename T, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Ownership> &vec)
template <typename T, std::size_t Bits, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Bits, Ownership> &vec)
{
vec.num_elements = reader.ReadOne<std::uint64_t>();
storage::serialization::read(reader, vec.vec);
}
template <typename T, storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, const detail::PackedVector<T, Ownership> &vec)
template <typename T, std::size_t Bits, storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, const detail::PackedVector<T, Bits, Ownership> &vec)
{
writer.WriteOne(vec.num_elements);
storage::serialization::write(writer, vec.vec);

2
src/extractor/edge_based_graph_factory.cpp
View File

@ -42,7 +42,7 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory(
const std::unordered_set<NodeID> &traffic_lights,
std::shared_ptr<const RestrictionMap> restriction_map,
const std::vector<util::Coordinate> &coordinates,
const util::PackedVector<OSMNodeID> &osm_node_ids,
const extractor::PackedOSMIDs &osm_node_ids,
ProfileProperties profile_properties,
const util::NameTable &name_table,
std::vector<std::uint32_t> &turn_lane_offsets,

6
src/extractor/extractor.cpp
View File

@ -263,7 +263,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
std::vector<bool> node_is_startpoint;
std::vector<EdgeWeight> edge_based_node_weights;
std::vector<util::Coordinate> coordinates;
util::PackedVector<OSMNodeID> osm_node_ids;
extractor::PackedOSMIDs osm_node_ids;
auto graph_size = BuildEdgeExpandedGraph(scripting_environment,
coordinates,
@ -407,7 +407,7 @@ std::shared_ptr<util::NodeBasedDynamicGraph>
Extractor::LoadNodeBasedGraph(std::unordered_set<NodeID> &barriers,
std::unordered_set<NodeID> &traffic_signals,
std::vector<util::Coordinate> &coordiantes,
util::PackedVector<OSMNodeID> &osm_node_ids)
extractor::PackedOSMIDs &osm_node_ids)
{
storage::io::FileReader file_reader(config.output_file_name,
storage::io::FileReader::VerifyFingerprint);
@ -439,7 +439,7 @@ Extractor::LoadNodeBasedGraph(std::unordered_set<NodeID> &barriers,
std::pair<std::size_t, EdgeID>
Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
std::vector<util::Coordinate> &coordinates,
util::PackedVector<OSMNodeID> &osm_node_ids,
extractor::PackedOSMIDs &osm_node_ids,
std::vector<EdgeBasedNode> &node_based_edge_list,
std::vector<bool> &node_is_startpoint,
std::vector<EdgeWeight> &edge_based_node_weights,

5
src/storage/storage.cpp
View File

@ -19,6 +19,7 @@
#include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp"
#include "extractor/travel_mode.hpp"
#include "extractor/packed_osm_ids.hpp"
#include "partition/cell_storage.hpp"
#include "partition/edge_based_graph_reader.hpp"
@ -326,7 +327,7 @@ void Storage::PopulateLayout(DataLayout &layout)
// number of items:
layout.SetBlockSize<std::uint64_t>(
DataLayout::OSM_NODE_ID_LIST,
util::PackedVectorView<OSMNodeID>::elements_to_blocks(coordinate_list_size));
extractor::PackedOSMIDsView::elements_to_blocks(coordinate_list_size));
}
// load geometries sizes
@ -694,7 +695,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
layout.GetBlockPtr<std::uint64_t, true>(memory_ptr, DataLayout::OSM_NODE_ID_LIST);
util::vector_view<util::Coordinate> coordinates(
coordinates_ptr, layout.num_entries[DataLayout::COORDINATE_LIST]);
util::PackedVectorView<OSMNodeID> osm_node_ids;
extractor::PackedOSMIDsView osm_node_ids;
osm_node_ids.reset(osmnodeid_ptr, layout.num_entries[DataLayout::OSM_NODE_ID_LIST]);
extractor::files::readNodes(config.nodes_data_path, coordinates, osm_node_ids);

4
src/tools/components.cpp
View File

@ -33,7 +33,7 @@ using TarjanEdge = util::static_graph_details::SortableEdgeWithData<void>;
std::size_t loadGraph(const std::string &path,
std::vector<util::Coordinate> &coordinate_list,
util::PackedVector<OSMNodeID> &osm_node_ids,
extractor::PackedOSMIDs &osm_node_ids,
std::vector<TarjanEdge> &graph_edge_list)
{
storage::io::FileReader file_reader(path, storage::io::FileReader::VerifyFingerprint);
@ -142,7 +142,7 @@ int main(int argc, char *argv[])
std::vector<tools::TarjanEdge> graph_edge_list;
std::vector<util::Coordinate> coordinate_list;
util::PackedVector<OSMNodeID> osm_node_ids;
extractor::PackedOSMIDs osm_node_ids;
auto number_of_nodes = tools::loadGraph(inpath, coordinate_list, osm_node_ids, graph_edge_list);
tbb::parallel_sort(graph_edge_list.begin(), graph_edge_list.end());

2
src/updater/updater.cpp
View File

@ -134,7 +134,7 @@ updateSegmentData(const UpdaterConfig &config,
extractor::SegmentDataContainer &segment_data)
{
std::vector<util::Coordinate> coordinates;
util::PackedVector<OSMNodeID> osm_node_ids;
extractor::PackedOSMIDs osm_node_ids;
extractor::files::readNodes(config.node_based_graph_path, coordinates, osm_node_ids);
// vector to count used speeds for logging

4
unit_tests/util/packed_vector.cpp
View File

@ -12,7 +12,7 @@ using namespace osrm::util;
// Verify that the packed vector behaves as expected
BOOST_AUTO_TEST_CASE(insert_and_retrieve_packed_test)
{
PackedVector<OSMNodeID> packed_ids;
PackedVector<OSMNodeID, 33> packed_ids;
std::vector<OSMNodeID> original_ids;
const constexpr std::size_t num_test_cases = 399;
@ -33,7 +33,7 @@ BOOST_AUTO_TEST_CASE(insert_and_retrieve_packed_test)
BOOST_AUTO_TEST_CASE(packed_vector_capacity_test)
{
PackedVector<OSMNodeID> packed_vec;
PackedVector<OSMNodeID, 33> packed_vec;
const std::size_t original_size = packed_vec.capacity();
std::vector<OSMNodeID> dummy_vec;