From 3940cc164197d5ec1205b201f4175ee01de69a56 Mon Sep 17 00:00:00 2001 From: Michael Krasnyk Date: Fri, 7 Jul 2017 12:09:52 +0200 Subject: [PATCH] Switch from stxxl::vector to std::vector in extractor --- include/extractor/extraction_containers.hpp | 38 +++++----- .../first_and_last_segment_of_way.hpp | 2 +- include/storage/serialization.hpp | 7 ++ include/util/meminfo.hpp | 7 +- include/util/vector_view.hpp | 14 +++- src/extractor/extraction_containers.cpp | 70 ++++++++++++------- unit_tests/util/range_table.cpp | 16 ++--- 7 files changed, 96 insertions(+), 58 deletions(-) diff --git a/include/extractor/extraction_containers.hpp b/include/extractor/extraction_containers.hpp index c5568af19..c45798ff0 100644 --- a/include/extractor/extraction_containers.hpp +++ b/include/extractor/extraction_containers.hpp @@ -11,9 +11,12 @@ #include "storage/io.hpp" #include -#include #include +#if USE_STXXL_LIBRARY +#include +#endif + namespace osrm { namespace extractor @@ -27,13 +30,12 @@ namespace extractor */ class ExtractionContainers { -#ifndef _MSC_VER - constexpr static unsigned stxxl_memory = - ((sizeof(std::size_t) == 4) ? std::numeric_limits::max() - : std::numeric_limits::max()); +#if USE_STXXL_LIBRARY + template using ExternalVector = stxxl::vector; #else - const static unsigned stxxl_memory = ((sizeof(std::size_t) == 4) ? INT_MAX : UINT_MAX); + template using ExternalVector = std::vector; #endif + void FlushVectors(); void PrepareNodes(); void PrepareRestrictions(); @@ -45,24 +47,24 @@ class ExtractionContainers void WriteCharData(const std::string &file_name); public: - using STXXLNodeIDVector = stxxl::vector; - using STXXLNodeVector = stxxl::vector; - using STXXLEdgeVector = stxxl::vector; + using NodeIDVector = ExternalVector; + using NodeVector = ExternalVector; + using EdgeVector = ExternalVector; using RestrictionsVector = std::vector; - using STXXLWayIDStartEndVector = stxxl::vector; - using STXXLNameCharData = stxxl::vector; - using STXXLNameOffsets = stxxl::vector; + using WayIDStartEndVector = ExternalVector; + using NameCharData = ExternalVector; + using NameOffsets = ExternalVector; std::vector barrier_nodes; std::vector traffic_lights; - STXXLNodeIDVector used_node_id_list; - STXXLNodeVector all_nodes_list; - STXXLEdgeVector all_edges_list; - STXXLNameCharData name_char_data; - STXXLNameOffsets name_offsets; + NodeIDVector used_node_id_list; + NodeVector all_nodes_list; + EdgeVector all_edges_list; + NameCharData name_char_data; + NameOffsets name_offsets; // an adjacency array containing all turn lane masks RestrictionsVector restrictions_list; - STXXLWayIDStartEndVector way_start_end_id_list; + WayIDStartEndVector way_start_end_id_list; unsigned max_internal_node_id; std::vector unconditional_turn_restrictions; diff --git a/include/extractor/first_and_last_segment_of_way.hpp b/include/extractor/first_and_last_segment_of_way.hpp index fd20dbacb..5759e97ad 100644 --- a/include/extractor/first_and_last_segment_of_way.hpp +++ b/include/extractor/first_and_last_segment_of_way.hpp @@ -43,7 +43,7 @@ struct FirstAndLastSegmentOfWay } }; -struct FirstAndLastSegmentOfWayStxxlCompare +struct FirstAndLastSegmentOfWayCompare { using value_type = FirstAndLastSegmentOfWay; bool operator()(const FirstAndLastSegmentOfWay &a, const FirstAndLastSegmentOfWay &b) const diff --git a/include/storage/serialization.hpp b/include/storage/serialization.hpp index c75757319..021bb8550 100644 --- a/include/storage/serialization.hpp +++ b/include/storage/serialization.hpp @@ -7,8 +7,13 @@ #include "storage/io.hpp" +#include #include +#if USE_STXXL_LIBRARY +#include +#endif + namespace osrm { namespace storage @@ -58,6 +63,7 @@ inline void write(storage::io::FileWriter &writer, const util::DeallocatingVecto writer.WriteFrom(vec.bucket_list.back(), last_block_size); } +#if USE_STXXL_LIBRARY template inline void read(storage::io::FileReader &reader, stxxl::vector &vec) { auto size = reader.ReadOne(); @@ -78,6 +84,7 @@ inline void write(storage::io::FileWriter &writer, const stxxl::vector &vec) writer.WriteOne(vec[idx]); } } +#endif template void read(io::FileReader &reader, std::vector &data) { diff --git a/include/util/meminfo.hpp b/include/util/meminfo.hpp index 435b5c32f..40abce33e 100644 --- a/include/util/meminfo.hpp +++ b/include/util/meminfo.hpp @@ -3,11 +3,14 @@ #include "util/log.hpp" -#include #ifndef _WIN32 #include #endif +#if USE_STXXL_LIBRARY +#include +#endif + namespace osrm { namespace util @@ -15,6 +18,7 @@ namespace util inline void DumpSTXXLStats() { +#if USE_STXXL_LIBRARY #if STXXL_VERSION_MAJOR > 1 || (STXXL_VERSION_MAJOR == 1 && STXXL_VERSION_MINOR >= 4) auto manager = stxxl::block_manager::get_instance(); util::Log() << "STXXL: peak bytes used: " << manager->get_maximum_allocation(); @@ -23,6 +27,7 @@ inline void DumpSTXXLStats() #warning STXXL 1.4+ recommended - STXXL memory summary will not be available util::Log() << "STXXL: memory summary not available, needs STXXL 1.4 or higher"; #endif +#endif } inline void DumpMemoryStats() diff --git a/include/util/vector_view.hpp b/include/util/vector_view.hpp index 23dd4734f..ebf99872b 100644 --- a/include/util/vector_view.hpp +++ b/include/util/vector_view.hpp @@ -6,8 +6,6 @@ #include "storage/shared_memory_ownership.hpp" -#include - #include #include #include @@ -21,6 +19,10 @@ #include #include +#if USE_STXXL_LIBRARY +#include +#endif + namespace osrm { namespace util @@ -209,10 +211,16 @@ template void swap(vector_view &lhs, vector_view std::swap(lhs.m_size, rhs.m_size); } +#if USE_STXXL_LIBRARY +template using ExternalVector = stxxl::vector; +#else +template using ExternalVector = std::vector; +#endif + template using InternalOrExternalVector = typename std::conditional, + ExternalVector, std::vector>::type; template diff --git a/src/extractor/extraction_containers.cpp b/src/extractor/extraction_containers.cpp index 13059d821..34c30a097 100644 --- a/src/extractor/extraction_containers.cpp +++ b/src/extractor/extraction_containers.cpp @@ -21,25 +21,30 @@ #include #include -#include +#include #include #include #include #include +#if USE_STXXL_LIBRARY +#include +#endif + namespace { namespace oe = osrm::extractor; // Needed for STXXL comparison - STXXL requires max_value(), min_value(), so we can not use // std::less{}. Anonymous namespace to keep translation unit local. -struct OSMNodeIDSTXXLLess +struct OSMNodeIDLess { - OSMNodeIDSTXXLLess() {} + OSMNodeIDLess() {} using value_type = OSMNodeID; bool operator()(const value_type left, const value_type right) const { return left < right; } + value_type max_value() { return MAX_OSM_NODEID; } value_type min_value() { return MIN_OSM_NODEID; } }; @@ -96,7 +101,7 @@ struct CmpEdgeByInternalSourceTargetAndName std::lock_guard lock(mutex); BOOST_ASSERT(!name_offsets.empty() && name_offsets.back() == name_data.size()); - const oe::ExtractionContainers::STXXLNameCharData::const_iterator data = name_data.begin(); + const oe::ExtractionContainers::NameCharData::const_iterator data = name_data.begin(); return std::lexicographical_compare(data + name_offsets[lhs.result.name_id], data + name_offsets[lhs.result.name_id + 1], data + name_offsets[rhs.result.name_id], @@ -107,18 +112,34 @@ struct CmpEdgeByInternalSourceTargetAndName value_type min_value() { return value_type::min_internal_value(); } std::mutex &mutex; - const oe::ExtractionContainers::STXXLNameCharData &name_data; - const oe::ExtractionContainers::STXXLNameOffsets &name_offsets; + const oe::ExtractionContainers::NameCharData &name_data; + const oe::ExtractionContainers::NameOffsets &name_offsets; }; template inline NodeID mapExternalToInternalNodeID(Iter first, Iter last, const OSMNodeID value) { - const OSMNodeIDSTXXLLess compare; + const OSMNodeIDLess compare; const auto it = std::lower_bound(first, last, value, compare); return (it == last || compare(value, *it)) ? SPECIAL_NODEID : static_cast(std::distance(first, it)); } + +template void sort_external_vector(T &vector, const Func &func) +{ +#if USE_STXXL_LIBRARY +#ifndef _MSC_VER + constexpr static unsigned stxxl_memory = + ((sizeof(std::size_t) == 4) ? std::numeric_limits::max() + : std::numeric_limits::max()); +#else + const static unsigned stxxl_memory = ((sizeof(std::size_t) == 4) ? INT_MAX : UINT_MAX); +#endif + stxxl::sort(vector.begin(), vector.end(), func, stxxl_memory); +#else + tbb::parallel_sort(vector.begin(), vector.end(), func); +#endif +} } namespace osrm @@ -128,8 +149,10 @@ namespace extractor ExtractionContainers::ExtractionContainers() { +#if USE_STXXL_LIBRARY // Check if stxxl can be instantiated stxxl::vector dummy_vector; +#endif // Insert four empty strings offsets for name, ref, destination, pronunciation, and exits name_offsets.push_back(0); @@ -143,12 +166,14 @@ ExtractionContainers::ExtractionContainers() void ExtractionContainers::FlushVectors() { +#if USE_STXXL_LIBRARY used_node_id_list.flush(); all_nodes_list.flush(); all_edges_list.flush(); name_char_data.flush(); name_offsets.flush(); way_start_end_id_list.flush(); +#endif } /** @@ -201,8 +226,7 @@ void ExtractionContainers::PrepareNodes() util::UnbufferedLog log; log << "Sorting used nodes ... " << std::flush; TIMER_START(sorting_used_nodes); - stxxl::sort( - used_node_id_list.begin(), used_node_id_list.end(), OSMNodeIDSTXXLLess(), stxxl_memory); + sort_external_vector(used_node_id_list, OSMNodeIDLess()); TIMER_STOP(sorting_used_nodes); log << "ok, after " << TIMER_SEC(sorting_used_nodes) << "s"; } @@ -218,7 +242,7 @@ void ExtractionContainers::PrepareNodes() } { - struct QueryNodeSTXXLCompare + struct QueryNodeCompare { using value_type = QueryNode; value_type max_value() { return value_type::max_value(); } @@ -233,8 +257,7 @@ void ExtractionContainers::PrepareNodes() util::UnbufferedLog log; log << "Sorting all nodes ... " << std::flush; TIMER_START(sorting_nodes); - stxxl::sort( - all_nodes_list.begin(), all_nodes_list.end(), QueryNodeSTXXLCompare(), stxxl_memory); + sort_external_vector(all_nodes_list, QueryNodeCompare()); TIMER_STOP(sorting_nodes); log << "ok, after " << TIMER_SEC(sorting_nodes) << "s"; } @@ -290,8 +313,7 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm util::UnbufferedLog log; log << "Sorting edges by start ... " << std::flush; TIMER_START(sort_edges_by_start); - stxxl::sort( - all_edges_list.begin(), all_edges_list.end(), CmpEdgeByOSMStartID(), stxxl_memory); + sort_external_vector(all_edges_list, CmpEdgeByOSMStartID()); TIMER_STOP(sort_edges_by_start); log << "ok, after " << TIMER_SEC(sort_edges_by_start) << "s"; } @@ -362,8 +384,7 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm util::UnbufferedLog log; log << "Sorting edges by target ... " << std::flush; TIMER_START(sort_edges_by_target); - stxxl::sort( - all_edges_list.begin(), all_edges_list.end(), CmpEdgeByOSMTargetID(), stxxl_memory); + sort_external_vector(all_edges_list, CmpEdgeByOSMTargetID()); TIMER_STOP(sort_edges_by_target); log << "ok, after " << TIMER_SEC(sort_edges_by_target) << "s"; } @@ -466,11 +487,9 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm log << "Sorting edges by renumbered start ... "; TIMER_START(sort_edges_by_renumbered_start); std::mutex name_data_mutex; - stxxl::sort( - all_edges_list.begin(), - all_edges_list.end(), - CmpEdgeByInternalSourceTargetAndName{name_data_mutex, name_char_data, name_offsets}, - stxxl_memory); + sort_external_vector( + all_edges_list, + CmpEdgeByInternalSourceTargetAndName{name_data_mutex, name_char_data, name_offsets}); TIMER_STOP(sort_edges_by_renumbered_start); log << "ok, after " << TIMER_SEC(sort_edges_by_renumbered_start) << "s"; } @@ -730,10 +749,7 @@ void ExtractionContainers::PrepareRestrictions() util::UnbufferedLog log; log << "Sorting used ways ... "; TIMER_START(sort_ways); - stxxl::sort(way_start_end_id_list.begin(), - way_start_end_id_list.end(), - FirstAndLastSegmentOfWayStxxlCompare(), - stxxl_memory); + sort_external_vector(way_start_end_id_list, FirstAndLastSegmentOfWayCompare()); TIMER_STOP(sort_ways); log << "ok, after " << TIMER_SEC(sort_ways) << "s"; } @@ -742,7 +758,7 @@ void ExtractionContainers::PrepareRestrictions() util::UnbufferedLog log; log << "Sorting " << restrictions_list.size() << " restriction. by from... "; TIMER_START(sort_restrictions); - std::sort( + tbb::parallel_sort( restrictions_list.begin(), restrictions_list.end(), CmpRestrictionContainerByFrom()); TIMER_STOP(sort_restrictions); log << "ok, after " << TIMER_SEC(sort_restrictions) << "s"; @@ -848,7 +864,7 @@ void ExtractionContainers::PrepareRestrictions() util::UnbufferedLog log; log << "Sorting restrictions. by to ... " << std::flush; TIMER_START(sort_restrictions_to); - std::sort( + tbb::parallel_sort( restrictions_list.begin(), restrictions_list.end(), CmpRestrictionContainerByTo()); TIMER_STOP(sort_restrictions_to); log << "ok, after " << TIMER_SEC(sort_restrictions_to) << "s"; diff --git a/unit_tests/util/range_table.cpp b/unit_tests/util/range_table.cpp index 4a7af4334..7c77e3507 100644 --- a/unit_tests/util/range_table.cpp +++ b/unit_tests/util/range_table.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include BOOST_AUTO_TEST_SUITE(range_table) @@ -15,7 +15,7 @@ using namespace osrm::util; constexpr unsigned BLOCK_SIZE = 16; typedef RangeTable TestRangeTable; -void ConstructionTest(stxxl::vector lengths, std::vector offsets) +void ConstructionTest(std::vector lengths, std::vector offsets) { BOOST_ASSERT(lengths.size() == offsets.size() - 1); @@ -29,7 +29,7 @@ void ConstructionTest(stxxl::vector lengths, std::vector off } } -void ComputeLengthsOffsets(stxxl::vector &lengths, +void ComputeLengthsOffsets(std::vector &lengths, std::vector &offsets, unsigned num) { @@ -54,12 +54,12 @@ void ComputeLengthsOffsets(stxxl::vector &lengths, BOOST_AUTO_TEST_CASE(construction_test) { // only offset empty block - stxxl::vector empty_lengths; + std::vector empty_lengths; empty_lengths.push_back(1); ConstructionTest(empty_lengths, {0, 1}); // first block almost full => sentinel is last element of block // [0] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, (16)} - stxxl::vector almost_full_lengths; + std::vector almost_full_lengths; std::vector almost_full_offsets; ComputeLengthsOffsets(almost_full_lengths, almost_full_offsets, BLOCK_SIZE); ConstructionTest(almost_full_lengths, almost_full_offsets); @@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(construction_test) // first block full => sentinel is offset of new block, next block empty // [0] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} // [(153)] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} - stxxl::vector full_lengths; + std::vector full_lengths; std::vector full_offsets; ComputeLengthsOffsets(full_lengths, full_offsets, BLOCK_SIZE + 1); ConstructionTest(full_lengths, full_offsets); @@ -75,13 +75,13 @@ BOOST_AUTO_TEST_CASE(construction_test) // first block full and offset of next block not sentinel, but the first differential value // [0] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} // [153] {(17), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} - stxxl::vector over_full_lengths; + std::vector over_full_lengths; std::vector over_full_offsets; ComputeLengthsOffsets(over_full_lengths, over_full_offsets, BLOCK_SIZE + 2); ConstructionTest(over_full_lengths, over_full_offsets); // test multiple blocks - stxxl::vector multiple_lengths; + std::vector multiple_lengths; std::vector multiple_offsets; ComputeLengthsOffsets(multiple_lengths, multiple_offsets, (BLOCK_SIZE + 1) * 10); ConstructionTest(multiple_lengths, multiple_offsets);