Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Switch from stxxl::vector to std::vector in extractor

Browse Source
This commit is contained in:
Michael Krasnyk 2017-07-07 12:09:52 +02:00 committed by Patrick Niklaus
parent a498ba6537
commit 3940cc1641
7 changed files with 96 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
include/extractor/extraction_containers.hpp
View File

@ -11,9 +11,12 @@
#include "storage/io.hpp" #include "storage/io.hpp"
#include <cstdint> #include <cstdint>
#include <stxxl/vector>
#include <unordered_map> #include <unordered_map>
#if USE_STXXL_LIBRARY
#include <stxxl/vector>
#endif
namespace osrm namespace osrm
{ {
namespace extractor namespace extractor
@ -27,13 +30,12 @@ namespace extractor
*/ */
class ExtractionContainers class ExtractionContainers
{ {
#ifndef _MSC_VER #if USE_STXXL_LIBRARY
constexpr static unsigned stxxl_memory = template <typename T> using ExternalVector = stxxl::vector<T>;
((sizeof(std::size_t) == 4) ? std::numeric_limits<int>::max()
: std::numeric_limits<unsigned>::max());
#else #else
const static unsigned stxxl_memory = ((sizeof(std::size_t) == 4) ? INT_MAX : UINT_MAX); template <typename T> using ExternalVector = std::vector<T>;
#endif #endif
void FlushVectors(); void FlushVectors();
void PrepareNodes(); void PrepareNodes();
void PrepareRestrictions(); void PrepareRestrictions();
@ -45,24 +47,24 @@ class ExtractionContainers
void WriteCharData(const std::string &file_name); void WriteCharData(const std::string &file_name);
public: public:
using STXXLNodeIDVector = stxxl::vector<OSMNodeID>; using NodeIDVector = ExternalVector<OSMNodeID>;
using STXXLNodeVector = stxxl::vector<QueryNode>; using NodeVector = ExternalVector<QueryNode>;
using STXXLEdgeVector = stxxl::vector<InternalExtractorEdge>; using EdgeVector = ExternalVector<InternalExtractorEdge>;
using RestrictionsVector = std::vector<InputRestrictionContainer>; using RestrictionsVector = std::vector<InputRestrictionContainer>;
using STXXLWayIDStartEndVector = stxxl::vector<FirstAndLastSegmentOfWay>; using WayIDStartEndVector = ExternalVector<FirstAndLastSegmentOfWay>;
using STXXLNameCharData = stxxl::vector<unsigned char>; using NameCharData = ExternalVector<unsigned char>;
using STXXLNameOffsets = stxxl::vector<unsigned>; using NameOffsets = ExternalVector<unsigned>;
std::vector<OSMNodeID> barrier_nodes; std::vector<OSMNodeID> barrier_nodes;
std::vector<OSMNodeID> traffic_lights; std::vector<OSMNodeID> traffic_lights;
STXXLNodeIDVector used_node_id_list; NodeIDVector used_node_id_list;
STXXLNodeVector all_nodes_list; NodeVector all_nodes_list;
STXXLEdgeVector all_edges_list; EdgeVector all_edges_list;
STXXLNameCharData name_char_data; NameCharData name_char_data;
STXXLNameOffsets name_offsets; NameOffsets name_offsets;
// an adjacency array containing all turn lane masks // an adjacency array containing all turn lane masks
RestrictionsVector restrictions_list; RestrictionsVector restrictions_list;
STXXLWayIDStartEndVector way_start_end_id_list; WayIDStartEndVector way_start_end_id_list;
unsigned max_internal_node_id; unsigned max_internal_node_id;
std::vector<TurnRestriction> unconditional_turn_restrictions; std::vector<TurnRestriction> unconditional_turn_restrictions;

2
include/extractor/first_and_last_segment_of_way.hpp
View File

@ -43,7 +43,7 @@ struct FirstAndLastSegmentOfWay
} }
}; };
struct FirstAndLastSegmentOfWayStxxlCompare struct FirstAndLastSegmentOfWayCompare
{ {
using value_type = FirstAndLastSegmentOfWay; using value_type = FirstAndLastSegmentOfWay;
bool operator()(const FirstAndLastSegmentOfWay &a, const FirstAndLastSegmentOfWay &b) const bool operator()(const FirstAndLastSegmentOfWay &a, const FirstAndLastSegmentOfWay &b) const

7
include/storage/serialization.hpp
View File

@ -7,8 +7,13 @@
#include "storage/io.hpp" #include "storage/io.hpp"
#include <cmath>
#include <cstdint> #include <cstdint>
#if USE_STXXL_LIBRARY
#include <stxxl/vector>
#endif
namespace osrm namespace osrm
{ {
namespace storage namespace storage
@ -58,6 +63,7 @@ inline void write(storage::io::FileWriter &writer, const util::DeallocatingVecto
writer.WriteFrom(vec.bucket_list.back(), last_block_size); writer.WriteFrom(vec.bucket_list.back(), last_block_size);
} }
#if USE_STXXL_LIBRARY
template <typename T> inline void read(storage::io::FileReader &reader, stxxl::vector<T> &vec) template <typename T> inline void read(storage::io::FileReader &reader, stxxl::vector<T> &vec)
{ {
auto size = reader.ReadOne<std::uint64_t>(); auto size = reader.ReadOne<std::uint64_t>();
@ -78,6 +84,7 @@ inline void write(storage::io::FileWriter &writer, const stxxl::vector<T> &vec)
writer.WriteOne<T>(vec[idx]); writer.WriteOne<T>(vec[idx]);
} }
} }
#endif
template <typename T> void read(io::FileReader &reader, std::vector<T> &data) template <typename T> void read(io::FileReader &reader, std::vector<T> &data)
{ {

7
include/util/meminfo.hpp
View File

@ -3,11 +3,14 @@
#include "util/log.hpp" #include "util/log.hpp"
#include <stxxl/mng>
#ifndef _WIN32 #ifndef _WIN32
#include <sys/resource.h> #include <sys/resource.h>
#endif #endif
#if USE_STXXL_LIBRARY
#include <stxxl/mng>
#endif
namespace osrm namespace osrm
{ {
namespace util namespace util
@ -15,6 +18,7 @@ namespace util
inline void DumpSTXXLStats() inline void DumpSTXXLStats()
{ {
#if USE_STXXL_LIBRARY
#if STXXL_VERSION_MAJOR > 1 || (STXXL_VERSION_MAJOR == 1 && STXXL_VERSION_MINOR >= 4) #if STXXL_VERSION_MAJOR > 1 || (STXXL_VERSION_MAJOR == 1 && STXXL_VERSION_MINOR >= 4)
auto manager = stxxl::block_manager::get_instance(); auto manager = stxxl::block_manager::get_instance();
util::Log() << "STXXL: peak bytes used: " << manager->get_maximum_allocation(); util::Log() << "STXXL: peak bytes used: " << manager->get_maximum_allocation();
@ -23,6 +27,7 @@ inline void DumpSTXXLStats()
#warning STXXL 1.4+ recommended - STXXL memory summary will not be available #warning STXXL 1.4+ recommended - STXXL memory summary will not be available
util::Log() << "STXXL: memory summary not available, needs STXXL 1.4 or higher"; util::Log() << "STXXL: memory summary not available, needs STXXL 1.4 or higher";
#endif #endif
#endif
} }
inline void DumpMemoryStats() inline void DumpMemoryStats()

14
include/util/vector_view.hpp
View File

@ -6,8 +6,6 @@
#include "storage/shared_memory_ownership.hpp" #include "storage/shared_memory_ownership.hpp"
#include <stxxl/vector>
#include <boost/assert.hpp> #include <boost/assert.hpp>
#include <boost/iterator/iterator_facade.hpp> #include <boost/iterator/iterator_facade.hpp>
#include <boost/iterator/reverse_iterator.hpp> #include <boost/iterator/reverse_iterator.hpp>
@ -21,6 +19,10 @@
#include <utility> #include <utility>
#include <vector> #include <vector>
#if USE_STXXL_LIBRARY
#include <stxxl/vector>
#endif
namespace osrm namespace osrm
{ {
namespace util namespace util
@ -209,10 +211,16 @@ template <typename DataT> void swap(vector_view<DataT> &lhs, vector_view<DataT>
std::swap(lhs.m_size, rhs.m_size); std::swap(lhs.m_size, rhs.m_size);
} }
#if USE_STXXL_LIBRARY
template <typename T> using ExternalVector = stxxl::vector<T>;
#else
template <typename T> using ExternalVector = std::vector<T>;
#endif
template <typename DataT, storage::Ownership Ownership> template <typename DataT, storage::Ownership Ownership>
using InternalOrExternalVector = using InternalOrExternalVector =
typename std::conditional<Ownership == storage::Ownership::External, typename std::conditional<Ownership == storage::Ownership::External,
stxxl::vector<DataT>, ExternalVector<DataT>,
std::vector<DataT>>::type; std::vector<DataT>>::type;
template <typename DataT, storage::Ownership Ownership> template <typename DataT, storage::Ownership Ownership>

70
src/extractor/extraction_containers.cpp
View File

@ -21,25 +21,30 @@
#include <boost/numeric/conversion/cast.hpp> #include <boost/numeric/conversion/cast.hpp>
#include <boost/ref.hpp> #include <boost/ref.hpp>
#include <stxxl/sort> #include <tbb/parallel_sort.h>
#include <chrono> #include <chrono>
#include <limits> #include <limits>
#include <mutex> #include <mutex>
#include <sstream> #include <sstream>
#if USE_STXXL_LIBRARY
#include <stxxl/sort>
#endif
namespace namespace
{ {
namespace oe = osrm::extractor; namespace oe = osrm::extractor;
// Needed for STXXL comparison - STXXL requires max_value(), min_value(), so we can not use // Needed for STXXL comparison - STXXL requires max_value(), min_value(), so we can not use
// std::less<OSMNodeId>{}. Anonymous namespace to keep translation unit local. // std::less<OSMNodeId>{}. Anonymous namespace to keep translation unit local.
struct OSMNodeIDSTXXLLess struct OSMNodeIDLess
{ {
OSMNodeIDSTXXLLess() {} OSMNodeIDLess() {}
using value_type = OSMNodeID; using value_type = OSMNodeID;
bool operator()(const value_type left, const value_type right) const { return left < right; } bool operator()(const value_type left, const value_type right) const { return left < right; }
value_type max_value() { return MAX_OSM_NODEID; } value_type max_value() { return MAX_OSM_NODEID; }
value_type min_value() { return MIN_OSM_NODEID; } value_type min_value() { return MIN_OSM_NODEID; }
}; };
@ -96,7 +101,7 @@ struct CmpEdgeByInternalSourceTargetAndName
std::lock_guard<std::mutex> lock(mutex); std::lock_guard<std::mutex> lock(mutex);
BOOST_ASSERT(!name_offsets.empty() && name_offsets.back() == name_data.size()); BOOST_ASSERT(!name_offsets.empty() && name_offsets.back() == name_data.size());
const oe::ExtractionContainers::STXXLNameCharData::const_iterator data = name_data.begin(); const oe::ExtractionContainers::NameCharData::const_iterator data = name_data.begin();
return std::lexicographical_compare(data + name_offsets[lhs.result.name_id], return std::lexicographical_compare(data + name_offsets[lhs.result.name_id],
data + name_offsets[lhs.result.name_id + 1], data + name_offsets[lhs.result.name_id + 1],
data + name_offsets[rhs.result.name_id], data + name_offsets[rhs.result.name_id],
@ -107,18 +112,34 @@ struct CmpEdgeByInternalSourceTargetAndName
value_type min_value() { return value_type::min_internal_value(); } value_type min_value() { return value_type::min_internal_value(); }
std::mutex &mutex; std::mutex &mutex;
const oe::ExtractionContainers::STXXLNameCharData &name_data; const oe::ExtractionContainers::NameCharData &name_data;
const oe::ExtractionContainers::STXXLNameOffsets &name_offsets; const oe::ExtractionContainers::NameOffsets &name_offsets;
}; };
template <typename Iter> template <typename Iter>
inline NodeID mapExternalToInternalNodeID(Iter first, Iter last, const OSMNodeID value) inline NodeID mapExternalToInternalNodeID(Iter first, Iter last, const OSMNodeID value)
{ {
const OSMNodeIDSTXXLLess compare; const OSMNodeIDLess compare;
const auto it = std::lower_bound(first, last, value, compare); const auto it = std::lower_bound(first, last, value, compare);
return (it == last || compare(value, *it)) ? SPECIAL_NODEID return (it == last || compare(value, *it)) ? SPECIAL_NODEID
: static_cast<NodeID>(std::distance(first, it)); : static_cast<NodeID>(std::distance(first, it));
} }
template <typename T, typename Func> void sort_external_vector(T &vector, const Func &func)
{
#if USE_STXXL_LIBRARY
#ifndef _MSC_VER
constexpr static unsigned stxxl_memory =
((sizeof(std::size_t) == 4) ? std::numeric_limits<int>::max()
: std::numeric_limits<unsigned>::max());
#else
const static unsigned stxxl_memory = ((sizeof(std::size_t) == 4) ? INT_MAX : UINT_MAX);
#endif
stxxl::sort(vector.begin(), vector.end(), func, stxxl_memory);
#else
tbb::parallel_sort(vector.begin(), vector.end(), func);
#endif
}
} }
namespace osrm namespace osrm
@ -128,8 +149,10 @@ namespace extractor
ExtractionContainers::ExtractionContainers() ExtractionContainers::ExtractionContainers()
{ {
#if USE_STXXL_LIBRARY
// Check if stxxl can be instantiated // Check if stxxl can be instantiated
stxxl::vector<unsigned> dummy_vector; stxxl::vector<unsigned> dummy_vector;
#endif
// Insert four empty strings offsets for name, ref, destination, pronunciation, and exits // Insert four empty strings offsets for name, ref, destination, pronunciation, and exits
name_offsets.push_back(0); name_offsets.push_back(0);
@ -143,12 +166,14 @@ ExtractionContainers::ExtractionContainers()
void ExtractionContainers::FlushVectors() void ExtractionContainers::FlushVectors()
{ {
#if USE_STXXL_LIBRARY
used_node_id_list.flush(); used_node_id_list.flush();
all_nodes_list.flush(); all_nodes_list.flush();
all_edges_list.flush(); all_edges_list.flush();
name_char_data.flush(); name_char_data.flush();
name_offsets.flush(); name_offsets.flush();
way_start_end_id_list.flush(); way_start_end_id_list.flush();
#endif
} }
/** /**
@ -201,8 +226,7 @@ void ExtractionContainers::PrepareNodes()
util::UnbufferedLog log; util::UnbufferedLog log;
log << "Sorting used nodes ... " << std::flush; log << "Sorting used nodes ... " << std::flush;
TIMER_START(sorting_used_nodes); TIMER_START(sorting_used_nodes);
stxxl::sort( sort_external_vector(used_node_id_list, OSMNodeIDLess());
used_node_id_list.begin(), used_node_id_list.end(), OSMNodeIDSTXXLLess(), stxxl_memory);
TIMER_STOP(sorting_used_nodes); TIMER_STOP(sorting_used_nodes);
log << "ok, after " << TIMER_SEC(sorting_used_nodes) << "s"; log << "ok, after " << TIMER_SEC(sorting_used_nodes) << "s";
} }
@ -218,7 +242,7 @@ void ExtractionContainers::PrepareNodes()
} }
{ {
struct QueryNodeSTXXLCompare struct QueryNodeCompare
{ {
using value_type = QueryNode; using value_type = QueryNode;
value_type max_value() { return value_type::max_value(); } value_type max_value() { return value_type::max_value(); }
@ -233,8 +257,7 @@ void ExtractionContainers::PrepareNodes()
util::UnbufferedLog log; util::UnbufferedLog log;
log << "Sorting all nodes ... " << std::flush; log << "Sorting all nodes ... " << std::flush;
TIMER_START(sorting_nodes); TIMER_START(sorting_nodes);
stxxl::sort( sort_external_vector(all_nodes_list, QueryNodeCompare());
all_nodes_list.begin(), all_nodes_list.end(), QueryNodeSTXXLCompare(), stxxl_memory);
TIMER_STOP(sorting_nodes); TIMER_STOP(sorting_nodes);
log << "ok, after " << TIMER_SEC(sorting_nodes) << "s"; log << "ok, after " << TIMER_SEC(sorting_nodes) << "s";
} }
@ -290,8 +313,7 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
util::UnbufferedLog log; util::UnbufferedLog log;
log << "Sorting edges by start ... " << std::flush; log << "Sorting edges by start ... " << std::flush;
TIMER_START(sort_edges_by_start); TIMER_START(sort_edges_by_start);
stxxl::sort( sort_external_vector(all_edges_list, CmpEdgeByOSMStartID());
all_edges_list.begin(), all_edges_list.end(), CmpEdgeByOSMStartID(), stxxl_memory);
TIMER_STOP(sort_edges_by_start); TIMER_STOP(sort_edges_by_start);
log << "ok, after " << TIMER_SEC(sort_edges_by_start) << "s"; log << "ok, after " << TIMER_SEC(sort_edges_by_start) << "s";
} }
@ -362,8 +384,7 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
util::UnbufferedLog log; util::UnbufferedLog log;
log << "Sorting edges by target ... " << std::flush; log << "Sorting edges by target ... " << std::flush;
TIMER_START(sort_edges_by_target); TIMER_START(sort_edges_by_target);
stxxl::sort( sort_external_vector(all_edges_list, CmpEdgeByOSMTargetID());
all_edges_list.begin(), all_edges_list.end(), CmpEdgeByOSMTargetID(), stxxl_memory);
TIMER_STOP(sort_edges_by_target); TIMER_STOP(sort_edges_by_target);
log << "ok, after " << TIMER_SEC(sort_edges_by_target) << "s"; log << "ok, after " << TIMER_SEC(sort_edges_by_target) << "s";
} }
@ -466,11 +487,9 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
log << "Sorting edges by renumbered start ... "; log << "Sorting edges by renumbered start ... ";
TIMER_START(sort_edges_by_renumbered_start); TIMER_START(sort_edges_by_renumbered_start);
std::mutex name_data_mutex; std::mutex name_data_mutex;
stxxl::sort( sort_external_vector(
all_edges_list.begin(), all_edges_list,
all_edges_list.end(), CmpEdgeByInternalSourceTargetAndName{name_data_mutex, name_char_data, name_offsets});
CmpEdgeByInternalSourceTargetAndName{name_data_mutex, name_char_data, name_offsets},
stxxl_memory);
TIMER_STOP(sort_edges_by_renumbered_start); TIMER_STOP(sort_edges_by_renumbered_start);
log << "ok, after " << TIMER_SEC(sort_edges_by_renumbered_start) << "s"; log << "ok, after " << TIMER_SEC(sort_edges_by_renumbered_start) << "s";
} }
@ -730,10 +749,7 @@ void ExtractionContainers::PrepareRestrictions()
util::UnbufferedLog log; util::UnbufferedLog log;
log << "Sorting used ways ... "; log << "Sorting used ways ... ";
TIMER_START(sort_ways); TIMER_START(sort_ways);
stxxl::sort(way_start_end_id_list.begin(), sort_external_vector(way_start_end_id_list, FirstAndLastSegmentOfWayCompare());
way_start_end_id_list.end(),
FirstAndLastSegmentOfWayStxxlCompare(),
stxxl_memory);
TIMER_STOP(sort_ways); TIMER_STOP(sort_ways);
log << "ok, after " << TIMER_SEC(sort_ways) << "s"; log << "ok, after " << TIMER_SEC(sort_ways) << "s";
} }
@ -742,7 +758,7 @@ void ExtractionContainers::PrepareRestrictions()
util::UnbufferedLog log; util::UnbufferedLog log;
log << "Sorting " << restrictions_list.size() << " restriction. by from... "; log << "Sorting " << restrictions_list.size() << " restriction. by from... ";
TIMER_START(sort_restrictions); TIMER_START(sort_restrictions);
std::sort( tbb::parallel_sort(
restrictions_list.begin(), restrictions_list.end(), CmpRestrictionContainerByFrom()); restrictions_list.begin(), restrictions_list.end(), CmpRestrictionContainerByFrom());
TIMER_STOP(sort_restrictions); TIMER_STOP(sort_restrictions);
log << "ok, after " << TIMER_SEC(sort_restrictions) << "s"; log << "ok, after " << TIMER_SEC(sort_restrictions) << "s";
@ -848,7 +864,7 @@ void ExtractionContainers::PrepareRestrictions()
util::UnbufferedLog log; util::UnbufferedLog log;
log << "Sorting restrictions. by to ... " << std::flush; log << "Sorting restrictions. by to ... " << std::flush;
TIMER_START(sort_restrictions_to); TIMER_START(sort_restrictions_to);
std::sort( tbb::parallel_sort(
restrictions_list.begin(), restrictions_list.end(), CmpRestrictionContainerByTo()); restrictions_list.begin(), restrictions_list.end(), CmpRestrictionContainerByTo());
TIMER_STOP(sort_restrictions_to); TIMER_STOP(sort_restrictions_to);
log << "ok, after " << TIMER_SEC(sort_restrictions_to) << "s"; log << "ok, after " << TIMER_SEC(sort_restrictions_to) << "s";

16
unit_tests/util/range_table.cpp
View File

@ -5,7 +5,7 @@
#include <boost/test/unit_test.hpp> #include <boost/test/unit_test.hpp>
#include <numeric> #include <numeric>
#include <stxxl/vector> #include <vector>
BOOST_AUTO_TEST_SUITE(range_table) BOOST_AUTO_TEST_SUITE(range_table)
@ -15,7 +15,7 @@ using namespace osrm::util;
constexpr unsigned BLOCK_SIZE = 16; constexpr unsigned BLOCK_SIZE = 16;
typedef RangeTable<BLOCK_SIZE, osrm::storage::Ownership::Container> TestRangeTable; typedef RangeTable<BLOCK_SIZE, osrm::storage::Ownership::Container> TestRangeTable;
void ConstructionTest(stxxl::vector<unsigned> lengths, std::vector<unsigned> offsets) void ConstructionTest(std::vector<unsigned> lengths, std::vector<unsigned> offsets)
{ {
BOOST_ASSERT(lengths.size() == offsets.size() - 1); BOOST_ASSERT(lengths.size() == offsets.size() - 1);
@ -29,7 +29,7 @@ void ConstructionTest(stxxl::vector<unsigned> lengths, std::vector<unsigned> off
} }
} }
void ComputeLengthsOffsets(stxxl::vector<unsigned> &lengths, void ComputeLengthsOffsets(std::vector<unsigned> &lengths,
std::vector<unsigned> &offsets, std::vector<unsigned> &offsets,
unsigned num) unsigned num)
{ {
@ -54,12 +54,12 @@ void ComputeLengthsOffsets(stxxl::vector<unsigned> &lengths,
BOOST_AUTO_TEST_CASE(construction_test) BOOST_AUTO_TEST_CASE(construction_test)
{ {
// only offset empty block // only offset empty block
stxxl::vector<unsigned> empty_lengths; std::vector<unsigned> empty_lengths;
empty_lengths.push_back(1); empty_lengths.push_back(1);
ConstructionTest(empty_lengths, {0, 1}); ConstructionTest(empty_lengths, {0, 1});
// first block almost full => sentinel is last element of block // first block almost full => sentinel is last element of block
// [0] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, (16)} // [0] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, (16)}
stxxl::vector<unsigned> almost_full_lengths; std::vector<unsigned> almost_full_lengths;
std::vector<unsigned> almost_full_offsets; std::vector<unsigned> almost_full_offsets;
ComputeLengthsOffsets(almost_full_lengths, almost_full_offsets, BLOCK_SIZE); ComputeLengthsOffsets(almost_full_lengths, almost_full_offsets, BLOCK_SIZE);
ConstructionTest(almost_full_lengths, almost_full_offsets); ConstructionTest(almost_full_lengths, almost_full_offsets);
@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(construction_test)
// first block full => sentinel is offset of new block, next block empty // first block full => sentinel is offset of new block, next block empty
// [0] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} // [0] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
// [(153)] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} // [(153)] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
stxxl::vector<unsigned> full_lengths; std::vector<unsigned> full_lengths;
std::vector<unsigned> full_offsets; std::vector<unsigned> full_offsets;
ComputeLengthsOffsets(full_lengths, full_offsets, BLOCK_SIZE + 1); ComputeLengthsOffsets(full_lengths, full_offsets, BLOCK_SIZE + 1);
ConstructionTest(full_lengths, full_offsets); ConstructionTest(full_lengths, full_offsets);
@ -75,13 +75,13 @@ BOOST_AUTO_TEST_CASE(construction_test)
// first block full and offset of next block not sentinel, but the first differential value // first block full and offset of next block not sentinel, but the first differential value
// [0] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} // [0] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
// [153] {(17), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} // [153] {(17), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
stxxl::vector<unsigned> over_full_lengths; std::vector<unsigned> over_full_lengths;
std::vector<unsigned> over_full_offsets; std::vector<unsigned> over_full_offsets;
ComputeLengthsOffsets(over_full_lengths, over_full_offsets, BLOCK_SIZE + 2); ComputeLengthsOffsets(over_full_lengths, over_full_offsets, BLOCK_SIZE + 2);
ConstructionTest(over_full_lengths, over_full_offsets); ConstructionTest(over_full_lengths, over_full_offsets);
// test multiple blocks // test multiple blocks
stxxl::vector<unsigned> multiple_lengths; std::vector<unsigned> multiple_lengths;
std::vector<unsigned> multiple_offsets; std::vector<unsigned> multiple_offsets;
ComputeLengthsOffsets(multiple_lengths, multiple_offsets, (BLOCK_SIZE + 1) * 10); ComputeLengthsOffsets(multiple_lengths, multiple_offsets, (BLOCK_SIZE + 1) * 10);
ConstructionTest(multiple_lengths, multiple_offsets); ConstructionTest(multiple_lengths, multiple_offsets);