From 9f9fde1f2bda12f069f2d2148366d9146cbccba9 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 13 Jun 2014 00:59:49 +0200 Subject: [PATCH 01/24] Fix missing include in BinaryHeap --- DataStructures/BinaryHeap.h | 1 + 1 file changed, 1 insertion(+) diff --git a/DataStructures/BinaryHeap.h b/DataStructures/BinaryHeap.h index 00d3782f3..6903be1b3 100644 --- a/DataStructures/BinaryHeap.h +++ b/DataStructures/BinaryHeap.h @@ -36,6 +36,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include template class ArrayStorage { From 8d46ee85f19e20f56c6a383c8624fb03d2a65626 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 13 Jun 2014 01:00:46 +0200 Subject: [PATCH 02/24] Add test for BinaryHeap --- CMakeLists.txt | 11 +- UnitTests/DataStructures/BinaryHeapTest.cpp | 154 ++++++++++++++++++++ UnitTests/datastructure_tests.cpp | 8 + 3 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 UnitTests/DataStructures/BinaryHeapTest.cpp create mode 100644 UnitTests/datastructure_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 7467dc08b..557a0b95c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,8 +41,9 @@ add_custom_command(OUTPUT ${CMAKE_SOURCE_DIR}/Util/FingerPrint.cpp FingerPrint.c VERBATIM) add_custom_target(FingerPrintConfigure DEPENDS ${CMAKE_SOURCE_DIR}/Util/FingerPrint.cpp) +add_custom_target(tests DEPENDS datastructure-tests) -set(BOOST_COMPONENTS date_time filesystem iostreams program_options regex system thread) +set(BOOST_COMPONENTS date_time filesystem iostreams program_options regex system thread unit_test_framework) configure_file( ${CMAKE_SOURCE_DIR}/Util/GitDescription.cpp.in @@ -65,6 +66,7 @@ file(GLOB CoordinateGlob DataStructures/Coordinate.cpp) file(GLOB AlgorithmGlob Algorithms/*.cpp) file(GLOB HttpGlob Server/Http/*.cpp) file(GLOB LibOSRMGlob Library/*.cpp) +file(GLOB DataStructureTestsGlob UnitTests/DataStructures/*.cpp) set( OSRMSources @@ -84,6 +86,9 @@ add_dependencies(FINGERPRINT FingerPrintConfigure) add_executable(osrm-routed routed.cpp ${ServerGlob}) add_executable(osrm-datastore datastore.cpp) +# Unit tests +add_executable(datastructure-tests EXCLUDE_FROM_ALL UnitTests/datastructure_tests.cpp ${DataStructureTestsGlob}) + # Check the release mode if(NOT CMAKE_BUILD_TYPE MATCHES Debug) set(CMAKE_BUILD_TYPE Release) @@ -114,6 +119,8 @@ if(CMAKE_BUILD_TYPE MATCHES Release) endif (HAS_LTO_FLAG) endif() +add_definitions(-DBOOST_TEST_DYN_LINK) + # Configuring compilers if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") # using Clang @@ -190,6 +197,8 @@ target_link_libraries(osrm-prepare ${Boost_LIBRARIES} FINGERPRINT GITDESCRIPTION target_link_libraries(osrm-routed ${Boost_LIBRARIES} ${OPTIONAL_SOCKET_LIBS} OSRM FINGERPRINT GITDESCRIPTION) target_link_libraries(osrm-datastore ${Boost_LIBRARIES} FINGERPRINT GITDESCRIPTION COORDLIB) +target_link_libraries(datastructure-tests ${Boost_LIBRARIES}) + find_package(Threads REQUIRED) target_link_libraries(osrm-extract ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(osrm-datastore ${CMAKE_THREAD_LIBS_INIT}) diff --git a/UnitTests/DataStructures/BinaryHeapTest.cpp b/UnitTests/DataStructures/BinaryHeapTest.cpp new file mode 100644 index 000000000..f4b200a1d --- /dev/null +++ b/UnitTests/DataStructures/BinaryHeapTest.cpp @@ -0,0 +1,154 @@ +#include "../../DataStructures/BinaryHeap.h" +#include "../../typedefs.h" + +#include +#include +#include + +#include + +BOOST_AUTO_TEST_SUITE(binary_heap) + +struct TestData +{ + unsigned value; +}; + +typedef NodeID TestNodeID; +typedef int TestKey; +typedef int TestWeight; +typedef boost::mpl::list, + MapStorage, + UnorderedMapStorage> storage_types; + + +template +struct RandomDataFixture +{ + RandomDataFixture() + { + for (unsigned i = 0; i < NUM_ELEM; i++) + { + data.push_back(TestData {i*3}); + weights.push_back((i+1)*100); + ids.push_back(i); + order.push_back(i); + } + + // Choosen by a fair W20 dice roll + std::mt19937 g(15); + + std::shuffle(order.begin(), order.end(), g); + } + + std::vector data; + std::vector weights; + std::vector ids; + std::vector order; +}; + +constexpr unsigned NUM_NODES = 100; + +BOOST_FIXTURE_TEST_CASE_TEMPLATE(insert_test, T, storage_types, RandomDataFixture) +{ + BinaryHeap heap(NUM_NODES); + + TestWeight min_weight = std::numeric_limits::max(); + TestNodeID min_id; + + for (unsigned idx : order) + { + BOOST_CHECK(!heap.WasInserted(ids[idx])); + + heap.Insert(ids[idx], weights[idx], data[idx]); + + BOOST_CHECK(heap.WasInserted(ids[idx])); + + if (weights[idx] < min_weight) + { + min_weight = weights[idx]; + min_id = ids[idx]; + } + BOOST_CHECK_EQUAL(min_id, heap.Min()); + } + + for (auto id : ids) + { + const auto& d = heap.GetData(id); + BOOST_CHECK_EQUAL(d.value, data[id].value); + + const auto& w = heap.GetKey(id); + BOOST_CHECK_EQUAL(w, weights[id]); + } +} + +BOOST_FIXTURE_TEST_CASE_TEMPLATE(delete_min_test, T, storage_types, RandomDataFixture) +{ + BinaryHeap heap(NUM_NODES); + + for (unsigned idx : order) + { + heap.Insert(ids[idx], weights[idx], data[idx]); + } + + for (auto id : ids) + { + BOOST_CHECK(!heap.WasRemoved(id)); + + BOOST_CHECK_EQUAL(heap.Min(), id); + BOOST_CHECK_EQUAL(id, heap.DeleteMin()); + if (id+1 < NUM_NODES) + BOOST_CHECK_EQUAL(heap.Min(), id+1); + + BOOST_CHECK(heap.WasRemoved(id)); + } +} + +BOOST_FIXTURE_TEST_CASE_TEMPLATE(delete_all_test, T, storage_types, RandomDataFixture) +{ + BinaryHeap heap(NUM_NODES); + + for (unsigned idx : order) + { + heap.Insert(ids[idx], weights[idx], data[idx]); + } + + heap.DeleteAll(); + + BOOST_CHECK(heap.Empty()); +} + +BOOST_FIXTURE_TEST_CASE_TEMPLATE(decrease_key_test, T, storage_types, RandomDataFixture<10>) +{ + BinaryHeap heap(10); + + for (unsigned idx : order) + { + heap.Insert(ids[idx], weights[idx], data[idx]); + } + + std::vector rids(ids); + std::reverse(rids.begin(), rids.end()); + + for (auto id : rids) + { + TestNodeID min_id = heap.Min(); + TestWeight min_weight = heap.GetKey(min_id); + + // decrease weight until we reach min weight + while (weights[id] > min_weight) + { + heap.DecreaseKey(id, weights[id]); + BOOST_CHECK_EQUAL(heap.Min(), min_id); + weights[id]--; + } + + // make weight smaller than min + weights[id] -= 2; + heap.DecreaseKey(id, weights[id]); + BOOST_CHECK_EQUAL(heap.Min(), id); + } +} + +BOOST_AUTO_TEST_SUITE_END() + diff --git a/UnitTests/datastructure_tests.cpp b/UnitTests/datastructure_tests.cpp new file mode 100644 index 000000000..2f7f6593b --- /dev/null +++ b/UnitTests/datastructure_tests.cpp @@ -0,0 +1,8 @@ +#define BOOST_TEST_MODULE datastructure tests + +#include + +/* + * This file will contain an automatically generated main function. + */ + From b32062f87587140ac328fe3e2356e5f6e2b5ed97 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 14 Jun 2014 15:23:56 +0200 Subject: [PATCH 03/24] Fix typo in RangeTable --- DataStructures/RangeTable.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DataStructures/RangeTable.h b/DataStructures/RangeTable.h index f9ec254bc..a45a2c249 100644 --- a/DataStructures/RangeTable.h +++ b/DataStructures/RangeTable.h @@ -117,7 +117,8 @@ public: { // the last value is used as sentinel block_offsets.push_back(lengths_prefix_sum); - block_idx = (block_idx + 1) % BLOCK_SIZE; + block_idx = 1; + last_length = 0; } while (0 != block_idx) From 69134f6d6a67fd56bdde8cba3ae2547f4e828852 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 14 Jun 2014 15:24:55 +0200 Subject: [PATCH 04/24] Add test for RangeTable --- UnitTests/DataStructures/RangeTableTest.cpp | 101 ++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 UnitTests/DataStructures/RangeTableTest.cpp diff --git a/UnitTests/DataStructures/RangeTableTest.cpp b/UnitTests/DataStructures/RangeTableTest.cpp new file mode 100644 index 000000000..60c299a01 --- /dev/null +++ b/UnitTests/DataStructures/RangeTableTest.cpp @@ -0,0 +1,101 @@ + +#include +#include + +#include "../../DataStructures/RangeTable.h" + +constexpr unsigned BLOCK_SIZE = 16; +typedef RangeTable TestRangeTable; + +BOOST_AUTO_TEST_SUITE(range_table) + +void ConstructionTest(std::vector lengths, std::vector offsets) +{ + BOOST_ASSERT(lengths.size() == offsets.size() - 1); + + TestRangeTable table(lengths); + + for (unsigned i = 0; i < lengths.size(); i++) + { + auto range = table.GetRange(i); + BOOST_CHECK_EQUAL(range.front(), offsets[i]); + BOOST_CHECK_EQUAL(range.back()+1, offsets[i+1]); + } +} + +void ComputeLengthsOffsets(std::vector& lengths, std::vector& offsets, unsigned num) +{ + lengths.resize(num); + offsets.resize(num+1); + std::iota(lengths.begin(), lengths.end(), 1); + offsets[0] = 0; + std::partial_sum(lengths.begin(), lengths.end(), offsets.begin()+1); + + std::stringstream l_ss; + l_ss << "Lengths: "; + for (auto l : lengths) + l_ss << l << ", "; + BOOST_TEST_MESSAGE(l_ss.str()); + std::stringstream o_ss; + o_ss << "Offsets: "; + for (auto o : offsets) + o_ss << o << ", "; + BOOST_TEST_MESSAGE(o_ss.str()); +} + +BOOST_AUTO_TEST_CASE(serialization_test) +{ + std::vector lengths; + std::vector offsets; + ComputeLengthsOffsets(lengths, offsets, (BLOCK_SIZE+1)*10); + + TestRangeTable in_table(lengths); + TestRangeTable out_table; + + std::stringstream ss; + ss << in_table; + ss >> out_table; + + for (unsigned i = 0; i < lengths.size(); i++) + { + auto range = out_table.GetRange(i); + BOOST_CHECK_EQUAL(range.front(), offsets[i]); + BOOST_CHECK_EQUAL(range.back()+1, offsets[i+1]); + } +} + +BOOST_AUTO_TEST_CASE(construction_test) +{ + // only offset empty block + ConstructionTest({1}, {0, 1}); + // first block almost full => sentinel is last element of block + // [0] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, (16)} + std::vector almost_full_lengths; + std::vector almost_full_offsets; + ComputeLengthsOffsets(almost_full_lengths, almost_full_offsets, BLOCK_SIZE); + ConstructionTest(almost_full_lengths, almost_full_offsets); + + // first block full => sentinel is offset of new block, next block empty + // [0] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} + // [(153)] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + std::vector full_lengths; + std::vector full_offsets; + ComputeLengthsOffsets(full_lengths, full_offsets, BLOCK_SIZE+1); + ConstructionTest(full_lengths, full_offsets); + + // first block full and offset of next block not sentinel, but the first differential value + // [0] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} + // [153] {(17), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + std::vector over_full_lengths; + std::vector over_full_offsets; + ComputeLengthsOffsets(over_full_lengths, over_full_offsets, BLOCK_SIZE+2); + ConstructionTest(over_full_lengths, over_full_offsets); + + // test multiple blocks + std::vector multiple_lengths; + std::vector multiple_offsets; + ComputeLengthsOffsets(multiple_lengths, multiple_offsets, (BLOCK_SIZE+1)*10); + ConstructionTest(multiple_lengths, multiple_offsets); +} + +BOOST_AUTO_TEST_SUITE_END() From 129e8ef98aa3bea31a35795b60fa168f8f08dd67 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 25 Jun 2014 00:20:58 +0200 Subject: [PATCH 05/24] Fix small errors in StaticGraph --- DataStructures/StaticGraph.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DataStructures/StaticGraph.h b/DataStructures/StaticGraph.h index 6a4649f48..c57d936af 100644 --- a/DataStructures/StaticGraph.h +++ b/DataStructures/StaticGraph.h @@ -162,11 +162,11 @@ template class StaticGraph #endif } - unsigned GetNumberOfNodes() const { return number_of_nodes -1; } + unsigned GetNumberOfNodes() const { return number_of_nodes; } unsigned GetNumberOfEdges() const { return number_of_edges; } - unsigned GetOutDegree(const NodeIterator n) const { return BeginEdges(n) - EndEdges(n) - 1; } + unsigned GetOutDegree(const NodeIterator n) const { return EndEdges(n) - BeginEdges(n); } inline NodeIterator GetTarget(const EdgeIterator e) const { From e776a51c73224b6433b087b1f7da450c2db6f0ff Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 25 Jun 2014 00:22:10 +0200 Subject: [PATCH 06/24] Added test for StaticGraph --- CMakeLists.txt | 4 +- UnitTests/DataStructures/StaticGraphTest.cpp | 175 +++++++++++++++++++ 2 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 UnitTests/DataStructures/StaticGraphTest.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 557a0b95c..2b0d71ace 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -196,7 +196,6 @@ target_link_libraries(osrm-extract ${Boost_LIBRARIES} FINGERPRINT GITDESCRIPTION target_link_libraries(osrm-prepare ${Boost_LIBRARIES} FINGERPRINT GITDESCRIPTION COORDLIB IMPORT) target_link_libraries(osrm-routed ${Boost_LIBRARIES} ${OPTIONAL_SOCKET_LIBS} OSRM FINGERPRINT GITDESCRIPTION) target_link_libraries(osrm-datastore ${Boost_LIBRARIES} FINGERPRINT GITDESCRIPTION COORDLIB) - target_link_libraries(datastructure-tests ${Boost_LIBRARIES}) find_package(Threads REQUIRED) @@ -205,6 +204,8 @@ target_link_libraries(osrm-datastore ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(osrm-prepare ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(OSRM ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(datastructure-tests ${CMAKE_THREAD_LIBS_INIT}) + find_package(TBB REQUIRED) if(WIN32 AND CMAKE_BUILD_TYPE MATCHES Debug) set(TBB_LIBRARIES ${TBB_DEBUG_LIBRARIES}) @@ -213,6 +214,7 @@ target_link_libraries(osrm-datastore ${TBB_LIBRARIES}) target_link_libraries(osrm-extract ${TBB_LIBRARIES}) target_link_libraries(osrm-prepare ${TBB_LIBRARIES}) target_link_libraries(osrm-routed ${TBB_LIBRARIES}) +target_link_libraries(datastructure-tests ${TBB_LIBRARIES}) include_directories(${TBB_INCLUDE_DIR}) find_package(Lua52) diff --git a/UnitTests/DataStructures/StaticGraphTest.cpp b/UnitTests/DataStructures/StaticGraphTest.cpp new file mode 100644 index 000000000..9434a3703 --- /dev/null +++ b/UnitTests/DataStructures/StaticGraphTest.cpp @@ -0,0 +1,175 @@ +#include "../../DataStructures/StaticGraph.h" +#include "../../typedefs.h" + +#include +#include +#include + +#include +#include + +BOOST_AUTO_TEST_SUITE(static_graph) + +struct TestData +{ + EdgeID id; + bool shortcut; + unsigned distance; +}; + +struct TestEdge +{ + unsigned source; + unsigned target; + unsigned distance; +}; + +typedef StaticGraph TestStaticGraph; +typedef TestStaticGraph::NodeArrayEntry TestNodeArrayEntry; +typedef TestStaticGraph::EdgeArrayEntry TestEdgeArrayEntry; +typedef TestStaticGraph::InputEdge TestInputEdge; + +constexpr unsigned TEST_NUM_NODES = 100; +constexpr unsigned TEST_NUM_EDGES = 500; +// Choosen by a fair W20 dice roll (this value is completely arbitrary) +constexpr unsigned RANDOM_SEED = 15; + +template +struct RandomArrayEntryFixture +{ + RandomArrayEntryFixture() + { + std::mt19937 g(RANDOM_SEED); + + std::uniform_int_distribution<> edge_udist(0, NUM_EDGES-1); + std::vector offsets; + for (unsigned i = 0; i < NUM_NODES; i++) + { + offsets.push_back(edge_udist(g)); + } + std::sort(offsets.begin(), offsets.end()); + // add sentinel + offsets.push_back(offsets.back()); + + // extract interval lengths + for(unsigned i = 0; i < offsets.size()-1; i++) + { + lengths.push_back(offsets[i+1] - offsets[i]); + } + lengths.push_back(NUM_EDGES - offsets[NUM_NODES-1]); + + for (auto offset : offsets) + { + nodes.emplace_back(TestNodeArrayEntry {offset}); + } + + std::uniform_int_distribution<> lengths_udist(0, 100000); + std::uniform_int_distribution<> node_udist(0, NUM_NODES-1); + for (unsigned i = 0; i < NUM_EDGES; i++) + { + edges.emplace_back( + TestEdgeArrayEntry { + static_cast(node_udist(g)), + TestData {i, false, static_cast(lengths_udist(g))} + } + ); + } + + for (unsigned i = 0; i < NUM_NODES; i++) + order.push_back(i); + std::shuffle(order.begin(), order.end(), g); + } + + typename ShM::vector nodes; + typename ShM::vector edges; + std::vector lengths; + std::vector order; +}; + +typedef RandomArrayEntryFixture TestRandomArrayEntryFixture; + +BOOST_FIXTURE_TEST_CASE(array_test, TestRandomArrayEntryFixture) +{ + auto nodes_copy = nodes; + + TestStaticGraph graph(nodes, edges); + + BOOST_CHECK_EQUAL(graph.GetNumberOfEdges(), TEST_NUM_EDGES); + BOOST_CHECK_EQUAL(graph.GetNumberOfNodes(), TEST_NUM_NODES); + + for (auto idx : order) + { + BOOST_CHECK_EQUAL(graph.BeginEdges((NodeID) idx), nodes_copy[idx].first_edge); + BOOST_CHECK_EQUAL(graph.EndEdges((NodeID) idx), nodes_copy[idx+1].first_edge); + BOOST_CHECK_EQUAL(graph.GetOutDegree((NodeID) idx), lengths[idx]); + } +} + +TestStaticGraph GraphFromEdgeList(const std::vector& edges) +{ + std::vector input_edges; + unsigned i = 0; + unsigned num_nodes = 0; + for (const auto& e : edges) + { + input_edges.push_back(TestInputEdge { + e.source, + e.target, + TestData {i++, false, e.distance} + }); + + num_nodes = std::max(num_nodes, std::max(e.source, e.target)); + } + + return TestStaticGraph(num_nodes, input_edges); +} + +BOOST_AUTO_TEST_CASE(find_test) +{ + /* + * (0) -1-> (1) + * ^ ^ + * 2 1 + * | | + * (3) -4-> (4) + * <-3- + */ + TestStaticGraph simple_graph = GraphFromEdgeList({ + TestEdge {0, 1, 1}, + TestEdge {3, 0, 2}, + TestEdge {3, 4, 4}, + TestEdge {4, 3, 3}, + TestEdge {3, 0, 1} + }); + + auto eit = simple_graph.FindEdge(0, 1); + BOOST_CHECK_EQUAL(simple_graph.GetEdgeData(eit).id, 0); + + eit = simple_graph.FindEdge(1, 0); + BOOST_CHECK_EQUAL(eit, SPECIAL_EDGEID); + + eit = simple_graph.FindEdgeInEitherDirection(1, 0); + BOOST_CHECK_EQUAL(simple_graph.GetEdgeData(eit).id, 0); + + bool reverse = false; + eit = simple_graph.FindEdgeIndicateIfReverse(1, 0, reverse); + BOOST_CHECK_EQUAL(simple_graph.GetEdgeData(eit).id, 0); + BOOST_CHECK(reverse); + + eit = simple_graph.FindEdge(3, 1); + BOOST_CHECK_EQUAL(eit, SPECIAL_EDGEID); + eit = simple_graph.FindEdge(0, 4); + BOOST_CHECK_EQUAL(eit, SPECIAL_EDGEID); + + eit = simple_graph.FindEdge(3, 4); + BOOST_CHECK_EQUAL(simple_graph.GetEdgeData(eit).id, 2); + eit = simple_graph.FindEdgeInEitherDirection(3, 4); + // I think this is wrong behaviour! Should be 3. + BOOST_CHECK_EQUAL(simple_graph.GetEdgeData(eit).id, 2); + + eit = simple_graph.FindEdge(3, 0); + BOOST_CHECK_EQUAL(simple_graph.GetEdgeData(eit).id, 4); +} + +BOOST_AUTO_TEST_SUITE_END() + From 3c4feecda05477500a22e512b1138d04f41bc9f6 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 27 Jun 2014 16:25:55 +0200 Subject: [PATCH 07/24] Make fstream non-static and StaticRTree thread-specific instead --- DataStructures/StaticRTree.h | 34 ++++++++----------- Server/DataStructures/BaseDataFacade.h | 6 ++-- Server/DataStructures/InternalDataFacade.h | 38 +++++++++++++++------- Server/DataStructures/SharedDataFacade.h | 36 ++++++++++++++------ 4 files changed, 69 insertions(+), 45 deletions(-) diff --git a/DataStructures/StaticRTree.h b/DataStructures/StaticRTree.h index c3603ce21..b3baa84f9 100644 --- a/DataStructures/StaticRTree.h +++ b/DataStructures/StaticRTree.h @@ -64,7 +64,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. const static uint32_t RTREE_BRANCHING_FACTOR = 64; const static uint32_t RTREE_LEAF_NODE_SIZE = 1024; -static boost::thread_specific_ptr thread_local_rtree_stream; // Implements a static, i.e. packed, R-tree template m_coordinate_list; + boost::filesystem::ifstream leaves_stream; public: StaticRTree() = delete; @@ -507,9 +507,8 @@ class StaticRTree throw OSRMException("mem index file is empty"); } - boost::filesystem::ifstream leaf_node_file(leaf_file, std::ios::binary); - leaf_node_file.read((char *)&m_element_count, sizeof(uint64_t)); - leaf_node_file.close(); + leaves_stream.open(leaf_file, std::ios::binary); + leaves_stream.read((char *)&m_element_count, sizeof(uint64_t)); // SimpleLogger().Write() << tree_size << " nodes in search tree"; // SimpleLogger().Write() << m_element_count << " elements in leafs"; @@ -532,14 +531,8 @@ class StaticRTree throw OSRMException("mem index file is empty"); } - boost::filesystem::ifstream leaf_node_file(leaf_file, std::ios::binary); - leaf_node_file.read((char *)&m_element_count, sizeof(uint64_t)); - leaf_node_file.close(); - - if (thread_local_rtree_stream.get()) - { - thread_local_rtree_stream->close(); - } + leaves_stream.open(leaf_file, std::ios::binary); + leaves_stream.read((char *)&m_element_count, sizeof(uint64_t)); // SimpleLogger().Write() << tree_size << " nodes in search tree"; // SimpleLogger().Write() << m_element_count << " elements in leafs"; @@ -988,22 +981,21 @@ class StaticRTree inline void LoadLeafFromDisk(const uint32_t leaf_id, LeafNode &result_node) { - if (!thread_local_rtree_stream.get() || !thread_local_rtree_stream->is_open()) + if (!leaves_stream.is_open()) { - thread_local_rtree_stream.reset(new boost::filesystem::ifstream( - m_leaf_node_filename, std::ios::in | std::ios::binary)); + leaves_stream.open(m_leaf_node_filename, std::ios::in | std::ios::binary); } - if (!thread_local_rtree_stream->good()) + if (!leaves_stream.good()) { - thread_local_rtree_stream->clear(std::ios::goodbit); + leaves_stream.clear(std::ios::goodbit); SimpleLogger().Write(logDEBUG) << "Resetting stale filestream"; } const uint64_t seek_pos = sizeof(uint64_t) + leaf_id * sizeof(LeafNode); - thread_local_rtree_stream->seekg(seek_pos); - BOOST_ASSERT_MSG(thread_local_rtree_stream->good(), + leaves_stream.seekg(seek_pos); + BOOST_ASSERT_MSG(leaves_stream.good(), "Seeking to position in leaf file failed."); - thread_local_rtree_stream->read((char *)&result_node, sizeof(LeafNode)); - BOOST_ASSERT_MSG(thread_local_rtree_stream->good(), "Reading from leaf file failed."); + leaves_stream.read((char *)&result_node, sizeof(LeafNode)); + BOOST_ASSERT_MSG(leaves_stream.good(), "Reading from leaf file failed."); } inline bool EdgesAreEquivalent(const FixedPointCoordinate &a, diff --git a/Server/DataStructures/BaseDataFacade.h b/Server/DataStructures/BaseDataFacade.h index 7897ca4c4..6152554bd 100644 --- a/Server/DataStructures/BaseDataFacade.h +++ b/Server/DataStructures/BaseDataFacade.h @@ -95,16 +95,16 @@ template class BaseDataFacade virtual bool LocateClosestEndPointForCoordinate(const FixedPointCoordinate &input_coordinate, FixedPointCoordinate &result, - const unsigned zoom_level = 18) const = 0; + const unsigned zoom_level = 18) = 0; virtual bool FindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate, PhantomNode &resulting_phantom_node, - const unsigned zoom_level) const = 0; + const unsigned zoom_level) = 0; virtual bool IncrementalFindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate, std::vector &resulting_phantom_node_vector, const unsigned zoom_level, - const unsigned number_of_results) const = 0; + const unsigned number_of_results) = 0; virtual unsigned GetCheckSum() const = 0; diff --git a/Server/DataStructures/InternalDataFacade.h b/Server/DataStructures/InternalDataFacade.h index f6b290d56..1c622e67f 100644 --- a/Server/DataStructures/InternalDataFacade.h +++ b/Server/DataStructures/InternalDataFacade.h @@ -71,8 +71,10 @@ template class InternalDataFacade : public BaseDataFacade::vector m_geometry_indices; ShM::vector m_geometry_list; - std::shared_ptr::vector, false>> + boost::thread_specific_ptr::vector, false>> m_static_rtree; + boost::filesystem::path ram_index_path; + boost::filesystem::path file_index_path; RangeTable<16, false> m_name_table; void LoadTimestamp(const boost::filesystem::path ×tamp_path) @@ -192,13 +194,13 @@ template class InternalDataFacade : public BaseDataFacadeempty(), "coordinates must be loaded before r-tree"); - m_static_rtree = std::make_shared>( - ram_index_path, file_index_path, m_coordinate_list); + m_static_rtree.reset( + new StaticRTree(ram_index_path, file_index_path, m_coordinate_list) + ); } void LoadStreetNames(const boost::filesystem::path &names_file) @@ -266,10 +268,10 @@ template class InternalDataFacade : public BaseDataFacadesecond; paths_iterator = server_paths.find("ramindex"); BOOST_ASSERT(server_paths.end() != paths_iterator); - const boost::filesystem::path &ram_index_path = paths_iterator->second; + ram_index_path = paths_iterator->second; paths_iterator = server_paths.find("fileindex"); BOOST_ASSERT(server_paths.end() != paths_iterator); - const boost::filesystem::path &file_index_path = paths_iterator->second; + file_index_path = paths_iterator->second; paths_iterator = server_paths.find("nodesdata"); BOOST_ASSERT(server_paths.end() != paths_iterator); const boost::filesystem::path &nodes_data_path = paths_iterator->second; @@ -297,7 +299,6 @@ template class InternalDataFacade : public BaseDataFacade class InternalDataFacade : public BaseDataFacadeLocateClosestEndPointForCoordinate( input_coordinate, result, zoom_level); } bool FindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate, PhantomNode &resulting_phantom_node, - const unsigned zoom_level) const + const unsigned zoom_level) { + if (!m_static_rtree.get()) + { + LoadRTree(); + } + return m_static_rtree->FindPhantomNodeForCoordinate( input_coordinate, resulting_phantom_node, zoom_level); } @@ -376,8 +387,13 @@ template class InternalDataFacade : public BaseDataFacade &resulting_phantom_node_vector, const unsigned zoom_level, - const unsigned number_of_results) const + const unsigned number_of_results) { + if (!m_static_rtree.get()) + { + LoadRTree(); + } + return m_static_rtree->IncrementalFindPhantomNodeForCoordinate( input_coordinate, resulting_phantom_node_vector, zoom_level, number_of_results); } diff --git a/Server/DataStructures/SharedDataFacade.h b/Server/DataStructures/SharedDataFacade.h index 5f89ea677..6eef51bdc 100644 --- a/Server/DataStructures/SharedDataFacade.h +++ b/Server/DataStructures/SharedDataFacade.h @@ -83,8 +83,9 @@ template class SharedDataFacade : public BaseDataFacade::vector m_geometry_indices; ShM::vector m_geometry_list; - std::shared_ptr::vector, true>> + boost::thread_specific_ptr::vector, true>> m_static_rtree; + boost::filesystem::path file_index_path; std::shared_ptr> m_name_table; @@ -105,18 +106,19 @@ template class SharedDataFacade : public BaseDataFacadeempty(), "coordinates must be loaded before r-tree"); RTreeNode *tree_ptr = data_layout->GetBlockPtr(shared_memory, SharedDataLayout::R_SEARCH_TREE); - m_static_rtree = - std::make_shared::vector, true>>( + m_static_rtree.reset( + new StaticRTree::vector, true>( tree_ptr, data_layout->num_entries[SharedDataLayout::R_SEARCH_TREE], file_index_path, - m_coordinate_list); + m_coordinate_list) + ); } void LoadGraph() @@ -248,7 +250,7 @@ template class SharedDataFacade : public BaseDataFacadeGetBlockPtr(shared_memory, SharedDataLayout::FILE_INDEX_PATH); - boost::filesystem::path file_index_path(file_index_ptr); + file_index_path = boost::filesystem::path(file_index_ptr); if (!boost::filesystem::exists(file_index_path)) { SimpleLogger().Write(logDEBUG) << "Leaf file name " << file_index_path.string(); @@ -260,7 +262,6 @@ template class SharedDataFacade : public BaseDataFacade class SharedDataFacade : public BaseDataFacadeLocateClosestEndPointForCoordinate( input_coordinate, result, zoom_level); } bool FindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate, PhantomNode &resulting_phantom_node, - const unsigned zoom_level) const + const unsigned zoom_level) { + if (!m_static_rtree.get()) + { + LoadRTree(); + } + return m_static_rtree->FindPhantomNodeForCoordinate( input_coordinate, resulting_phantom_node, zoom_level); } @@ -367,8 +378,13 @@ template class SharedDataFacade : public BaseDataFacade &resulting_phantom_node_vector, const unsigned zoom_level, - const unsigned number_of_results) const + const unsigned number_of_results) { + if (!m_static_rtree.get()) + { + LoadRTree(); + } + return m_static_rtree->IncrementalFindPhantomNodeForCoordinate( input_coordinate, resulting_phantom_node_vector, zoom_level, number_of_results); } From 8f05fc0a842ebab886912d56375b0b05d2576ad1 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 27 Jun 2014 16:59:11 +0200 Subject: [PATCH 08/24] Make tuning constants template agruments in StaticRTree --- DataStructures/StaticRTree.h | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/DataStructures/StaticRTree.h b/DataStructures/StaticRTree.h index b3baa84f9..4b5ed0f53 100644 --- a/DataStructures/StaticRTree.h +++ b/DataStructures/StaticRTree.h @@ -60,15 +60,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -// tuning parameters -const static uint32_t RTREE_BRANCHING_FACTOR = 64; -const static uint32_t RTREE_LEAF_NODE_SIZE = 1024; - - // Implements a static, i.e. packed, R-tree template , - bool UseSharedMemory = false> + bool UseSharedMemory = false, + uint32_t BRANCHING_FACTOR=64, + uint32_t LEAF_NODE_SIZE=1024> class StaticRTree { public: @@ -79,7 +76,7 @@ class StaticRTree int32_t min_lon, max_lon; int32_t min_lat, max_lat; - inline void InitializeMBRectangle(const std::array &objects, + inline void InitializeMBRectangle(const std::array &objects, const uint32_t element_count, const std::vector &coordinate_list) { @@ -219,7 +216,7 @@ class StaticRTree RectangleT minimum_bounding_rectangle; uint32_t child_count : 31; bool child_is_on_disk : 1; - uint32_t children[RTREE_BRANCHING_FACTOR]; + uint32_t children[BRANCHING_FACTOR]; }; private: @@ -243,9 +240,9 @@ class StaticRTree struct LeafNode { - LeafNode() : object_count(0) {} + LeafNode() : object_count(0), objects() {} uint32_t object_count; - std::array objects; + std::array objects; }; struct QueryCandidate @@ -369,7 +366,7 @@ class StaticRTree TreeNode current_node; // SimpleLogger().Write() << "reading " << tree_size << " tree nodes in " << // (sizeof(TreeNode)*tree_size) << " bytes"; - for (uint32_t current_element_index = 0; RTREE_LEAF_NODE_SIZE > current_element_index; + for (uint32_t current_element_index = 0; LEAF_NODE_SIZE > current_element_index; ++current_element_index) { if (m_element_count > (processed_objects_count + current_element_index)) @@ -406,9 +403,9 @@ class StaticRTree while (processed_tree_nodes_in_level < tree_nodes_in_level.size()) { TreeNode parent_node; - // pack RTREE_BRANCHING_FACTOR elements into tree_nodes each + // pack BRANCHING_FACTOR elements into tree_nodes each for (uint32_t current_child_node_index = 0; - RTREE_BRANCHING_FACTOR > current_child_node_index; + BRANCHING_FACTOR > current_child_node_index; ++current_child_node_index) { if (processed_tree_nodes_in_level < tree_nodes_in_level.size()) @@ -621,7 +618,7 @@ class StaticRTree std::vector &result_phantom_node_vector, const unsigned zoom_level, const unsigned number_of_results, - const unsigned max_checked_segments = 4*RTREE_LEAF_NODE_SIZE) + const unsigned max_checked_segments = 4*LEAF_NODE_SIZE) { // TIMER_START(samet); // SimpleLogger().Write(logDEBUG) << "searching for " << number_of_results << " results"; From a3dd9c3e57b25ba59e51314d12c2669c888630d8 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 28 Jun 2014 20:42:48 +0200 Subject: [PATCH 09/24] Change StaticRTree serialization constructor to static function Since the constructor does not satisfy the requirements for a constructor (the RTree is not properly initialized) make it a static function instead. --- Contractor/Prepare.cpp | 18 ++++------------ Contractor/Prepare.h | 1 - DataStructures/StaticRTree.h | 40 +++++++++++++++++++----------------- 3 files changed, 25 insertions(+), 34 deletions(-) diff --git a/Contractor/Prepare.cpp b/Contractor/Prepare.cpp index eee1124b1..e12a61b42 100644 --- a/Contractor/Prepare.cpp +++ b/Contractor/Prepare.cpp @@ -170,7 +170,10 @@ int Prepare::Process(int argc, char *argv[]) TIMER_STOP(expansion); - BuildRTree(node_based_edge_list); + StaticRTree::Build(node_based_edge_list, + rtree_nodes_path.c_str(), + rtree_leafs_path.c_str(), + internal_to_external_node_map); IteratorbasedCRC32> crc32; const unsigned node_based_edge_list_CRC32 = @@ -544,16 +547,3 @@ void Prepare::WriteNodeMapping() internal_to_external_node_map.shrink_to_fit(); } -/** - \brief Building rtree-based nearest-neighbor data structure - - Saves info to files: '.ramIndex' and '.fileIndex'. - */ -void Prepare::BuildRTree(std::vector &node_based_edge_list) -{ - SimpleLogger().Write() << "building r-tree ..."; - StaticRTree(node_based_edge_list, - rtree_nodes_path.c_str(), - rtree_leafs_path.c_str(), - internal_to_external_node_map); -} diff --git a/Contractor/Prepare.h b/Contractor/Prepare.h index 0a5c35985..6386a5227 100644 --- a/Contractor/Prepare.h +++ b/Contractor/Prepare.h @@ -40,7 +40,6 @@ class Prepare DeallocatingVector &edgeBasedEdgeList, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile); void WriteNodeMapping(); - void BuildRTree(std::vector &node_based_edge_list); private: std::vector internal_to_external_node_map; diff --git a/DataStructures/StaticRTree.h b/DataStructures/StaticRTree.h index 4b5ed0f53..1cea86e84 100644 --- a/DataStructures/StaticRTree.h +++ b/DataStructures/StaticRTree.h @@ -307,24 +307,24 @@ class StaticRTree StaticRTree(const StaticRTree &) = delete; // Construct a packed Hilbert-R-Tree with Kamel-Faloutsos algorithm [1] - explicit StaticRTree(std::vector &input_data_vector, + static void Build(std::vector &input_data_vector, const std::string tree_node_filename, const std::string leaf_node_filename, const std::vector &coordinate_list) - : m_element_count(input_data_vector.size()), m_leaf_node_filename(leaf_node_filename) { - SimpleLogger().Write() << "constructing r-tree of " << m_element_count + uint64_t element_count = input_data_vector.size(); + SimpleLogger().Write() << "constructing r-tree of " << element_count << " edge elements build on-top of " << coordinate_list.size() << " coordinates"; TIMER_START(construction); - std::vector input_wrapper_vector(m_element_count); + std::vector input_wrapper_vector(element_count); HilbertCode get_hilbert_number; // generate auxiliary vector of hilbert-values tbb::parallel_for( - tbb::blocked_range(0, m_element_count), + tbb::blocked_range(0, element_count), [&input_data_vector, &input_wrapper_vector, &get_hilbert_number, &coordinate_list]( const tbb::blocked_range &range) { @@ -351,7 +351,7 @@ class StaticRTree // open leaf file boost::filesystem::ofstream leaf_node_file(leaf_node_filename, std::ios::binary); - leaf_node_file.write((char *)&m_element_count, sizeof(uint64_t)); + leaf_node_file.write((char *) &element_count, sizeof(uint64_t)); // sort the hilbert-value representatives tbb::parallel_sort(input_wrapper_vector.begin(), input_wrapper_vector.end()); @@ -359,7 +359,7 @@ class StaticRTree // pack M elements into leaf node and write to leaf file uint64_t processed_objects_count = 0; - while (processed_objects_count < m_element_count) + while (processed_objects_count < element_count) { LeafNode current_leaf; @@ -369,7 +369,7 @@ class StaticRTree for (uint32_t current_element_index = 0; LEAF_NODE_SIZE > current_element_index; ++current_element_index) { - if (m_element_count > (processed_objects_count + current_element_index)) + if (element_count > (processed_objects_count + current_element_index)) { uint32_t index_of_next_object = input_wrapper_vector[processed_objects_count + current_element_index] @@ -395,6 +395,8 @@ class StaticRTree // close leaf file leaf_node_file.close(); + typename ShM::vector search_tree; + uint32_t processing_level = 0; while (1 < tree_nodes_in_level.size()) { @@ -413,8 +415,8 @@ class StaticRTree TreeNode ¤t_child_node = tree_nodes_in_level[processed_tree_nodes_in_level]; // add tree node to parent entry - parent_node.children[current_child_node_index] = m_search_tree.size(); - m_search_tree.emplace_back(current_child_node); + parent_node.children[current_child_node_index] = search_tree.size(); + search_tree.emplace_back(current_child_node); // merge MBRs parent_node.minimum_bounding_rectangle.MergeBoundingBoxes( current_child_node.minimum_bounding_rectangle); @@ -430,18 +432,18 @@ class StaticRTree } BOOST_ASSERT_MSG(1 == tree_nodes_in_level.size(), "tree broken, more than one root node"); // last remaining entry is the root node, store it - m_search_tree.emplace_back(tree_nodes_in_level[0]); + search_tree.emplace_back(tree_nodes_in_level[0]); // reverse and renumber tree to have root at index 0 - std::reverse(m_search_tree.begin(), m_search_tree.end()); + std::reverse(search_tree.begin(), search_tree.end()); - uint32_t search_tree_size = m_search_tree.size(); + uint32_t search_tree_size = search_tree.size(); tbb::parallel_for(tbb::blocked_range(0, search_tree_size), - [this, &search_tree_size](const tbb::blocked_range &range) - { + [&search_tree, &search_tree_size](const tbb::blocked_range &range) + { for (uint32_t i = range.begin(); i != range.end(); ++i) { - TreeNode ¤t_tree_node = this->m_search_tree[i]; + TreeNode ¤t_tree_node = search_tree[i]; for (uint32_t j = 0; j < current_tree_node.child_count; ++j) { const uint32_t old_id = current_tree_node.children[j]; @@ -454,10 +456,10 @@ class StaticRTree // open tree file boost::filesystem::ofstream tree_node_file(tree_node_filename, std::ios::binary); - uint32_t size_of_tree = m_search_tree.size(); + uint32_t size_of_tree = search_tree.size(); BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty"); - tree_node_file.write((char *)&size_of_tree, sizeof(uint32_t)); - tree_node_file.write((char *)&m_search_tree[0], sizeof(TreeNode) * size_of_tree); + tree_node_file.write((char *) &size_of_tree, sizeof(uint32_t)); + tree_node_file.write((char *) &search_tree[0], sizeof(TreeNode) * size_of_tree); // close tree node file. tree_node_file.close(); From 1c80584206eb0ceb42622c3dc39b17347d7347c9 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 30 Jun 2014 01:18:36 +0200 Subject: [PATCH 10/24] Fix GetMinDistance --- DataStructures/StaticRTree.h | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/DataStructures/StaticRTree.h b/DataStructures/StaticRTree.h index 1cea86e84..a11daf034 100644 --- a/DataStructures/StaticRTree.h +++ b/DataStructures/StaticRTree.h @@ -143,19 +143,25 @@ class StaticRTree return 0.; } + const FixedPointCoordinate upper_left(max_lat, min_lon); + const FixedPointCoordinate upper_right(max_lat, max_lon); + const FixedPointCoordinate lower_right(min_lat, max_lon); + const FixedPointCoordinate lower_left(min_lat, min_lon); + float min_dist = std::numeric_limits::max(); min_dist = std::min(min_dist, - FixedPointCoordinate::ApproximateEuclideanDistance( - location.lat, location.lon, max_lat, min_lon)); + FixedPointCoordinate::ComputePerpendicularDistance( + upper_left, upper_right, location)); min_dist = std::min(min_dist, - FixedPointCoordinate::ApproximateEuclideanDistance( - location.lat, location.lon, max_lat, max_lon)); + FixedPointCoordinate::ComputePerpendicularDistance( + upper_right, lower_right, location)); min_dist = std::min(min_dist, - FixedPointCoordinate::ApproximateEuclideanDistance( - location.lat, location.lon, min_lat, max_lon)); + FixedPointCoordinate::ComputePerpendicularDistance( + lower_right, lower_left, location)); min_dist = std::min(min_dist, - FixedPointCoordinate::ApproximateEuclideanDistance( - location.lat, location.lon, min_lat, min_lon)); + FixedPointCoordinate::ComputePerpendicularDistance( + lower_left, upper_left, location)); + return min_dist; } @@ -163,10 +169,10 @@ class StaticRTree { float min_max_dist = std::numeric_limits::max(); // Get minmax distance to each of the four sides - FixedPointCoordinate upper_left(max_lat, min_lon); - FixedPointCoordinate upper_right(max_lat, max_lon); - FixedPointCoordinate lower_right(min_lat, max_lon); - FixedPointCoordinate lower_left(min_lat, min_lon); + const FixedPointCoordinate upper_left(max_lat, min_lon); + const FixedPointCoordinate upper_right(max_lat, max_lon); + const FixedPointCoordinate lower_right(min_lat, max_lon); + const FixedPointCoordinate lower_left(min_lat, min_lon); min_max_dist = std::min( min_max_dist, From 8108ecc4d33abdcb768d5891632a2a81692a77d4 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 27 Jun 2014 01:27:45 +0200 Subject: [PATCH 11/24] Add test for StaticRTree --- CMakeLists.txt | 4 +- UnitTests/DataStructures/StaticRTreeTest.cpp | 347 +++++++++++++++++++ 2 files changed, 349 insertions(+), 2 deletions(-) create mode 100644 UnitTests/DataStructures/StaticRTreeTest.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b0d71ace..b743d818f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,7 +66,7 @@ file(GLOB CoordinateGlob DataStructures/Coordinate.cpp) file(GLOB AlgorithmGlob Algorithms/*.cpp) file(GLOB HttpGlob Server/Http/*.cpp) file(GLOB LibOSRMGlob Library/*.cpp) -file(GLOB DataStructureTestsGlob UnitTests/DataStructures/*.cpp) +file(GLOB DataStructureTestsGlob UnitTests/DataStructures/*.cpp DataStructures/HilbertValue.cpp) set( OSRMSources @@ -196,7 +196,7 @@ target_link_libraries(osrm-extract ${Boost_LIBRARIES} FINGERPRINT GITDESCRIPTION target_link_libraries(osrm-prepare ${Boost_LIBRARIES} FINGERPRINT GITDESCRIPTION COORDLIB IMPORT) target_link_libraries(osrm-routed ${Boost_LIBRARIES} ${OPTIONAL_SOCKET_LIBS} OSRM FINGERPRINT GITDESCRIPTION) target_link_libraries(osrm-datastore ${Boost_LIBRARIES} FINGERPRINT GITDESCRIPTION COORDLIB) -target_link_libraries(datastructure-tests ${Boost_LIBRARIES}) +target_link_libraries(datastructure-tests ${Boost_LIBRARIES} COORDLIB) find_package(Threads REQUIRED) target_link_libraries(osrm-extract ${CMAKE_THREAD_LIBS_INIT}) diff --git a/UnitTests/DataStructures/StaticRTreeTest.cpp b/UnitTests/DataStructures/StaticRTreeTest.cpp new file mode 100644 index 000000000..deef96b5a --- /dev/null +++ b/UnitTests/DataStructures/StaticRTreeTest.cpp @@ -0,0 +1,347 @@ +#include "../../DataStructures/StaticRTree.h" +#include "../../DataStructures/QueryNode.h" +#include "../../DataStructures/EdgeBasedNode.h" +#include "../../Include/osrm/Coordinate.h" +#include "../../typedefs.h" + +#include +#include +#include + +#include +#include + +BOOST_AUTO_TEST_SUITE(static_rtree) + +constexpr uint32_t TEST_BRANCHING_FACTOR = 8; +constexpr uint32_t TEST_LEAF_NODE_SIZE = 64; + +typedef EdgeBasedNode TestData; +typedef StaticRTree, + false, + TEST_BRANCHING_FACTOR, + TEST_LEAF_NODE_SIZE> + TestStaticRTree; + +// Choosen by a fair W20 dice roll (this value is completely arbitrary) +constexpr unsigned RANDOM_SEED = 15; +constexpr int32_t WORLD_MIN_LAT = -90*COORDINATE_PRECISION; +constexpr int32_t WORLD_MAX_LAT = 90*COORDINATE_PRECISION; +constexpr int32_t WORLD_MIN_LON = -180*COORDINATE_PRECISION; +constexpr int32_t WORLD_MAX_LON = 180*COORDINATE_PRECISION; + +class LinearSearchNN +{ +public: + LinearSearchNN(const std::shared_ptr>& coords, + const std::vector& edges) + : coords(coords) + , edges(edges) + { } + + bool LocateClosestEndPointForCoordinate(const FixedPointCoordinate &input_coordinate, + FixedPointCoordinate &result_coordinate, + const unsigned zoom_level) + { + bool ignore_tiny_components = (zoom_level <= 14); + + float min_dist = std::numeric_limits::max(); + FixedPointCoordinate min_coord; + for (const TestData& e : edges) + { + if (ignore_tiny_components && e.is_in_tiny_cc) + continue; + + const FixedPointCoordinate& start = coords->at(e.u); + const FixedPointCoordinate& end = coords->at(e.v); + float distance = FixedPointCoordinate::ApproximateEuclideanDistance( + input_coordinate.lat, + input_coordinate.lon, + start.lat, + start.lon); + if (distance < min_dist) + { + min_coord = start; + min_dist = distance; + } + + distance = FixedPointCoordinate::ApproximateEuclideanDistance( + input_coordinate.lat, + input_coordinate.lon, + end.lat, + end.lon); + if (distance < min_dist) + { + min_coord = end; + min_dist = distance; + } + } + + result_coordinate = min_coord; + return result_coordinate.isValid(); + } + + bool FindPhantomNodeForCoordinate(const FixedPointCoordinate &input_coordinate, + PhantomNode &result_phantom_node, + const unsigned zoom_level) + { + bool ignore_tiny_components = (zoom_level <= 14); + + float min_dist = std::numeric_limits::max(); + TestData nearest_edge; + for (const TestData& e : edges) + { + if (ignore_tiny_components && e.is_in_tiny_cc) + continue; + + float current_ratio = 0.; + FixedPointCoordinate nearest; + const float current_perpendicular_distance = + FixedPointCoordinate::ComputePerpendicularDistance( + coords->at(e.u), + coords->at(e.v), + input_coordinate, + nearest, + current_ratio); + + if ((current_perpendicular_distance < min_dist) && + !EpsilonCompare(current_perpendicular_distance, min_dist)) + { // found a new minimum + min_dist = current_perpendicular_distance; + result_phantom_node = { e.forward_edge_based_node_id, + e.reverse_edge_based_node_id, + e.name_id, + e.forward_weight, + e.reverse_weight, + e.forward_offset, + e.reverse_offset, + e.packed_geometry_id, + nearest, + e.fwd_segment_position}; + nearest_edge = e; + } + } + + if (result_phantom_node.location.isValid()) + { + // Hack to fix rounding errors and wandering via nodes. + if (1 == std::abs(input_coordinate.lon - result_phantom_node.location.lon)) + { + result_phantom_node.location.lon = input_coordinate.lon; + } + if (1 == std::abs(input_coordinate.lat - result_phantom_node.location.lat)) + { + result_phantom_node.location.lat = input_coordinate.lat; + } + + const float distance_1 = FixedPointCoordinate::ApproximateEuclideanDistance( + coords->at(nearest_edge.u), + result_phantom_node.location); + const float distance_2 = FixedPointCoordinate::ApproximateEuclideanDistance( + coords->at(nearest_edge.u), + coords->at(nearest_edge.v)); + const float ratio = std::min(1.f, distance_1 / distance_2); + + if (SPECIAL_NODEID != result_phantom_node.forward_node_id) + { + result_phantom_node.forward_weight *= ratio; + } + if (SPECIAL_NODEID != result_phantom_node.reverse_node_id) + { + result_phantom_node.reverse_weight *= (1. - ratio); + } + } + + return result_phantom_node.location.isValid(); + } + + template + inline bool EpsilonCompare(const FloatT d1, const FloatT d2) const + { + return (std::abs(d1 - d2) < std::numeric_limits::epsilon()); + } + +private: + const std::shared_ptr>& coords; + const std::vector& edges; +}; + + +template +struct RandomGraphFixture +{ + struct TupleHash + { + typedef std::pair argument_type; + typedef std::size_t result_type; + + result_type operator()(const argument_type & t) const + { + std::size_t val { 0 }; + boost::hash_combine(val, t.first); + boost::hash_combine(val, t.second); + return val; + } + }; + + RandomGraphFixture() + : coords(std::make_shared>()) + { + BOOST_TEST_MESSAGE("Constructing " << NUM_NODES << " nodes and " << NUM_EDGES << " edges."); + + std::mt19937 g(RANDOM_SEED); + + std::uniform_int_distribution<> lat_udist(MIN_LAT, MAX_LAT); + std::uniform_int_distribution<> lon_udist(MIN_LON, MAX_LON); + + for (unsigned i = 0; i < NUM_NODES; i++) + { + int lat = lat_udist(g); + int lon = lon_udist(g); + nodes.emplace_back(NodeInfo(lat, lon, i)); + coords->emplace_back(FixedPointCoordinate(lat, lon)); + } + + std::uniform_int_distribution<> edge_udist(0, nodes.size() - 1); + + std::unordered_set, TupleHash> used_edges; + + while(edges.size() < NUM_EDGES) + { + TestData data; + data.u = edge_udist(g); + data.v = edge_udist(g); + if (used_edges.find(std::pair(std::min(data.u, data.v), std::max(data.u, data.v))) == used_edges.end()) + { + data.is_in_tiny_cc = false; + edges.emplace_back(data); + used_edges.emplace(std::min(data.u, data.v), std::max(data.u, data.v)); + } + } + + } + + std::vector nodes; + std::shared_ptr> coords; + std::vector edges; +}; + + +typedef RandomGraphFixture TestRandomGraphFixture_LeafHalfFull; +typedef RandomGraphFixture TestRandomGraphFixture_LeafFull; +typedef RandomGraphFixture TestRandomGraphFixture_TwoLeaves; +typedef RandomGraphFixture TestRandomGraphFixture_Branch; +typedef RandomGraphFixture TestRandomGraphFixture_MultipleLevels; + +void simple_verify_rtree(TestStaticRTree& rtree, const std::shared_ptr>& coords, const std::vector& edges) +{ + BOOST_TEST_MESSAGE("Verify end points"); + for (const auto& e : edges) + { + FixedPointCoordinate result_u, result_v; + const FixedPointCoordinate& pu = coords->at(e.u); + const FixedPointCoordinate& pv = coords->at(e.v); + bool found_u = rtree.LocateClosestEndPointForCoordinate(pu, result_u, 1); + bool found_v = rtree.LocateClosestEndPointForCoordinate(pv, result_v, 1); + BOOST_CHECK(found_u && found_v); + float dist_u = FixedPointCoordinate::ApproximateEuclideanDistance( + result_u.lat, + result_u.lon, + pu.lat, + pu.lon); + BOOST_CHECK_LE(dist_u, std::numeric_limits::epsilon()); + float dist_v = FixedPointCoordinate::ApproximateEuclideanDistance( + result_v.lat, + result_v.lon, + pv.lat, + pv.lon); + BOOST_CHECK_LE(dist_v, std::numeric_limits::epsilon()); + } +} + +void sampling_verify_rtree(TestStaticRTree& rtree, LinearSearchNN& lsnn, unsigned num_samples) +{ + std::mt19937 g(RANDOM_SEED); + std::uniform_int_distribution<> lat_udist(WORLD_MIN_LAT, WORLD_MAX_LAT); + std::uniform_int_distribution<> lon_udist(WORLD_MIN_LON, WORLD_MAX_LON); + std::vector queries; + for (unsigned i = 0; i < num_samples; i++) + { + queries.emplace_back( + FixedPointCoordinate(lat_udist(g), lon_udist(g)) + ); + } + + BOOST_TEST_MESSAGE("Sampling queries"); + for (const auto& q : queries) + { + FixedPointCoordinate result_rtree; + rtree.LocateClosestEndPointForCoordinate(q, result_rtree, 1); + FixedPointCoordinate result_ln; + lsnn.LocateClosestEndPointForCoordinate(q, result_ln, 1); + BOOST_CHECK_EQUAL(result_ln, result_rtree); + + PhantomNode phantom_rtree; + rtree.FindPhantomNodeForCoordinate(q, phantom_rtree, 1); + PhantomNode phantom_ln; + lsnn.FindPhantomNodeForCoordinate(q, phantom_ln, 1); + BOOST_CHECK_EQUAL(phantom_rtree, phantom_ln); + } +} + +template +void construction_test(const std::string& prefix, FixtureT* fixture) +{ + const std::string nodes_path = prefix + ".ramIndex"; + const std::string leaves_path = prefix + ".fileIndex"; + const std::string coords_path = prefix + ".nodes"; + + boost::filesystem::ofstream node_stream(coords_path, std::ios::binary); + const unsigned num_nodes = fixture->nodes.size(); + node_stream.write((char *)&num_nodes, sizeof(unsigned)); + node_stream.write((char *)&(fixture->nodes[0]), num_nodes * sizeof(NodeInfo)); + node_stream.close(); + + TestStaticRTree::Build(fixture->edges, nodes_path, leaves_path, fixture->nodes); + TestStaticRTree rtree_querry(nodes_path, leaves_path, fixture->coords); + LinearSearchNN lsnn(fixture->coords, fixture->edges); + + simple_verify_rtree(rtree_querry, fixture->coords, fixture->edges); + sampling_verify_rtree(rtree_querry, lsnn, 100); +} + +BOOST_FIXTURE_TEST_CASE(construct_half_leaf_test, TestRandomGraphFixture_LeafHalfFull) +{ + construction_test("test_1", this); +} + +BOOST_FIXTURE_TEST_CASE(construct_full_leaf_test, TestRandomGraphFixture_LeafFull) +{ + construction_test("test_2", this); +} + +BOOST_FIXTURE_TEST_CASE(construct_two_leaves_test, TestRandomGraphFixture_TwoLeaves) +{ + construction_test("test_3", this); +} + +BOOST_FIXTURE_TEST_CASE(construct_branch_test, TestRandomGraphFixture_Branch) +{ + construction_test("test_4", this); +} + +BOOST_FIXTURE_TEST_CASE(construct_multiple_levels_test, TestRandomGraphFixture_MultipleLevels) +{ + construction_test("test_5", this); +} + + +BOOST_AUTO_TEST_SUITE_END() + From bc013925b8b7fc9b15b9e79dbf2568ea4adb4d3a Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 1 Jul 2014 01:30:51 +0200 Subject: [PATCH 12/24] Consider points on the edge of the rectangle as inside --- DataStructures/StaticRTree.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DataStructures/StaticRTree.h b/DataStructures/StaticRTree.h index a11daf034..f84e5d61e 100644 --- a/DataStructures/StaticRTree.h +++ b/DataStructures/StaticRTree.h @@ -200,8 +200,8 @@ class StaticRTree inline bool Contains(const FixedPointCoordinate &location) const { - const bool lats_contained = (location.lat > min_lat) && (location.lat < max_lat); - const bool lons_contained = (location.lon > min_lon) && (location.lon < max_lon); + const bool lats_contained = (location.lat >= min_lat) && (location.lat <= max_lat); + const bool lons_contained = (location.lon >= min_lon) && (location.lon <= max_lon); return lats_contained && lons_contained; } From 2a6585e664167a1d4e93a11fa9e0ee95af461986 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 1 Jul 2014 01:32:21 +0200 Subject: [PATCH 13/24] Add regression test for 'gap' querry bug --- UnitTests/DataStructures/StaticRTreeTest.cpp | 113 +++++++++++++++++-- 1 file changed, 103 insertions(+), 10 deletions(-) diff --git a/UnitTests/DataStructures/StaticRTreeTest.cpp b/UnitTests/DataStructures/StaticRTreeTest.cpp index deef96b5a..59d1a4728 100644 --- a/UnitTests/DataStructures/StaticRTreeTest.cpp +++ b/UnitTests/DataStructures/StaticRTreeTest.cpp @@ -231,6 +231,35 @@ struct RandomGraphFixture std::vector edges; }; +struct GraphFixture +{ + GraphFixture(const std::vector>& input_coords, + const std::vector>& input_edges) + : coords(std::make_shared>()) + { + + for (unsigned i = 0; i < input_coords.size(); i++) + { + FixedPointCoordinate c(input_coords[i].first * COORDINATE_PRECISION, input_coords[i].second * COORDINATE_PRECISION); + coords->emplace_back(c); + nodes.emplace_back(NodeInfo(c.lat, c.lon, i)); + } + + for (const auto& pair : input_edges) + { + TestData d; + d.u = pair.first; + d.v = pair.second; + edges.emplace_back(d); + } + + } + + std::vector nodes; + std::shared_ptr> coords; + std::vector edges; +}; + typedef RandomGraphFixture TestRandomGraphFixture_LeafHalfFull; typedef RandomGraphFixture TestRandomGraphFixture_LeafFull; @@ -240,7 +269,8 @@ typedef RandomGraphFixture TestRandomGraphFixture_MultipleLevels; -void simple_verify_rtree(TestStaticRTree& rtree, const std::shared_ptr>& coords, const std::vector& edges) +template +void simple_verify_rtree(RTreeT& rtree, const std::shared_ptr>& coords, const std::vector& edges) { BOOST_TEST_MESSAGE("Verify end points"); for (const auto& e : edges) @@ -266,7 +296,8 @@ void simple_verify_rtree(TestStaticRTree& rtree, const std::shared_ptr +void sampling_verify_rtree(RTreeT& rtree, LinearSearchNN& lsnn, unsigned num_samples) { std::mt19937 g(RANDOM_SEED); std::uniform_int_distribution<> lat_udist(WORLD_MIN_LAT, WORLD_MAX_LAT); @@ -296,11 +327,11 @@ void sampling_verify_rtree(TestStaticRTree& rtree, LinearSearchNN& lsnn, unsigne } } -template -void construction_test(const std::string& prefix, FixtureT* fixture) +template +void build_rtree(const std::string& prefix, FixtureT* fixture, std::string& leaves_path, std::string& nodes_path) { - const std::string nodes_path = prefix + ".ramIndex"; - const std::string leaves_path = prefix + ".fileIndex"; + nodes_path = prefix + ".ramIndex"; + leaves_path = prefix + ".fileIndex"; const std::string coords_path = prefix + ".nodes"; boost::filesystem::ofstream node_stream(coords_path, std::ios::binary); @@ -309,12 +340,20 @@ void construction_test(const std::string& prefix, FixtureT* fixture) node_stream.write((char *)&(fixture->nodes[0]), num_nodes * sizeof(NodeInfo)); node_stream.close(); - TestStaticRTree::Build(fixture->edges, nodes_path, leaves_path, fixture->nodes); - TestStaticRTree rtree_querry(nodes_path, leaves_path, fixture->coords); + RTreeT::Build(fixture->edges, nodes_path, leaves_path, fixture->nodes); +} + +template +void construction_test(const std::string& prefix, FixtureT* fixture) +{ + std::string leaves_path; + std::string nodes_path; + build_rtree(prefix, fixture, leaves_path, nodes_path); + RTreeT rtree(nodes_path, leaves_path, fixture->coords); LinearSearchNN lsnn(fixture->coords, fixture->edges); - simple_verify_rtree(rtree_querry, fixture->coords, fixture->edges); - sampling_verify_rtree(rtree_querry, lsnn, 100); + simple_verify_rtree(rtree, fixture->coords, fixture->edges); + sampling_verify_rtree(rtree, lsnn, 100); } BOOST_FIXTURE_TEST_CASE(construct_half_leaf_test, TestRandomGraphFixture_LeafHalfFull) @@ -342,6 +381,60 @@ BOOST_FIXTURE_TEST_CASE(construct_multiple_levels_test, TestRandomGraphFixture_M construction_test("test_5", this); } +/* + * Bug: If you querry a point that lies between two BBs that have a gap, + * one BB will be pruned, even if it could contain a nearer match. + */ +BOOST_AUTO_TEST_CASE(regression_test) +{ + typedef std::pair Coord; + typedef std::pair Edge; + GraphFixture fixture({ + Coord(40.0, 0.0), + Coord(35.0, 5.0), + + Coord(5.0, 5.0), + Coord(0.0, 10.0), + + Coord(20.0, 10.0), + Coord(20.0, 5.0), + + Coord(40.0, 100.0), + Coord(35.0, 105.0), + + Coord(5.0, 105.0), + Coord(0.0, 110.0), + }, + { + Edge(0, 1), + Edge(2, 3), + Edge(4, 5), + Edge(6, 7), + Edge(8, 9) + } + ); + + typedef StaticRTree, + false, + 2, + 3> MiniStaticRTree; + + std::string leaves_path; + std::string nodes_path; + build_rtree("test_regression", &fixture, leaves_path, nodes_path); + MiniStaticRTree rtree(nodes_path, leaves_path, fixture.coords); + + // query a node just right of the center of the gap + FixedPointCoordinate input(20.0 * COORDINATE_PRECISION, 55.1 * COORDINATE_PRECISION); + FixedPointCoordinate result; + rtree.LocateClosestEndPointForCoordinate(input, result, 1); + FixedPointCoordinate result_ln; + LinearSearchNN lsnn(fixture.coords, fixture.edges); + lsnn.LocateClosestEndPointForCoordinate(input, result_ln, 1); + + BOOST_CHECK_EQUAL(result_ln, result); +} BOOST_AUTO_TEST_SUITE_END() From b453a42f77c4108d462501acfdf6bd244b092a9c Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 4 Jul 2014 01:04:24 +0200 Subject: [PATCH 14/24] Fixed perpendicular distance calculation of segment endpoint is on equator --- DataStructures/Coordinate.cpp | 47 +++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/DataStructures/Coordinate.cpp b/DataStructures/Coordinate.cpp index 9c453e70d..37da2f8f3 100644 --- a/DataStructures/Coordinate.cpp +++ b/DataStructures/Coordinate.cpp @@ -159,10 +159,10 @@ FixedPointCoordinate::ComputePerpendicularDistance(const FixedPointCoordinate &s // initialize values const float x_value = lat2y(point.lat / COORDINATE_PRECISION); const float y_value = point.lon / COORDINATE_PRECISION; - const float a = lat2y(source_coordinate.lat / COORDINATE_PRECISION); - const float b = source_coordinate.lon / COORDINATE_PRECISION; - const float c = lat2y(target_coordinate.lat / COORDINATE_PRECISION); - const float d = target_coordinate.lon / COORDINATE_PRECISION; + float a = lat2y(source_coordinate.lat / COORDINATE_PRECISION); + float b = source_coordinate.lon / COORDINATE_PRECISION; + float c = lat2y(target_coordinate.lat / COORDINATE_PRECISION); + float d = target_coordinate.lon / COORDINATE_PRECISION; float p, q; if (std::abs(a - c) > std::numeric_limits::epsilon()) { @@ -178,15 +178,35 @@ FixedPointCoordinate::ComputePerpendicularDistance(const FixedPointCoordinate &s q = y_value; } - float nY = (d * p - c * q) / (a * d - b * c); - // discretize the result to coordinate precision. it's a hack! - if (std::abs(nY) < (1.f / COORDINATE_PRECISION)) + float ratio; + bool inverse_ratio = false; + + // straight line segment on equator + if (std::abs(c) < std::numeric_limits::epsilon() && std::abs(a) < std::numeric_limits::epsilon()) { - nY = 0.f; + ratio = (q - b) / (d - b); + } + else + { + if (std::abs(c) < std::numeric_limits::epsilon()) + { + // swap start/end + std::swap(a, c); + std::swap(b, d); + inverse_ratio = true; + } + + float nY = (d * p - c * q) / (a * d - b * c); + // discretize the result to coordinate precision. it's a hack! + if (std::abs(nY) < (1.f / COORDINATE_PRECISION)) + { + nY = 0.f; + } + + // compute ratio + ratio = (p - nY * a) / c; } - // compute ratio - float ratio = (p - nY * a) / c; if (std::isnan(ratio)) { ratio = (target_coordinate == point ? 1.f : 0.f); @@ -200,6 +220,12 @@ FixedPointCoordinate::ComputePerpendicularDistance(const FixedPointCoordinate &s ratio = 1.f; } + // we need to do this, if we switched start/end coordinates + if (inverse_ratio) + { + ratio = 1.0f - ratio; + } + //compute the nearest location FixedPointCoordinate nearest_location; BOOST_ASSERT(!std::isnan(ratio)); @@ -216,6 +242,7 @@ FixedPointCoordinate::ComputePerpendicularDistance(const FixedPointCoordinate &s nearest_location.lat = static_cast(y2lat(p) * COORDINATE_PRECISION); nearest_location.lon = static_cast(q * COORDINATE_PRECISION); } + BOOST_ASSERT(nearest_location.isValid()); return FixedPointCoordinate::ApproximateEuclideanDistance(point, nearest_location); } From db98f6191ea0f60a0d35c904b0322fb708c48d03 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 4 Jul 2014 01:42:32 +0200 Subject: [PATCH 15/24] Prepare alrady writes the correct number of nodes --- Util/GraphLoader.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Util/GraphLoader.h b/Util/GraphLoader.h index 97c5c93e1..c943b789c 100644 --- a/Util/GraphLoader.h +++ b/Util/GraphLoader.h @@ -417,7 +417,7 @@ unsigned readHSGRFromStream(const boost::filesystem::path &hsgr_file, << ", number_of_edges: " << number_of_edges; // BOOST_ASSERT_MSG( 0 != number_of_edges, "number of edges is zero"); - node_list.resize(number_of_nodes + 1); + node_list.resize(number_of_nodes); hsgr_input_stream.read((char *)&(node_list[0]), number_of_nodes * sizeof(NodeT)); edge_list.resize(number_of_edges); From d754e4eecaff4ae3f264e75ef72731bbb10b9c8f Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 4 Jul 2014 11:44:41 +0200 Subject: [PATCH 16/24] Add benchmark for StaticRTree Build with 'make benchmarks' --- Benchmarks/StaticRTreeBench.cpp | 118 ++++++++++++++++++++++++++++++++ CMakeLists.txt | 8 ++- 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 Benchmarks/StaticRTreeBench.cpp diff --git a/Benchmarks/StaticRTreeBench.cpp b/Benchmarks/StaticRTreeBench.cpp new file mode 100644 index 000000000..a7c9a5c2e --- /dev/null +++ b/Benchmarks/StaticRTreeBench.cpp @@ -0,0 +1,118 @@ +#include "../DataStructures/OriginalEdgeData.h" +#include "../DataStructures/QueryNode.h" +#include "../DataStructures/SharedMemoryVectorWrapper.h" +#include "../DataStructures/StaticRTree.h" +#include "../Util/BoostFileSystemFix.h" +#include "../DataStructures/EdgeBasedNode.h" + +#include + +#include + +// Choosen by a fair W20 dice roll (this value is completely arbitrary) +constexpr unsigned RANDOM_SEED = 13; +constexpr int32_t WORLD_MIN_LAT = -90*COORDINATE_PRECISION; +constexpr int32_t WORLD_MAX_LAT = 90*COORDINATE_PRECISION; +constexpr int32_t WORLD_MIN_LON = -180*COORDINATE_PRECISION; +constexpr int32_t WORLD_MAX_LON = 180*COORDINATE_PRECISION; + +typedef EdgeBasedNode RTreeLeaf; +typedef std::shared_ptr> FixedPointCoordinateListPtr; +typedef StaticRTree::vector, false> BenchStaticRTree; + +FixedPointCoordinateListPtr LoadCoordinates(const boost::filesystem::path& nodes_file) +{ + boost::filesystem::ifstream nodes_input_stream(nodes_file, std::ios::binary); + + NodeInfo current_node; + unsigned number_of_coordinates = 0; + nodes_input_stream.read((char *)&number_of_coordinates, sizeof(unsigned)); + auto coords = std::make_shared>(number_of_coordinates); + for (unsigned i = 0; i < number_of_coordinates; ++i) + { + nodes_input_stream.read((char *)¤t_node, sizeof(NodeInfo)); + coords->at(i) = FixedPointCoordinate(current_node.lat, current_node.lon); + BOOST_ASSERT((std::abs(coords->at(i).lat) >> 30) == 0); + BOOST_ASSERT((std::abs(coords->at(i).lon) >> 30) == 0); + } + nodes_input_stream.close(); + return coords; +} + +void Benchmark(BenchStaticRTree& rtree, unsigned num_queries) +{ + std::mt19937 g(RANDOM_SEED); + std::uniform_int_distribution<> lat_udist(WORLD_MIN_LAT, WORLD_MAX_LAT); + std::uniform_int_distribution<> lon_udist(WORLD_MIN_LON, WORLD_MAX_LON); + std::vector queries; + for (unsigned i = 0; i < num_queries; i++) + { + queries.emplace_back( + FixedPointCoordinate(lat_udist(g), lon_udist(g)) + ); + } + + const unsigned num_results = 5; + std::cout << "#### IncrementalFindPhantomNodeForCoordinate : " << num_results << " phantom nodes" << std::endl; + + TIMER_START(query_phantom); + std::vector resulting_phantom_node_vector; + for (const auto& q : queries) + { + resulting_phantom_node_vector.clear(); + rtree.IncrementalFindPhantomNodeForCoordinate(q, resulting_phantom_node_vector, 3, num_results); + resulting_phantom_node_vector.clear(); + rtree.IncrementalFindPhantomNodeForCoordinate(q, resulting_phantom_node_vector, 17, num_results); + } + TIMER_STOP(query_phantom); + + std::cout << "Took " << TIMER_MSEC(query_phantom) << " msec for " << num_queries << " queries." << std::endl; + std::cout << TIMER_MSEC(query_phantom)/((double) num_queries) << " msec/query." << std::endl; + + std::cout << "#### LocateClosestEndPointForCoordinate" << std::endl; + + TIMER_START(query_endpoint); + FixedPointCoordinate result; + for (const auto& q : queries) + { + rtree.LocateClosestEndPointForCoordinate(q, result, 3); + } + TIMER_STOP(query_endpoint); + + std::cout << "Took " << TIMER_MSEC(query_endpoint) << " msec for " << num_queries << " queries." << std::endl; + std::cout << TIMER_MSEC(query_endpoint)/((double) num_queries) << " msec/query." << std::endl; + + std::cout << "#### FindPhantomNodeForCoordinate" << std::endl; + + TIMER_START(query_phantomnode); + for (const auto& q : queries) + { + PhantomNode phantom; + rtree.FindPhantomNodeForCoordinate(q, phantom, 3); + } + TIMER_STOP(query_phantomnode); + + std::cout << "Took " << TIMER_MSEC(query_phantomnode) << " msec for " << num_queries << " queries." << std::endl; + std::cout << TIMER_MSEC(query_phantomnode)/((double) num_queries) << " msec/query." << std::endl; +} + +int main(int argc, char** argv) +{ + if (argc < 4) + { + std::cout << "./rtree-bench file.ramIndex file.fileIndx file.nodes" << std::endl; + return 1; + } + + const char* ramPath = argv[1]; + const char* filePath = argv[2]; + const char* nodesPath = argv[3]; + + auto coords = LoadCoordinates(nodesPath); + + BenchStaticRTree rtree(ramPath, filePath, coords); + + Benchmark(rtree, 10000); + + return 0; +} diff --git a/CMakeLists.txt b/CMakeLists.txt index b743d818f..db254b702 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,7 @@ add_custom_command(OUTPUT ${CMAKE_SOURCE_DIR}/Util/FingerPrint.cpp FingerPrint.c add_custom_target(FingerPrintConfigure DEPENDS ${CMAKE_SOURCE_DIR}/Util/FingerPrint.cpp) add_custom_target(tests DEPENDS datastructure-tests) +add_custom_target(benchmarks DEPENDS rtree-bench) set(BOOST_COMPONENTS date_time filesystem iostreams program_options regex system thread unit_test_framework) @@ -89,6 +90,9 @@ add_executable(osrm-datastore datastore.cpp) # Unit tests add_executable(datastructure-tests EXCLUDE_FROM_ALL UnitTests/datastructure_tests.cpp ${DataStructureTestsGlob}) +# Benchmarks +add_executable(rtree-bench EXCLUDE_FROM_ALL Benchmarks/StaticRTreeBench.cpp) + # Check the release mode if(NOT CMAKE_BUILD_TYPE MATCHES Debug) set(CMAKE_BUILD_TYPE Release) @@ -197,14 +201,15 @@ target_link_libraries(osrm-prepare ${Boost_LIBRARIES} FINGERPRINT GITDESCRIPTION target_link_libraries(osrm-routed ${Boost_LIBRARIES} ${OPTIONAL_SOCKET_LIBS} OSRM FINGERPRINT GITDESCRIPTION) target_link_libraries(osrm-datastore ${Boost_LIBRARIES} FINGERPRINT GITDESCRIPTION COORDLIB) target_link_libraries(datastructure-tests ${Boost_LIBRARIES} COORDLIB) +target_link_libraries(rtree-bench ${Boost_LIBRARIES} COORDLIB) find_package(Threads REQUIRED) target_link_libraries(osrm-extract ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(osrm-datastore ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(osrm-prepare ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(OSRM ${CMAKE_THREAD_LIBS_INIT}) - target_link_libraries(datastructure-tests ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(rtree-bench ${CMAKE_THREAD_LIBS_INIT}) find_package(TBB REQUIRED) if(WIN32 AND CMAKE_BUILD_TYPE MATCHES Debug) @@ -215,6 +220,7 @@ target_link_libraries(osrm-extract ${TBB_LIBRARIES}) target_link_libraries(osrm-prepare ${TBB_LIBRARIES}) target_link_libraries(osrm-routed ${TBB_LIBRARIES}) target_link_libraries(datastructure-tests ${TBB_LIBRARIES}) +target_link_libraries(rtree-bench ${TBB_LIBRARIES}) include_directories(${TBB_INCLUDE_DIR}) find_package(Lua52) From 782baf54a3affb15c8c076a285f07d2a991968ee Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 5 Jul 2014 00:57:47 +0200 Subject: [PATCH 17/24] Remove perpendicular distance call Since we know that the MBB is axis aligned we can compute the distance to the line segments a lot simpler. --- DataStructures/StaticRTree.h | 71 ++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/DataStructures/StaticRTree.h b/DataStructures/StaticRTree.h index f84e5d61e..20406fd53 100644 --- a/DataStructures/StaticRTree.h +++ b/DataStructures/StaticRTree.h @@ -143,24 +143,63 @@ class StaticRTree return 0.; } - const FixedPointCoordinate upper_left(max_lat, min_lon); - const FixedPointCoordinate upper_right(max_lat, max_lon); - const FixedPointCoordinate lower_right(min_lat, max_lon); - const FixedPointCoordinate lower_left(min_lat, min_lon); + enum Direction + { + INVALID = 0, + NORTH = 1, + SOUTH = 2, + EAST = 4, + NORTH_EAST = 5, + SOUTH_EAST = 6, + WEST = 8, + NORTH_WEST = 9, + SOUTH_WEST = 10 + }; + + Direction d = INVALID; + if (location.lat > max_lat) + d = (Direction) (d | NORTH); + else if (location.lat < min_lat) + d = (Direction) (d | SOUTH); + if (location.lon > max_lon) + d = (Direction) (d | EAST); + else if (location.lon < min_lon) + d = (Direction) (d | WEST); + + BOOST_ASSERT(d != INVALID); float min_dist = std::numeric_limits::max(); - min_dist = std::min(min_dist, - FixedPointCoordinate::ComputePerpendicularDistance( - upper_left, upper_right, location)); - min_dist = std::min(min_dist, - FixedPointCoordinate::ComputePerpendicularDistance( - upper_right, lower_right, location)); - min_dist = std::min(min_dist, - FixedPointCoordinate::ComputePerpendicularDistance( - lower_right, lower_left, location)); - min_dist = std::min(min_dist, - FixedPointCoordinate::ComputePerpendicularDistance( - lower_left, upper_left, location)); + switch (d) + { + case NORTH: + min_dist = FixedPointCoordinate::ApproximateEuclideanDistance(location, FixedPointCoordinate(max_lat, location.lon)); + break; + case SOUTH: + min_dist = FixedPointCoordinate::ApproximateEuclideanDistance(location, FixedPointCoordinate(min_lat, location.lon)); + break; + case WEST: + min_dist = FixedPointCoordinate::ApproximateEuclideanDistance(location, FixedPointCoordinate(location.lat, min_lon)); + break; + case EAST: + min_dist = FixedPointCoordinate::ApproximateEuclideanDistance(location, FixedPointCoordinate(location.lat, max_lon)); + break; + case NORTH_EAST: + min_dist = FixedPointCoordinate::ApproximateEuclideanDistance(location, FixedPointCoordinate(max_lat, max_lon)); + break; + case NORTH_WEST: + min_dist = FixedPointCoordinate::ApproximateEuclideanDistance(location, FixedPointCoordinate(max_lat, min_lon)); + break; + case SOUTH_EAST: + min_dist = FixedPointCoordinate::ApproximateEuclideanDistance(location, FixedPointCoordinate(min_lat, max_lon)); + break; + case SOUTH_WEST: + min_dist = FixedPointCoordinate::ApproximateEuclideanDistance(location, FixedPointCoordinate(min_lat, min_lon)); + break; + default: + break; + } + + BOOST_ASSERT(min_dist != std::numeric_limits::max()); return min_dist; } From 84a604f70b3aef3c7ca803268d35119633e64052 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 5 Jul 2014 00:59:31 +0200 Subject: [PATCH 18/24] Add test for new GetMinDist --- UnitTests/DataStructures/StaticRTreeTest.cpp | 67 ++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/UnitTests/DataStructures/StaticRTreeTest.cpp b/UnitTests/DataStructures/StaticRTreeTest.cpp index 59d1a4728..06658f107 100644 --- a/UnitTests/DataStructures/StaticRTreeTest.cpp +++ b/UnitTests/DataStructures/StaticRTreeTest.cpp @@ -436,5 +436,72 @@ BOOST_AUTO_TEST_CASE(regression_test) BOOST_CHECK_EQUAL(result_ln, result); } +void TestRectangle(double width, double height, double center_lat, double center_lon) +{ + FixedPointCoordinate center(center_lat*COORDINATE_PRECISION, center_lon*COORDINATE_PRECISION); + + TestStaticRTree::RectangleT rect; + rect.min_lat = center.lat - height/2.0 * COORDINATE_PRECISION; + rect.max_lat = center.lat + height/2.0 * COORDINATE_PRECISION; + rect.min_lon = center.lon - width/2.0 * COORDINATE_PRECISION; + rect.max_lon = center.lon + width/2.0 * COORDINATE_PRECISION; + + unsigned offset = 5*COORDINATE_PRECISION; + FixedPointCoordinate north(rect.max_lat + offset, center.lon); + FixedPointCoordinate south(rect.min_lat - offset, center.lon); + FixedPointCoordinate west(center.lat, rect.min_lon - offset); + FixedPointCoordinate east(center.lat, rect.max_lon + offset); + FixedPointCoordinate north_east(rect.max_lat + offset, rect.max_lon + offset); + FixedPointCoordinate north_west(rect.max_lat + offset, rect.min_lon - offset); + FixedPointCoordinate south_east(rect.min_lat - offset, rect.max_lon + offset); + FixedPointCoordinate south_west(rect.min_lat - offset, rect.min_lon - offset); + + + /* Distance to line segments of rectangle */ + BOOST_CHECK_EQUAL( + rect.GetMinDist(north), + FixedPointCoordinate::ApproximateEuclideanDistance(north, FixedPointCoordinate(rect.max_lat, north.lon)) + ); + BOOST_CHECK_EQUAL( + rect.GetMinDist(south), + FixedPointCoordinate::ApproximateEuclideanDistance(south, FixedPointCoordinate(rect.min_lat, south.lon)) + ); + BOOST_CHECK_EQUAL( + rect.GetMinDist(west), + FixedPointCoordinate::ApproximateEuclideanDistance(west, FixedPointCoordinate(west.lat, rect.min_lon)) + ); + BOOST_CHECK_EQUAL( + rect.GetMinDist(east), + FixedPointCoordinate::ApproximateEuclideanDistance(east, FixedPointCoordinate(east.lat, rect.max_lon)) + ); + + /* Distance to corner points */ + BOOST_CHECK_EQUAL( + rect.GetMinDist(north_east), + FixedPointCoordinate::ApproximateEuclideanDistance(north_east, FixedPointCoordinate(rect.max_lat, rect.max_lon)) + ); + BOOST_CHECK_EQUAL( + rect.GetMinDist(north_west), + FixedPointCoordinate::ApproximateEuclideanDistance(north_west, FixedPointCoordinate(rect.max_lat, rect.min_lon)) + ); + BOOST_CHECK_EQUAL( + rect.GetMinDist(south_east), + FixedPointCoordinate::ApproximateEuclideanDistance(south_east, FixedPointCoordinate(rect.min_lat, rect.max_lon)) + ); + BOOST_CHECK_EQUAL( + rect.GetMinDist(south_west), + FixedPointCoordinate::ApproximateEuclideanDistance(south_west, FixedPointCoordinate(rect.min_lat, rect.min_lon)) + ); +} + +BOOST_AUTO_TEST_CASE(rectangle_test) +{ + TestRectangle(10, 10, 5, 5); + TestRectangle(10, 10, -5, 5); + TestRectangle(10, 10, 5, -5); + TestRectangle(10, 10, -5, -5); + TestRectangle(10, 10, 0, 0); +} + BOOST_AUTO_TEST_SUITE_END() From d38e3bd72948db15521dc03c135b3f9d75c990fe Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 8 Jul 2014 23:09:03 +0200 Subject: [PATCH 19/24] Add tests and benchmarks to appveyor --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 7d6f8350e..a73873d09 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -38,6 +38,8 @@ build_script: - set TBB_ARCH_PLATFORM=intel64/vc12 - cmake .. -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=%Configuration% -DBZIP2_INCLUDE_DIR=%P%/libs/include -DBZIP2_LIBRARIES=%P%/libs/lib/libbz2.lib -DCMAKE_INSTALL_PREFIX=%P%/libs -DBOOST_ROOT=%P%/boost_min -DBoost_USE_STATIC_LIBS=ON - nmake + - nmake tests + - nmake benchmarks - if "%APPVEYOR_REPO_BRANCH%"=="develop" (7z a %P%/osrm_%Configuration%.zip *.exe *.pdb %P%/libs/bin/*.dll -tzip) test: off From 020d0cfb499024c74361eef783dcaa414e240ee0 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 21 Jul 2014 00:24:40 +0200 Subject: [PATCH 20/24] Revert "Change StaticRTree serialization constructor to static function" This makes clang 3.4 crash on ubuntu because it can not handle lambda expressions + binding in static member functions correctly. This reverts commit d6dd6693b18e042c0068da579dcc64d1e5a2e002. --- Contractor/Prepare.cpp | 18 +++++++-- Contractor/Prepare.h | 1 + DataStructures/StaticRTree.h | 40 ++++++++++---------- UnitTests/DataStructures/StaticRTreeTest.cpp | 2 +- 4 files changed, 35 insertions(+), 26 deletions(-) diff --git a/Contractor/Prepare.cpp b/Contractor/Prepare.cpp index e12a61b42..eee1124b1 100644 --- a/Contractor/Prepare.cpp +++ b/Contractor/Prepare.cpp @@ -170,10 +170,7 @@ int Prepare::Process(int argc, char *argv[]) TIMER_STOP(expansion); - StaticRTree::Build(node_based_edge_list, - rtree_nodes_path.c_str(), - rtree_leafs_path.c_str(), - internal_to_external_node_map); + BuildRTree(node_based_edge_list); IteratorbasedCRC32> crc32; const unsigned node_based_edge_list_CRC32 = @@ -547,3 +544,16 @@ void Prepare::WriteNodeMapping() internal_to_external_node_map.shrink_to_fit(); } +/** + \brief Building rtree-based nearest-neighbor data structure + + Saves info to files: '.ramIndex' and '.fileIndex'. + */ +void Prepare::BuildRTree(std::vector &node_based_edge_list) +{ + SimpleLogger().Write() << "building r-tree ..."; + StaticRTree(node_based_edge_list, + rtree_nodes_path.c_str(), + rtree_leafs_path.c_str(), + internal_to_external_node_map); +} diff --git a/Contractor/Prepare.h b/Contractor/Prepare.h index 6386a5227..0a5c35985 100644 --- a/Contractor/Prepare.h +++ b/Contractor/Prepare.h @@ -40,6 +40,7 @@ class Prepare DeallocatingVector &edgeBasedEdgeList, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile); void WriteNodeMapping(); + void BuildRTree(std::vector &node_based_edge_list); private: std::vector internal_to_external_node_map; diff --git a/DataStructures/StaticRTree.h b/DataStructures/StaticRTree.h index 20406fd53..ab711f51b 100644 --- a/DataStructures/StaticRTree.h +++ b/DataStructures/StaticRTree.h @@ -352,24 +352,24 @@ class StaticRTree StaticRTree(const StaticRTree &) = delete; // Construct a packed Hilbert-R-Tree with Kamel-Faloutsos algorithm [1] - static void Build(std::vector &input_data_vector, + explicit StaticRTree(std::vector &input_data_vector, const std::string tree_node_filename, const std::string leaf_node_filename, const std::vector &coordinate_list) + : m_element_count(input_data_vector.size()), m_leaf_node_filename(leaf_node_filename) { - uint64_t element_count = input_data_vector.size(); - SimpleLogger().Write() << "constructing r-tree of " << element_count + SimpleLogger().Write() << "constructing r-tree of " << m_element_count << " edge elements build on-top of " << coordinate_list.size() << " coordinates"; TIMER_START(construction); - std::vector input_wrapper_vector(element_count); + std::vector input_wrapper_vector(m_element_count); HilbertCode get_hilbert_number; // generate auxiliary vector of hilbert-values tbb::parallel_for( - tbb::blocked_range(0, element_count), + tbb::blocked_range(0, m_element_count), [&input_data_vector, &input_wrapper_vector, &get_hilbert_number, &coordinate_list]( const tbb::blocked_range &range) { @@ -396,7 +396,7 @@ class StaticRTree // open leaf file boost::filesystem::ofstream leaf_node_file(leaf_node_filename, std::ios::binary); - leaf_node_file.write((char *) &element_count, sizeof(uint64_t)); + leaf_node_file.write((char *)&m_element_count, sizeof(uint64_t)); // sort the hilbert-value representatives tbb::parallel_sort(input_wrapper_vector.begin(), input_wrapper_vector.end()); @@ -404,7 +404,7 @@ class StaticRTree // pack M elements into leaf node and write to leaf file uint64_t processed_objects_count = 0; - while (processed_objects_count < element_count) + while (processed_objects_count < m_element_count) { LeafNode current_leaf; @@ -414,7 +414,7 @@ class StaticRTree for (uint32_t current_element_index = 0; LEAF_NODE_SIZE > current_element_index; ++current_element_index) { - if (element_count > (processed_objects_count + current_element_index)) + if (m_element_count > (processed_objects_count + current_element_index)) { uint32_t index_of_next_object = input_wrapper_vector[processed_objects_count + current_element_index] @@ -440,8 +440,6 @@ class StaticRTree // close leaf file leaf_node_file.close(); - typename ShM::vector search_tree; - uint32_t processing_level = 0; while (1 < tree_nodes_in_level.size()) { @@ -460,8 +458,8 @@ class StaticRTree TreeNode ¤t_child_node = tree_nodes_in_level[processed_tree_nodes_in_level]; // add tree node to parent entry - parent_node.children[current_child_node_index] = search_tree.size(); - search_tree.emplace_back(current_child_node); + parent_node.children[current_child_node_index] = m_search_tree.size(); + m_search_tree.emplace_back(current_child_node); // merge MBRs parent_node.minimum_bounding_rectangle.MergeBoundingBoxes( current_child_node.minimum_bounding_rectangle); @@ -477,18 +475,18 @@ class StaticRTree } BOOST_ASSERT_MSG(1 == tree_nodes_in_level.size(), "tree broken, more than one root node"); // last remaining entry is the root node, store it - search_tree.emplace_back(tree_nodes_in_level[0]); + m_search_tree.emplace_back(tree_nodes_in_level[0]); // reverse and renumber tree to have root at index 0 - std::reverse(search_tree.begin(), search_tree.end()); + std::reverse(m_search_tree.begin(), m_search_tree.end()); - uint32_t search_tree_size = search_tree.size(); + uint32_t search_tree_size = m_search_tree.size(); tbb::parallel_for(tbb::blocked_range(0, search_tree_size), - [&search_tree, &search_tree_size](const tbb::blocked_range &range) - { + [this, &search_tree_size](const tbb::blocked_range &range) + { for (uint32_t i = range.begin(); i != range.end(); ++i) { - TreeNode ¤t_tree_node = search_tree[i]; + TreeNode ¤t_tree_node = this->m_search_tree[i]; for (uint32_t j = 0; j < current_tree_node.child_count; ++j) { const uint32_t old_id = current_tree_node.children[j]; @@ -501,10 +499,10 @@ class StaticRTree // open tree file boost::filesystem::ofstream tree_node_file(tree_node_filename, std::ios::binary); - uint32_t size_of_tree = search_tree.size(); + uint32_t size_of_tree = m_search_tree.size(); BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty"); - tree_node_file.write((char *) &size_of_tree, sizeof(uint32_t)); - tree_node_file.write((char *) &search_tree[0], sizeof(TreeNode) * size_of_tree); + tree_node_file.write((char *)&size_of_tree, sizeof(uint32_t)); + tree_node_file.write((char *)&m_search_tree[0], sizeof(TreeNode) * size_of_tree); // close tree node file. tree_node_file.close(); diff --git a/UnitTests/DataStructures/StaticRTreeTest.cpp b/UnitTests/DataStructures/StaticRTreeTest.cpp index 06658f107..d5cd47fb2 100644 --- a/UnitTests/DataStructures/StaticRTreeTest.cpp +++ b/UnitTests/DataStructures/StaticRTreeTest.cpp @@ -340,7 +340,7 @@ void build_rtree(const std::string& prefix, FixtureT* fixture, std::string& leav node_stream.write((char *)&(fixture->nodes[0]), num_nodes * sizeof(NodeInfo)); node_stream.close(); - RTreeT::Build(fixture->edges, nodes_path, leaves_path, fixture->nodes); + RTreeT r(fixture->edges, nodes_path, leaves_path, fixture->nodes); } template From bcff6c192cc76b258e2b93e00b393679294c2b2a Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 21 Jul 2014 00:37:54 +0200 Subject: [PATCH 21/24] Remove template paramters and constexpr in test Stops clang from complaining about non-constexpr. --- UnitTests/DataStructures/StaticRTreeTest.cpp | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/UnitTests/DataStructures/StaticRTreeTest.cpp b/UnitTests/DataStructures/StaticRTreeTest.cpp index d5cd47fb2..55aeb9e34 100644 --- a/UnitTests/DataStructures/StaticRTreeTest.cpp +++ b/UnitTests/DataStructures/StaticRTreeTest.cpp @@ -26,10 +26,10 @@ typedef StaticRTree +template struct RandomGraphFixture { struct TupleHash @@ -196,8 +192,8 @@ struct RandomGraphFixture std::mt19937 g(RANDOM_SEED); - std::uniform_int_distribution<> lat_udist(MIN_LAT, MAX_LAT); - std::uniform_int_distribution<> lon_udist(MIN_LON, MAX_LON); + std::uniform_int_distribution<> lat_udist(WORLD_MIN_LAT, WORLD_MAX_LAT); + std::uniform_int_distribution<> lon_udist(WORLD_MIN_LON, WORLD_MAX_LON); for (unsigned i = 0; i < NUM_NODES; i++) { From 61151535e66546d2845f2229c4589b903cb717b0 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 21 Jul 2014 10:16:50 +0200 Subject: [PATCH 22/24] Fix include in RangeTableTest --- UnitTests/DataStructures/RangeTableTest.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/UnitTests/DataStructures/RangeTableTest.cpp b/UnitTests/DataStructures/RangeTableTest.cpp index 60c299a01..252e070ad 100644 --- a/UnitTests/DataStructures/RangeTableTest.cpp +++ b/UnitTests/DataStructures/RangeTableTest.cpp @@ -1,8 +1,9 @@ +#include "../../DataStructures/RangeTable.h" #include #include -#include "../../DataStructures/RangeTable.h" +#include constexpr unsigned BLOCK_SIZE = 16; typedef RangeTable TestRangeTable; From 4722988bf0e41537f2e3392a1b21ba3784ee9eb5 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 22 Jul 2014 13:44:02 +0200 Subject: [PATCH 23/24] Try including typedefs --- UnitTests/DataStructures/RangeTableTest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/UnitTests/DataStructures/RangeTableTest.cpp b/UnitTests/DataStructures/RangeTableTest.cpp index 252e070ad..311c77ec3 100644 --- a/UnitTests/DataStructures/RangeTableTest.cpp +++ b/UnitTests/DataStructures/RangeTableTest.cpp @@ -1,4 +1,5 @@ #include "../../DataStructures/RangeTable.h" +#include "../../typedefs.h" #include #include From 4d0571fd73bb439ad13dbee019cb5ad77f9ec358 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 22 Jul 2014 17:17:22 +0200 Subject: [PATCH 24/24] Fix win32 linking and run test automatically in AppVayor --- CMakeLists.txt | 2 ++ appveyor.yml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index db254b702..154edaeac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,7 +123,9 @@ if(CMAKE_BUILD_TYPE MATCHES Release) endif (HAS_LTO_FLAG) endif() +if (NOT WIN32) add_definitions(-DBOOST_TEST_DYN_LINK) +endif() # Configuring compilers if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") diff --git a/appveyor.yml b/appveyor.yml index a73873d09..6635c5117 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -41,6 +41,9 @@ build_script: - nmake tests - nmake benchmarks - if "%APPVEYOR_REPO_BRANCH%"=="develop" (7z a %P%/osrm_%Configuration%.zip *.exe *.pdb %P%/libs/bin/*.dll -tzip) + - set PATH=%PATH%;c:/projects/osrm/libs/bin + - cd c:/projects/osrm/build + - datastructure-tests.exe test: off