Merge pull request #5760 from woltapp/nostxxl

Removed STXXL support.

.gitignore

@@ -72,12 +72,6 @@ Thumbs.db
 ###########################
 .idea/
 
-# stxxl related files #
-#######################
-.stxxl
-stxxl.log
-stxxl.errlog
-
 # Compiled Binary Files #
 ####################################
 /osrm-extract

.travis.yml

@@ -151,14 +151,6 @@ matrix:
         TARGET_ARCH='i686' CCOMPILER='gcc-7' CXXCOMPILER='g++-7' BUILD_TYPE='Release'
         CFLAGS='-m32 -msse2 -mfpmath=sse' CXXFLAGS='-m32 -msse2 -mfpmath=sse'
 
-    - os: linux
-      compiler: "gcc-7-stxxl"
-      addons: &gcc7
-        apt:
-          sources: ['ubuntu-toolchain-r-test']
-          packages: ['g++-7', 'libbz2-dev', 'libstxxl-dev', 'libxml2-dev', 'libzip-dev', 'liblua5.2-dev', 'libtbb-dev', 'libboost-all-dev']
-      env: CCOMPILER='gcc-7' CXXCOMPILER='g++-7' BUILD_TYPE='Release' ENABLE_STXXL=On
-
     - os: linux
       compiler: "gcc-5-release"
       addons: &gcc49
@@ -483,7 +475,6 @@ install:
              -DENABLE_COVERAGE=${ENABLE_COVERAGE:-OFF} \
              -DENABLE_NODE_BINDINGS=${ENABLE_NODE_BINDINGS:-OFF} \
              -DENABLE_SANITIZER=${ENABLE_SANITIZER:-OFF} \
-             -DENABLE_STXXL=${ENABLE_STXXL:-OFF} \
              -DBUILD_TOOLS=ON \
              -DENABLE_CCACHE=ON \
              -DCMAKE_INSTALL_PREFIX=${OSRM_INSTALL_DIR} \

CHANGELOG.md

@@ -20,7 +20,8 @@
     - Misc:
       - CHANGED: Reduce memory usage for raster source handling. [#5572](https://github.com/Project-OSRM/osrm-backend/pull/5572)
       - CHANGED: Add cmake option `ENABLE_DEBUG_LOGGING` to control whether output debug logging. [#3427](https://github.com/Project-OSRM/osrm-backend/issues/3427)
-
+    - Infrastructure
+      - REMOVED: STXXL support removed as STXXL became abandonware. [#5760](https://github.com/Project-OSRM/osrm-backend/pull/5760)
 # 5.21.0
   - Changes from 5.20.0
     - Features:

CMakeLists.txt

@@ -26,7 +26,6 @@ option(ENABLE_ASSERTIONS "Use assertions in release mode" OFF)
 option(ENABLE_DEBUG_LOGGING "Use debug logging in release mode" OFF)
 option(ENABLE_COVERAGE "Build with coverage instrumentalisation" OFF)
 option(ENABLE_SANITIZER "Use memory sanitizer for Debug build" OFF)
-option(ENABLE_STXXL "Use STXXL library" OFF)
 option(ENABLE_LTO "Use LTO if available" OFF)
 option(ENABLE_FUZZING "Fuzz testing using LLVM's libFuzzer" OFF)
 option(ENABLE_GOLD_LINKER "Use GNU gold linker if available" ON)
@@ -38,7 +37,6 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 if(ENABLE_MASON)
   # versions in use
   set(MASON_BOOST_VERSION "1.65.1")
-  set(MASON_STXXL_VERSION "1.4.1-1")
   set(MASON_EXPAT_VERSION "2.2.0")
   set(MASON_LUA_VERSION "5.2.4")
   set(MASON_BZIP2_VERSION "1.0.6")
@@ -480,13 +478,6 @@ if(ENABLE_MASON)
   mason_use(boost_libsystem VERSION ${MASON_BOOST_VERSION})
   set(Boost_SYSTEM_LIBRARY ${MASON_PACKAGE_boost_libsystem_STATIC_LIBS})
 
-  if (ENABLE_STXXL)
-    mason_use(stxxl VERSION ${MASON_STXXL_VERSION})
-    add_dependency_includes(${MASON_PACKAGE_stxxl_INCLUDE_DIRS})
-    set(MAYBE_STXXL_LIBRARY ${MASON_PACKAGE_stxxl_STATIC_LIBS})
-    add_definitions(-DUSE_STXXL_LIBRARY)
-  endif()
-
   mason_use(expat VERSION ${MASON_EXPAT_VERSION})
   add_dependency_includes(${MASON_PACKAGE_expat_INCLUDE_DIRS})
   set(EXPAT_LIBRARIES ${MASON_PACKAGE_expat_STATIC_LIBS})
@@ -542,17 +533,6 @@ else()
   find_package(EXPAT REQUIRED)
   add_dependency_includes(${EXPAT_INCLUDE_DIRS})
 
-  if (ENABLE_STXXL)
-    find_package(STXXL)
-    if (STXXL_FOUND)
-      add_dependency_includes(${STXXL_INCLUDE_DIR})
-      set(MAYBE_STXXL_LIBRARY ${STXXL_LIBRARY})
-      add_definitions(-DUSE_STXXL_LIBRARY)
-    else()
-      MESSAGE(STATUS "STXXL was requested but not found, default STL will be used")
-    endif()
-  endif()
-
   find_package(BZip2 REQUIRED)
   add_dependency_includes(${BZIP2_INCLUDE_DIR})
 
@@ -612,15 +592,6 @@ add_dependency_defines(-DBOOST_SPIRIT_USE_PHOENIX_V3)
 add_dependency_defines(-DBOOST_RESULT_OF_USE_DECLTYPE)
 add_dependency_defines(-DBOOST_FILESYSTEM_NO_DEPRECATED)
 
-if (ENABLE_STXXL)
-  set(OpenMP_FIND_QUIETLY ON)
-  find_package(OpenMP)
-  if(OPENMP_FOUND)
-    message(STATUS "OpenMP support found. Linking just in case for stxxl")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-  endif()
-endif()
-
 add_definitions(${OSRM_DEFINES})
 include_directories(SYSTEM ${DEPENDENCIES_INCLUDE_DIRS})
 
@@ -653,7 +624,6 @@ set(EXTRACTOR_LIBRARIES
     ${EXPAT_LIBRARIES}
     ${USED_LUA_LIBRARIES}
     ${OSMIUM_LIBRARIES}
-    ${MAYBE_STXXL_LIBRARY}
     ${TBB_LIBRARIES}
     ${ZLIB_LIBRARY}
     ${MAYBE_COVERAGE_LIBRARIES})
@@ -687,7 +657,6 @@ set(CONTRACTOR_LIBRARIES
     ${BOOST_BASE_LIBRARIES}
     ${CMAKE_THREAD_LIBS_INIT}
     ${USED_LUA_LIBRARIES}
-    ${MAYBE_STXXL_LIBRARY}
     ${TBB_LIBRARIES}
     ${MAYBE_RT_LIBRARY}
     ${MAYBE_COVERAGE_LIBRARIES})
@@ -707,7 +676,6 @@ set(STORAGE_LIBRARIES
 set(UTIL_LIBRARIES
     ${BOOST_BASE_LIBRARIES}
     ${CMAKE_THREAD_LIBS_INIT}
-    ${MAYBE_STXXL_LIBRARY}
     ${TBB_LIBRARIES}
     ${MAYBE_COVERAGE_LIBRARIES}
     ${ZLIB_LIBRARY})

cmake/FindSTXXL.cmake

@@ -1,51 +0,0 @@
-# Locate STXXL library
-# This module defines
-#  STXXL_FOUND, if false, do not try to link to libstxxl
-#  STXXL_LIBRARY
-#  STXXL_INCLUDE_DIR, where to find stxxl.h
-#
-
-
-IF( NOT STXXL_FIND_QUIETLY )
-    MESSAGE(STATUS "Looking for STXXL...")
-ENDIF()
-
-FIND_PATH(STXXL_INCLUDE_DIR stxxl.h
-  HINTS
-  $ENV{STXXL_DIR}
-  PATH_SUFFIXES stxxl include/stxxl/stxxl include/stxxl include
-  PATHS
-  ~/Library/Frameworks
-  /Library/Frameworks
-  /usr/local
-  /usr
-  /opt/local # DarwinPorts
-  /opt
-)
-
-FIND_LIBRARY(STXXL_LIBRARY
-  NAMES stxxl stxxl_debug
-  HINTS
-  $ENV{STXXL_DIR}
-  PATH_SUFFIXES lib64 lib
-  PATHS
-  ~/Library/Frameworks
-  /Library/Frameworks
-  /usr/local
-  /usr
-  /opt/local
-  /opt
-)
-
-INCLUDE(FindPackageHandleStandardArgs)
-# handle the QUIETLY and REQUIRED arguments and set STXXL_FOUND to TRUE if
-# all listed variables are TRUE
-FIND_PACKAGE_HANDLE_STANDARD_ARGS(STXXL  DEFAULT_MSG  STXXL_LIBRARY STXXL_INCLUDE_DIR)
-
-IF( NOT STXXL_FIND_QUIETLY )
-    IF( STXXL_FOUND )
-        MESSAGE(STATUS "Found STXXL: ${STXXL_LIBRARY}" )
-    ENDIF()
-ENDIF()
-
-MARK_AS_ADVANCED(STXXL_INCLUDE_DIR STXXL_LIBRARY)

features/support/env.js

@@ -22,14 +22,9 @@ module.exports = function () {
         this.PROFILES_PATH = path.resolve(this.ROOT_PATH, 'profiles');
         this.FIXTURES_PATH = path.resolve(this.ROOT_PATH, 'unit_tests/fixtures');
         this.BIN_PATH = process.env.OSRM_BUILD_DIR && process.env.OSRM_BUILD_DIR || path.resolve(this.ROOT_PATH, 'build');
-        var stxxl_config = path.resolve(this.ROOT_PATH, 'test/.stxxl');
-        if (!fs.existsSync(stxxl_config)) {
-            return callback(new Error('*** '+stxxl_config+ 'does not exist'));
-        }
-
         this.DATASET_NAME = 'cucumber';
         this.PLATFORM_WINDOWS = process.platform.match(/^win.*/);
-        this.DEFAULT_ENVIRONMENT = Object.assign({STXXLCFG: stxxl_config}, process.env);
+        this.DEFAULT_ENVIRONMENT = process.env;
         this.DEFAULT_PROFILE = 'bicycle';
         this.DEFAULT_INPUT_FORMAT = 'osm';
         this.DEFAULT_LOAD_METHOD = process.argv[process.argv.indexOf('-m') +1].match('mmap') ? 'mmap' : 'datastore';

include/extractor/internal_extractor_edge.hpp

@@ -85,33 +85,6 @@ struct InternalExtractorEdge
     DurationData duration_data;
     // coordinate of the source node
     util::Coordinate source_coordinate;
-
-    // necessary static util functions for stxxl's sorting
-    static InternalExtractorEdge min_osm_value()
-    {
-        return InternalExtractorEdge(
-            MIN_OSM_NODEID, MIN_OSM_NODEID, WeightData(), DurationData(), util::Coordinate());
-    }
-    static InternalExtractorEdge max_osm_value()
-    {
-        return InternalExtractorEdge(
-            MAX_OSM_NODEID, MAX_OSM_NODEID, WeightData(), DurationData(), util::Coordinate());
-    }
-
-    static InternalExtractorEdge min_internal_value()
-    {
-        auto v = min_osm_value();
-        v.result.source = 0;
-        v.result.target = 0;
-        return v;
-    }
-    static InternalExtractorEdge max_internal_value()
-    {
-        auto v = max_osm_value();
-        v.result.source = std::numeric_limits<NodeID>::max();
-        v.result.target = std::numeric_limits<NodeID>::max();
-        return v;
-    }
 };
 }
 }

include/storage/serialization.hpp

@@ -17,10 +17,6 @@
 #include <cstdint>
 #include <tuple>
 
-#if USE_STXXL_LIBRARY
-#include <stxxl/vector>
-#endif
-
 namespace osrm
 {
 namespace storage
@@ -134,24 +130,6 @@ inline void write(storage::tar::FileWriter &writer,
     writer.WriteStreaming<T>(name, vec.begin(), vec.size());
 }
 
-#if USE_STXXL_LIBRARY
-template <typename T>
-inline void read(storage::tar::FileReader &reader, const std::string &name, stxxl::vector<T> &vec)
-{
-    auto size = reader.ReadElementCount64(name);
-    vec.reserve(size);
-    reader.ReadStreaming<T>(name, std::back_inserter(vec), size);
-}
-
-template <typename T>
-inline void
-write(storage::tar::FileWriter &writer, const std::string &name, const stxxl::vector<T> &vec)
-{
-    writer.WriteElementCount64(name, vec.size());
-    writer.WriteStreaming<T>(name, vec.begin(), vec.size());
-}
-#endif
-
 template <typename T> void read(io::BufferReader &reader, std::vector<T> &data)
 {
     const auto count = reader.ReadElementCount64();

include/util/meminfo.hpp

@@ -7,29 +7,11 @@
 #include <sys/resource.h>
 #endif
 
-#if USE_STXXL_LIBRARY
-#include <stxxl/mng>
-#endif
-
 namespace osrm
 {
 namespace util
 {
 
-inline void DumpSTXXLStats()
-{
-#if USE_STXXL_LIBRARY
-#if STXXL_VERSION_MAJOR > 1 || (STXXL_VERSION_MAJOR == 1 && STXXL_VERSION_MINOR >= 4)
-    auto manager = stxxl::block_manager::get_instance();
-    util::Log() << "STXXL: peak bytes used: " << manager->get_maximum_allocation();
-    util::Log() << "STXXL: total disk allocated: " << manager->get_total_bytes();
-#else
-#warning STXXL 1.4+ recommended - STXXL memory summary will not be available
-    util::Log() << "STXXL: memory summary not available, needs STXXL 1.4 or higher";
-#endif
-#endif
-}
-
 inline void DumpMemoryStats()
 {
 #ifndef _WIN32

include/util/vector_view.hpp

@@ -19,10 +19,6 @@
 #include <utility>
 #include <vector>
 
-#if USE_STXXL_LIBRARY
-#include <stxxl/vector>
-#endif
-
 namespace osrm
 {
 namespace util
@@ -253,16 +249,10 @@ template <typename DataT> void swap(vector_view<DataT> &lhs, vector_view<DataT>
     std::swap(lhs.m_size, rhs.m_size);
 }
 
-#if USE_STXXL_LIBRARY
-template <typename T> using ExternalVector = stxxl::vector<T>;
-#else
-template <typename T> using ExternalVector = std::vector<T>;
-#endif
-
 template <typename DataT, storage::Ownership Ownership>
 using InternalOrExternalVector =
     typename std::conditional<Ownership == storage::Ownership::External,
-                              ExternalVector<DataT>,
+                              std::vector<DataT>,
                               std::vector<DataT>>::type;
 
 template <typename DataT, storage::Ownership Ownership>

scripts/travis/before_install.armhf.sh

@@ -18,11 +18,3 @@ sudo apt-get update -qq --yes || true
 
 sudo apt-get install -qq --yes --force-yes g++-4.8-arm-linux-gnueabihf g++-4.8-multilib-arm-linux-gnueabihf gcc-4.8-arm-linux-gnueabihf gcc-4.8-multilib-arm-linux-gnueabihf
 sudo apt-get install -qq --yes --force-yes libexpat1-dev:armhf zlib1g-dev:armhf libbz2-dev:armhf libboost-date-time-dev:armhf libboost-filesystem-dev:armhf libboost-iostreams-dev:armhf libboost-program-options-dev:armhf libboost-regex-dev:armhf libboost-system-dev:armhf libboost-thread-dev:armhf libtbb-dev:armhf libboost-test-dev:armhf
-
-## build libstxxl1v5:armhf from sources, no package in trusty
-if [ $UBUNTU_RELEASE == trusty ] ; then
-    ( cd /tmp && wget http://sourceforge.net/projects/stxxl/files/stxxl/1.4.1/stxxl-1.4.1.tar.gz && tar xf stxxl-1.4.1.tar.gz )
-    ( cd /tmp/stxxl-1.4.1 && mkdir build && cd build && CC=arm-linux-gnueabihf-gcc-4.8 CXX=arm-linux-gnueabihf-g++-4.8 cmake .. && make && sudo make install )
-else
-    sudo apt-get install -qq --yes --force-yes libstxxl1v5:armhf
-fi

scripts/travis/before_install.i686.sh

@@ -3,4 +3,4 @@
 sudo dpkg --add-architecture i386
 sudo add-apt-repository --yes ppa:ubuntu-toolchain-r/test && ( sudo apt-get update -qq --yes || true )
 
-sudo apt-get install -qq --yes --force-yes g++-7-multilib libxml2-dev:i386 libexpat1-dev:i386 libzip-dev:i386 libbz2-dev:i386 libstxxl-dev:i386 libtbb-dev:i386 lua5.2:i386 liblua5.2-dev:i386 libboost-date-time-dev:i386 libboost-filesystem-dev:i386 libboost-iostreams-dev:i386 libboost-program-options-dev:i386 libboost-regex-dev:i386 libboost-system-dev:i386 libboost-thread-dev:i386 libboost-test-dev:i386
+sudo apt-get install -qq --yes --force-yes g++-7-multilib libxml2-dev:i386 libexpat1-dev:i386 libzip-dev:i386 libbz2-dev:i386 libtbb-dev:i386 lua5.2:i386 liblua5.2-dev:i386 libboost-date-time-dev:i386 libboost-filesystem-dev:i386 libboost-iostreams-dev:i386 libboost-program-options-dev:i386 libboost-regex-dev:i386 libboost-system-dev:i386 libboost-thread-dev:i386 libboost-test-dev:i386

src/tools/contract.cpp

@@ -189,21 +189,18 @@ int main(int argc, char *argv[]) try
 
     osrm::contract(contractor_config);
 
-    util::DumpSTXXLStats();
     util::DumpMemoryStats();
 
     return EXIT_SUCCESS;
 }
 catch (const osrm::RuntimeError &e)
 {
-    util::DumpSTXXLStats();
     util::DumpMemoryStats();
     util::Log(logERROR) << e.what();
     return e.GetCode();
 }
 catch (const std::bad_alloc &e)
 {
-    util::DumpSTXXLStats();
     util::DumpMemoryStats();
     util::Log(logERROR) << e.what();
     util::Log(logERROR) << "Please provide more memory or consider using a larger swapfile";

src/tools/extract.cpp

@@ -186,28 +186,24 @@ int main(int argc, char *argv[]) try
 
     osrm::extract(extractor_config);
 
-    util::DumpSTXXLStats();
     util::DumpMemoryStats();
 
     return EXIT_SUCCESS;
 }
 catch (const osrm::RuntimeError &e)
 {
-    util::DumpSTXXLStats();
     util::DumpMemoryStats();
     util::Log(logERROR) << e.what();
     return e.GetCode();
 }
 catch (const std::system_error &e)
 {
-    util::DumpSTXXLStats();
     util::DumpMemoryStats();
     util::Log(logERROR) << e.what();
     return e.code().value();
 }
 catch (const std::bad_alloc &e)
 {
-    util::DumpSTXXLStats();
     util::DumpMemoryStats();
     util::Log(logERROR) << "[exception] " << e.what();
     util::Log(logERROR) << "Please provide more memory or consider using a larger swapfile";

test/.stxxl

@@ -1 +0,0 @@
-disk=###.stxxl,20,memory

test/data/.stxxl

@@ -1 +0,0 @@
-disk=/tmp/stxxl_###,200M,syscall autogrow