Add mmap allocator

include/engine/datafacade/mmap_memory_allocator.hpp

@@ -0,0 +1,45 @@
+#ifndef OSRM_ENGINE_DATAFACADE_MMAP_MEMORY_ALLOCATOR_HPP_
+#define OSRM_ENGINE_DATAFACADE_MMAP_MEMORY_ALLOCATOR_HPP_
+
+#include "engine/datafacade/contiguous_block_allocator.hpp"
+
+#include "storage/storage_config.hpp"
+
+#include "util/vector_view.hpp"
+
+#include <boost/iostreams/device/mapped_file.hpp>
+
+#include <memory>
+
+namespace osrm
+{
+namespace engine
+{
+namespace datafacade
+{
+
+/**
+ * This allocator uses file backed mmap memory block as the data location.
+ */
+class MMapMemoryAllocator : public ContiguousBlockAllocator
+{
+  public:
+    explicit MMapMemoryAllocator(const storage::StorageConfig &config,
+                                 const boost::filesystem::path &memory_file);
+    ~MMapMemoryAllocator() override final;
+
+    // interface to give access to the datafacades
+    storage::DataLayout &GetLayout() override final;
+    char *GetMemory() override final;
+
+  private:
+    storage::DataLayout *data_layout;
+    util::vector_view<char> mapped_memory;
+    boost::iostreams::mapped_file mapped_memory_file;
+};
+
+} // namespace datafacade
+} // namespace engine
+} // namespace osrm
+
+#endif // OSRM_ENGINE_DATAFACADE_SHARED_MEMORY_ALLOCATOR_HPP_

include/engine/datafacade_provider.hpp

@@ -4,6 +4,7 @@
 #include "engine/data_watchdog.hpp"
 #include "engine/datafacade.hpp"
 #include "engine/datafacade/contiguous_internalmem_datafacade.hpp"
+#include "engine/datafacade/mmap_memory_allocator.hpp"
 #include "engine/datafacade/process_memory_allocator.hpp"
 #include "engine/datafacade_factory.hpp"
 
@@ -25,6 +26,31 @@ template <typename AlgorithmT, template <typename A> class FacadeT> class DataFa
     virtual std::shared_ptr<const Facade> Get(const api::TileParameters &) const = 0;
 };
 
+template <typename AlgorithmT, template <typename A> class FacadeT>
+class ExternalProvider final : public DataFacadeProvider<AlgorithmT, FacadeT>
+{
+  public:
+    using Facade = typename DataFacadeProvider<AlgorithmT, FacadeT>::Facade;
+
+    ExternalProvider(const storage::StorageConfig &config,
+                     const boost::filesystem::path &memory_file)
+        : facade_factory(std::make_shared<datafacade::MMapMemoryAllocator>(config, memory_file))
+    {
+    }
+
+    std::shared_ptr<const Facade> Get(const api::TileParameters &params) const override final
+    {
+        return facade_factory.Get(params);
+    }
+    std::shared_ptr<const Facade> Get(const api::BaseParameters &params) const override final
+    {
+        return facade_factory.Get(params);
+    }
+
+  private:
+    DataFacadeFactory<FacadeT, AlgorithmT> facade_factory;
+};
+
 template <typename AlgorithmT, template <typename A> class FacadeT>
 class ImmutableProvider final : public DataFacadeProvider<AlgorithmT, FacadeT>
 {
@@ -74,6 +100,8 @@ template <typename AlgorithmT>
 using WatchingProvider = detail::WatchingProvider<AlgorithmT, DataFacade>;
 template <typename AlgorithmT>
 using ImmutableProvider = detail::ImmutableProvider<AlgorithmT, DataFacade>;
+template <typename AlgorithmT>
+using ExternalProvider = detail::ExternalProvider<AlgorithmT, DataFacade>;
 }
 }
 

include/engine/engine.hpp

@@ -67,6 +67,13 @@ template <typename Algorithm> class Engine final : public EngineInterface
                                 << routing_algorithms::name<Algorithm>();
             facade_provider = std::make_unique<WatchingProvider<Algorithm>>();
         }
+        else if (!config.memory_file.empty())
+        {
+            util::Log(logDEBUG) << "Using memory mapped filed at " << config.memory_file
+                                << " with algorithm " << routing_algorithms::name<Algorithm>();
+            facade_provider = std::make_unique<ExternalProvider<Algorithm>>(config.storage_config,
+                                                                            config.memory_file);
+        }
         else
         {
             util::Log(logDEBUG) << "Using internal memory with algorithm "

include/engine/engine_config.hpp

@@ -88,6 +88,7 @@ struct EngineConfig final
     int max_results_nearest = -1;
     int max_alternatives = 3; // set an arbitrary upper bound; can be adjusted by user
     bool use_shared_memory = true;
+    boost::filesystem::path memory_file;
     Algorithm algorithm = Algorithm::CH;
     std::string verbosity;
 };

include/nodejs/node_osrm_support.hpp

@@ -115,14 +115,31 @@ inline engine_config_ptr argumentsToEngineConfig(const Nan::FunctionCallbackInfo
     if (path.IsEmpty())
         return engine_config_ptr();
 
+    auto memory_file = params->Get(Nan::New("memory_file").ToLocalChecked());
+    if (memory_file.IsEmpty())
+        return engine_config_ptr();
+
     auto shared_memory = params->Get(Nan::New("shared_memory").ToLocalChecked());
     if (shared_memory.IsEmpty())
         return engine_config_ptr();
 
+    if (!memory_file->IsUndefined())
+    {
+        if (path->IsUndefined())
+        {
+            Nan::ThrowError("memory_file option requires a path to a file.");
+            return engine_config_ptr();
+        }
+
+        engine_config->memory_file =
+            *v8::String::Utf8Value(Nan::To<v8::String>(memory_file).ToLocalChecked());
+    }
+
     if (!path->IsUndefined())
     {
         engine_config->storage_config =
             osrm::StorageConfig(*v8::String::Utf8Value(Nan::To<v8::String>(path).ToLocalChecked()));
+
         engine_config->use_shared_memory = false;
     }
     if (!shared_memory->IsUndefined())

include/util/mmap_file.hpp

@@ -33,6 +33,32 @@ util::vector_view<T> mmapFile(const boost::filesystem::path &file, RegionT &regi
             SOURCE_REF);
     }
 }
+
+template <typename T, typename RegionT>
+util::vector_view<T>
+mmapFile(const boost::filesystem::path &file, RegionT &region, const std::size_t size)
+{
+    try
+    {
+        // Create a new file with the given size in bytes
+        boost::iostreams::mapped_file_params params;
+        params.path = file.string();
+        params.flags = boost::iostreams::mapped_file::readwrite;
+        params.new_file_size = size;
+        region.open(params);
+
+        std::size_t num_objects = size / sizeof(T);
+        auto data_ptr = region.data();
+        BOOST_ASSERT(reinterpret_cast<uintptr_t>(data_ptr) % alignof(T) == 0);
+        return util::vector_view<T>(reinterpret_cast<T *>(data_ptr), num_objects);
+    }
+    catch (const std::exception &exc)
+    {
+        throw exception(
+            boost::str(boost::format("File %1% mapping failed: %2%") % file % exc.what()) +
+            SOURCE_REF);
+    }
+}
 }
 
 template <typename T>
@@ -48,6 +74,14 @@ util::vector_view<T> mmapFile(const boost::filesystem::path &file,
 {
     return detail::mmapFile<T>(file, region);
 }
+
+template <typename T>
+util::vector_view<T> mmapFile(const boost::filesystem::path &file,
+                              boost::iostreams::mapped_file &region,
+                              std::size_t size)
+{
+    return detail::mmapFile<T>(file, region, size);
+}
 }
 }