diff --git a/CHANGELOG.md b/CHANGELOG.md index 078e95dd4..d9aca267d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # UNRELEASED - Changes from 5.19.0: + - Features: + - ADDED: direct mmapping of datafiles is now supported via the `-mmap` switch. [#5242](https://github.com/Project-OSRM/osrm-backend/pull/5242) + - REMOVED: the previous `--memory_file` switch is now deprecated and will fallback to `--mmap` [#5242](https://github.com/Project-OSRM/osrm-backend/pull/5242) # 5.19.0 - Changes from 5.18.0: diff --git a/features/lib/osrm_loader.js b/features/lib/osrm_loader.js index 51410d228..a29d53b8e 100644 --- a/features/lib/osrm_loader.js +++ b/features/lib/osrm_loader.js @@ -84,7 +84,47 @@ class OSRMDirectLoader extends OSRMBaseLoader { throw new Error(util.format('osrm-routed %s: %s', errorReason(err), err.cmd)); } }); - callback(); + + this.child.readyFunc = (data) => { + if (/running and waiting for requests/.test(data)) { + this.child.stdout.removeListener('data', this.child.readyFunc); + callback(); + } + }; + this.child.stdout.on('data',this.child.readyFunc); + } +}; + +class OSRMmmapLoader extends OSRMBaseLoader { + constructor (scope) { + super(scope); + } + + load (inputFile, callback) { + this.inputFile = inputFile; + this.shutdown(() => { + this.launch(callback); + }); + } + + osrmUp (callback) { + if (this.osrmIsRunning()) return callback(new Error("osrm-routed already running!")); + + const command_arguments = util.format('%s -p %d -i %s -a %s --mmap', this.inputFile, this.scope.OSRM_PORT, this.scope.OSRM_IP, this.scope.ROUTING_ALGORITHM); + this.child = this.scope.runBin('osrm-routed', command_arguments, this.scope.environment, (err) => { + if (err && err.signal !== 'SIGINT') { + this.child = null; + throw new Error(util.format('osrm-routed %s: %s', errorReason(err), err.cmd)); + } + }); + + this.child.readyFunc = (data) => { + if (/running and waiting for requests/.test(data)) { + this.child.stdout.removeListener('data', this.child.readyFunc); + callback(); + } + }; + this.child.stdout.on('data',this.child.readyFunc); } }; @@ -135,22 +175,32 @@ class OSRMLoader { this.scope = scope; this.sharedLoader = new OSRMDatastoreLoader(this.scope); this.directLoader = new OSRMDirectLoader(this.scope); + this.mmapLoader = new OSRMmmapLoader(this.scope); this.method = scope.DEFAULT_LOAD_METHOD; } load (inputFile, callback) { + if (!this.loader) { + this.loader = {shutdown: (cb) => cb() }; + } if (this.method === 'datastore') { - this.directLoader.shutdown((err) => { + this.loader.shutdown((err) => { if (err) return callback(err); this.loader = this.sharedLoader; this.sharedLoader.load(inputFile, callback); }); } else if (this.method === 'directly') { - this.sharedLoader.shutdown((err) => { + this.loader.shutdown((err) => { if (err) return callback(err); this.loader = this.directLoader; this.directLoader.load(inputFile, callback); }); + } else if (this.method === 'mmap') { + this.loader.shutdown((err) => { + if (err) return callback(err); + this.loader = this.mmapLoader; + this.mmapLoader.load(inputFile, callback); + }); } else { callback(new Error('*** Unknown load method ' + method)); } diff --git a/features/support/env.js b/features/support/env.js index 7866e0756..b1fd3ef95 100644 --- a/features/support/env.js +++ b/features/support/env.js @@ -32,7 +32,7 @@ module.exports = function () { this.DEFAULT_ENVIRONMENT = Object.assign({STXXLCFG: stxxl_config}, process.env); this.DEFAULT_PROFILE = 'bicycle'; this.DEFAULT_INPUT_FORMAT = 'osm'; - this.DEFAULT_LOAD_METHOD = 'datastore'; + this.DEFAULT_LOAD_METHOD = process.argv[process.argv.indexOf('-m') +1].match('mmap') ? 'mmap' : 'datastore'; this.DEFAULT_ORIGIN = [1,1]; this.OSM_USER = 'osrm'; this.OSM_UID = 1; diff --git a/include/engine/engine.hpp b/include/engine/engine.hpp index 025dc2deb..9121c7b0e 100644 --- a/include/engine/engine.hpp +++ b/include/engine/engine.hpp @@ -63,12 +63,16 @@ template class Engine final : public EngineInterface << "\" with algorithm " << routing_algorithms::name(); facade_provider = std::make_unique>(config.dataset_name); } - else if (!config.memory_file.empty()) + else if (!config.memory_file.empty() || config.use_mmap) { - util::Log(logDEBUG) << "Using memory mapped filed at " << config.memory_file - << " with algorithm " << routing_algorithms::name(); - facade_provider = std::make_unique>(config.storage_config, - config.memory_file); + if (!config.memory_file.empty()) + { + util::Log(logWARNING) + << "The 'memory_file' option is DEPRECATED - using direct mmaping instead"; + } + util::Log(logDEBUG) << "Using direct memory mapping with algorithm " + << routing_algorithms::name(); + facade_provider = std::make_unique>(config.storage_config); } else { diff --git a/include/engine/engine_config.hpp b/include/engine/engine_config.hpp index 149cdbd39..cc0cc4a93 100644 --- a/include/engine/engine_config.hpp +++ b/include/engine/engine_config.hpp @@ -89,6 +89,7 @@ struct EngineConfig final int max_alternatives = 3; // set an arbitrary upper bound; can be adjusted by user bool use_shared_memory = true; boost::filesystem::path memory_file; + bool use_mmap = true; Algorithm algorithm = Algorithm::CH; std::string verbosity; std::string dataset_name; diff --git a/include/nodejs/node_osrm_support.hpp b/include/nodejs/node_osrm_support.hpp index 4a93eb7a7..27752e431 100644 --- a/include/nodejs/node_osrm_support.hpp +++ b/include/nodejs/node_osrm_support.hpp @@ -142,6 +142,10 @@ inline engine_config_ptr argumentsToEngineConfig(const Nan::FunctionCallbackInfo if (shared_memory.IsEmpty()) return engine_config_ptr(); + auto mmap_memory = params->Get(Nan::New("mmap_memory").ToLocalChecked()); + if (mmap_memory.IsEmpty()) + return engine_config_ptr(); + if (!memory_file->IsUndefined()) { if (path->IsUndefined()) @@ -190,6 +194,18 @@ inline engine_config_ptr argumentsToEngineConfig(const Nan::FunctionCallbackInfo return engine_config_ptr(); } } + if (!mmap_memory->IsUndefined()) + { + if (mmap_memory->IsBoolean()) + { + engine_config->use_mmap = Nan::To(mmap_memory).FromJust(); + } + else + { + Nan::ThrowError("mmap_memory option must be a boolean"); + return engine_config_ptr(); + } + } if (path->IsUndefined() && !engine_config->use_shared_memory) { diff --git a/package.json b/package.json index d56bc5afc..7c0f7001b 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ }, "scripts": { "lint": "node ./node_modules/eslint/bin/eslint.js -c ./.eslintrc features/step_definitions/ features/support/", - "test": "npm run lint && node ./node_modules/cucumber/bin/cucumber.js features/ -p verify && node ./node_modules/cucumber/bin/cucumber.js features/ -p mld", + "test": "npm run lint && node ./node_modules/cucumber/bin/cucumber.js features/ -p verify && node ./node_modules/cucumber/bin/cucumber.js features/ -p verify -m mmap && node ./node_modules/cucumber/bin/cucumber.js features/ -p mld && node ./node_modules/cucumber/bin/cucumber.js features/ -p mld -m mmap", "clean": "rm -rf test/cache", "docs": "./scripts/build_api_docs.sh", "install": "node-pre-gyp install --fallback-to-build=false || ./scripts/node_install.sh", diff --git a/src/engine/engine_config.cpp b/src/engine/engine_config.cpp index 94f1f0a56..ac7723ba9 100644 --- a/src/engine/engine_config.cpp +++ b/src/engine/engine_config.cpp @@ -23,7 +23,9 @@ bool EngineConfig::IsValid() const unlimited_or_more_than(max_results_nearest, 0) && max_alternatives >= 0; - return ((use_shared_memory && all_path_are_empty) || storage_config.IsValid()) && limits_valid; + return ((use_shared_memory && all_path_are_empty) || (use_mmap && storage_config.IsValid()) || + storage_config.IsValid()) && + limits_valid; } } } diff --git a/src/tools/routed.cpp b/src/tools/routed.cpp index 25a62b03a..718e3e4d0 100644 --- a/src/tools/routed.cpp +++ b/src/tools/routed.cpp @@ -119,7 +119,10 @@ inline unsigned generateServerProgramOptions(const int argc, "Load data from shared memory") // ("memory_file", value(&config.memory_file), - "Store data in a memory mapped file rather than in process memory.") // + "DEPRECATED: Will behave the same as --mmap.")( + "mmap,m", + value(&config.use_mmap)->implicit_value(true)->default_value(false), + "Map datafiles directly, do not use any additional memory.") // ("dataset-name", value(&config.dataset_name), "Name of the shared memory dataset to connect to.") //