From 2c80f76004404609eb2978f687c7959d2870ad46 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 4 Apr 2018 12:51:46 +0000 Subject: [PATCH] Add support for naming the dataset --- CHANGELOG.md | 4 ++ include/engine/data_watchdog.hpp | 10 +++-- include/engine/datafacade_provider.hpp | 2 + include/engine/engine.hpp | 2 +- include/engine/engine_config.hpp | 1 + include/storage/shared_datatype.hpp | 12 ++++++ include/storage/shared_monitor.hpp | 7 ++-- include/storage/storage.hpp | 2 +- src/storage/storage.cpp | 8 ++-- src/tools/routed.cpp | 3 ++ src/tools/store.cpp | 51 ++++++++++++++++++++++---- 11 files changed, 82 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d817bd0e..3e975ca30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,12 @@ - ADDED: Maneuver relation now supports `straight` as a direction [#4995](https://github.com/Project-OSRM/osrm-backend/pull/4995) - Tools: - ADDED: `osrm-routed` accepts a new property `--memory_file` to store memory in a file on disk. [#4881](https://github.com/Project-OSRM/osrm-backend/pull/4881) + - ADDED: `osrm-datastore` accepts a new parameter `--dataset-name` to select the name of the dataset. [#4982](https://github.com/Project-OSRM/osrm-backend/pull/4982) + - ADDED: `osrm-datastore` accepts a new parameter `--list` to list all datasets loaded into memory. [#4982](https://github.com/Project-OSRM/osrm-backend/pull/4982) + - ADDED: `osrm-routed` accepts a new parameter `--dataset-name` to select the shared-memory dataset to use. [#4982](https://github.com/Project-OSRM/osrm-backend/pull/4982) - NodeJS: - ADDED: `OSRM` object accepts a new option `memory_file` that stores the memory in a file on disk. [#4881](https://github.com/Project-OSRM/osrm-backend/pull/4881) + - ADDED: `OSRM` object accepts a new option `dataset_name` to select the shared-memory dataset. [#4982](https://github.com/Project-OSRM/osrm-backend/pull/4982) - Internals - CHANGED: Updated segregated intersection identification [#4845](https://github.com/Project-OSRM/osrm-backend/pull/4845) [#4968](https://github.com/Project-OSRM/osrm-backend/pull/4968) - REMOVED: Remove `.timestamp` file since it was unused [#4960](https://github.com/Project-OSRM/osrm-backend/pull/4960) diff --git a/include/engine/data_watchdog.hpp b/include/engine/data_watchdog.hpp index fc6c428fd..9330d80b7 100644 --- a/include/engine/data_watchdog.hpp +++ b/include/engine/data_watchdog.hpp @@ -36,17 +36,18 @@ class DataWatchdogImpl; public: - DataWatchdogImpl() : active(true) + DataWatchdogImpl(const std::string &dataset_name) : dataset_name(dataset_name), active(true) { // create the initial facade before launching the watchdog thread { boost::interprocess::scoped_lock current_region_lock(barrier.get_mutex()); - auto& shared_register = barrier.data(); - auto region_id = shared_register.Find("data"); + auto &shared_register = barrier.data(); + auto region_id = shared_register.Find(dataset_name + "/data"); if (region_id == storage::SharedRegionRegister::INVALID_REGION_ID) { - throw util::exception("Could not find shared memory region. Did you run osrm-datastore?"); + throw util::exception( + "Could not find shared memory region. Did you run osrm-datastore?"); } shared_region = &shared_register.GetRegion(region_id); region = *shared_region; @@ -101,6 +102,7 @@ class DataWatchdogImpl barrier; std::thread watcher; bool active; diff --git a/include/engine/datafacade_provider.hpp b/include/engine/datafacade_provider.hpp index ee2f310a7..a624ee009 100644 --- a/include/engine/datafacade_provider.hpp +++ b/include/engine/datafacade_provider.hpp @@ -83,6 +83,8 @@ class WatchingProvider : public DataFacadeProvider public: using Facade = typename DataFacadeProvider::Facade; + WatchingProvider(const std::string &dataset_name) : watchdog(dataset_name) {} + std::shared_ptr Get(const api::TileParameters ¶ms) const override final { return watchdog.Get(params); diff --git a/include/engine/engine.hpp b/include/engine/engine.hpp index 97cd65b02..acce332b8 100644 --- a/include/engine/engine.hpp +++ b/include/engine/engine.hpp @@ -61,7 +61,7 @@ template class Engine final : public EngineInterface { util::Log(logDEBUG) << "Using shared memory with algorithm " << routing_algorithms::name(); - facade_provider = std::make_unique>(); + facade_provider = std::make_unique>(config.dataset_name); } else if (!config.memory_file.empty()) { diff --git a/include/engine/engine_config.hpp b/include/engine/engine_config.hpp index dcb8b145e..149cdbd39 100644 --- a/include/engine/engine_config.hpp +++ b/include/engine/engine_config.hpp @@ -91,6 +91,7 @@ struct EngineConfig final boost::filesystem::path memory_file; Algorithm algorithm = Algorithm::CH; std::string verbosity; + std::string dataset_name; }; } } diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp index f18462897..1783297a4 100644 --- a/include/storage/shared_datatype.hpp +++ b/include/storage/shared_datatype.hpp @@ -221,6 +221,18 @@ struct SharedRegionRegister } } + template + void List(OutIter out) const + { + for (const auto& region : regions) + { + if (!region.IsEmpty()) + { + *out++ = region.name; + } + } + } + void Deregister(const RegionID key) { regions[key] = SharedRegion{}; } const auto &GetRegion(const RegionID key) const { return regions[key]; } diff --git a/include/storage/shared_monitor.hpp b/include/storage/shared_monitor.hpp index 2d0e5d086..7c79b0ee9 100644 --- a/include/storage/shared_monitor.hpp +++ b/include/storage/shared_monitor.hpp @@ -126,7 +126,6 @@ template struct SharedMonitor bi::interprocess_condition condition; }; - #else // Implement a conditional variable using a queue of semaphores. // OSX checks the virtual address of a mutex in pthread_cond_wait and fails with EINVAL @@ -207,8 +206,10 @@ template struct SharedMonitor static_assert(buffer_size >= 2, "buffer size is too small"); #endif - static constexpr int rounded_internal_size = ((sizeof(InternalData) + alignof(Data) - 1) / alignof(Data)) * alignof(Data); - static_assert(rounded_internal_size < sizeof(InternalData) + sizeof(Data), "Data and internal data need to fit into shared memory"); + static constexpr int rounded_internal_size = + ((sizeof(InternalData) + alignof(Data) - 1) / alignof(Data)) * alignof(Data); + static_assert(rounded_internal_size < sizeof(InternalData) + sizeof(Data), + "Data and internal data need to fit into shared memory"); InternalData &internal() const { diff --git a/include/storage/storage.hpp b/include/storage/storage.hpp index 0ec48c1e9..8d9b436fe 100644 --- a/include/storage/storage.hpp +++ b/include/storage/storage.hpp @@ -44,7 +44,7 @@ class Storage public: Storage(StorageConfig config); - int Run(int max_wait); + int Run(int max_wait, const std::string &name); void PopulateLayout(DataLayout &layout); void PopulateData(const DataLayout &layout, char *memory_ptr); diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 06ad9ca4d..f21c7f266 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -65,7 +65,7 @@ using Monitor = SharedMonitor; Storage::Storage(StorageConfig config_) : config(std::move(config_)) {} -int Storage::Run(int max_wait) +int Storage::Run(int max_wait, const std::string &dataset_name) { BOOST_ASSERT_MSG(config.IsValid(), "Invalid storage config"); @@ -161,14 +161,14 @@ int Storage::Run(int max_wait) lock.lock(); } - auto region_id = shared_register.Find("data"); + auto region_id = shared_register.Find(dataset_name + "/data"); if (region_id == SharedRegionRegister::INVALID_REGION_ID) { - region_id = shared_register.Register("data", shm_key); + region_id = shared_register.Register(dataset_name + "/data", shm_key); } else { - auto& shared_region = shared_register.GetRegion(region_id); + auto &shared_region = shared_register.GetRegion(region_id); next_timestamp = shared_region.timestamp + 1; in_use_key = shared_region.shm_key; shared_region.shm_key = shm_key; diff --git a/src/tools/routed.cpp b/src/tools/routed.cpp index 15aea1cde..9d669a014 100644 --- a/src/tools/routed.cpp +++ b/src/tools/routed.cpp @@ -115,6 +115,9 @@ inline unsigned generateServerProgramOptions(const int argc, ("memory_file", value(&config.memory_file), "Store data in a memory mapped file rather than in process memory.") // + ("dataset-name", + value(&config.dataset_name), + "Name of the shared memory dataset to connect to.") // ("algorithm,a", value(&config.algorithm) ->default_value(EngineConfig::Algorithm::CH, "CH"), diff --git a/src/tools/store.cpp b/src/tools/store.cpp index 17d0a1dfd..f63b0397b 100644 --- a/src/tools/store.cpp +++ b/src/tools/store.cpp @@ -25,6 +25,23 @@ void deleteRegion(const storage::SharedRegionRegister::ShmKey key) } } +void listRegions() +{ + + storage::SharedMonitor monitor; + std::vector names; + const auto &shared_register = monitor.data(); + shared_register.List(std::back_inserter(names)); + osrm::util::Log() << "name\tshm key\ttimestamp"; + for (const auto &name : names) + { + auto id = shared_register.Find(name); + auto region = shared_register.GetRegion(id); + osrm::util::Log() << name << "\t" + << (int) region.shm_key << "\t" << region.timestamp; + } +} + void springClean() { osrm::util::Log() << "Releasing all locks"; @@ -55,7 +72,9 @@ bool generateDataStoreOptions(const int argc, const char *argv[], std::string &verbosity, boost::filesystem::path &base_path, - int &max_wait) + int &max_wait, + std::string &dataset_name, + bool &list_datasets) { // declare a group of options that will be allowed only on command line boost::program_options::options_description generic_options("Options"); @@ -69,10 +88,19 @@ bool generateDataStoreOptions(const int argc, // declare a group of options that will be allowed both on command line // as well as in a config file boost::program_options::options_description config_options("Configuration"); - config_options.add_options()("max-wait", - boost::program_options::value(&max_wait)->default_value(-1), - "Maximum number of seconds to wait on a running data update " - "before aquiring the lock by force."); + config_options.add_options() // + ("max-wait", + boost::program_options::value(&max_wait)->default_value(-1), + "Maximum number of seconds to wait on a running data update " + "before aquiring the lock by force.") // + ("dataset-name", + boost::program_options::value(&dataset_name)->default_value(""), + "Name of the dataset to load into memory. This allows having multiple datasets in memory " + "at the same time.") // + ("list", + boost::program_options::value(&list_datasets)->default_value(false)->implicit_value(true), + "Name of the dataset to load into memory. This allows having multiple datasets in memory " + "at the same time."); // hidden options, will be allowed on command line but will not be shown to the user boost::program_options::options_description hidden_options("Hidden options"); @@ -165,13 +193,22 @@ int main(const int argc, const char *argv[]) try std::string verbosity; boost::filesystem::path base_path; int max_wait = -1; - if (!generateDataStoreOptions(argc, argv, verbosity, base_path, max_wait)) + std::string dataset_name; + bool list_datasets = false; + if (!generateDataStoreOptions( + argc, argv, verbosity, base_path, max_wait, dataset_name, list_datasets)) { return EXIT_SUCCESS; } util::LogPolicy::GetInstance().SetLevel(verbosity); + if (list_datasets) + { + listRegions(); + return EXIT_SUCCESS; + } + storage::StorageConfig config(base_path); if (!config.IsValid()) { @@ -180,7 +217,7 @@ int main(const int argc, const char *argv[]) try } storage::Storage storage(std::move(config)); - return storage.Run(max_wait); + return storage.Run(max_wait, dataset_name); } catch (const osrm::RuntimeError &e) {