From a91554291644dbc2df17aa16d36bfdac05a165f0 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 6 Apr 2018 00:49:23 +0000 Subject: [PATCH] Add option to only update the metric dependent data --- features/options/datastore/datastore.feature | 17 +++++++- include/storage/storage.hpp | 2 +- src/storage/storage.cpp | 44 ++++++++++++-------- src/tools/store.cpp | 16 +++++-- 4 files changed, 54 insertions(+), 25 deletions(-) diff --git a/features/options/datastore/datastore.feature b/features/options/datastore/datastore.feature index c48410160..26c875d93 100644 --- a/features/options/datastore/datastore.feature +++ b/features/options/datastore/datastore.feature @@ -1,5 +1,5 @@ @datastore @options @help -Feature: osrm-datastore command line options: list +Feature: osrm-datastore command line options Background: Given the profile "testbot" @@ -17,4 +17,17 @@ Feature: osrm-datastore command line options: list Then it should exit successfully When I try to run "osrm-datastore --list" Then it should exit successfully - And stdout should contain "test_dataset_42/data" + And stdout should contain "test_dataset_42/static" + + Scenario: osrm-datastore - Only metric update should work + Given the speed file + """ + 0,1,50 + """ + And the data has been extracted + When I try to run "osrm-datastore {processed_file} --dataset-name cucumber/only_metric_test" + Then it should exit successfully + When I try to run "osrm-customize --segment-speed-file {speeds_file} {processed_file}" + Then it should exit successfully + When I try to run "osrm-datastore {processed_file} --dataset-name cucumber/only_metric_test --only-metric" + Then it should exit successfully diff --git a/include/storage/storage.hpp b/include/storage/storage.hpp index 52364b9b3..c8add348c 100644 --- a/include/storage/storage.hpp +++ b/include/storage/storage.hpp @@ -45,7 +45,7 @@ class Storage public: Storage(StorageConfig config); - int Run(int max_wait, const std::string &name); + int Run(int max_wait, const std::string &name, bool only_metric); void PopulateStaticLayout(DataLayout &layout); void PopulateUpdatableLayout(DataLayout &layout); diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 8e845afa6..22e8aec6b 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -186,7 +186,7 @@ bool swapData(Monitor &monitor, Storage::Storage(StorageConfig config_) : config(std::move(config_)) {} -int Storage::Run(int max_wait, const std::string &dataset_name) +int Storage::Run(int max_wait, const std::string &dataset_name, bool only_metric) { BOOST_ASSERT_MSG(config.IsValid(), "Invalid storage config"); @@ -225,24 +225,32 @@ int Storage::Run(int max_wait, const std::string &dataset_name) auto &shared_register = monitor.data(); // Populate a memory layout into stack memory - DataLayout static_layout; - PopulateStaticLayout(static_layout); + std::vector regions; + std::map handles; + + if (!only_metric) + { + DataLayout static_layout; + PopulateStaticLayout(static_layout); + auto static_handle = setupRegion(shared_register, static_layout); + regions.push_back({static_handle.data_ptr, static_layout}); + handles[dataset_name + "/static"] = std::move(static_handle); + } + DataLayout updatable_layout; PopulateUpdatableLayout(updatable_layout); - - auto static_handle = setupRegion(shared_register, static_layout); auto updatable_handle = setupRegion(shared_register, updatable_layout); - - SharedDataIndex index{ - {{static_handle.data_ptr, static_layout}, {updatable_handle.data_ptr, updatable_layout}}}; - - PopulateStaticData(index); - PopulateUpdatableData(index); - - std::map handles; - handles[dataset_name + "/static"] = std::move(static_handle); + regions.push_back({updatable_handle.data_ptr, updatable_layout}); handles[dataset_name + "/updatable"] = std::move(updatable_handle); + SharedDataIndex index{std::move(regions)}; + + if (!only_metric) + { + PopulateStaticData(index); + } + PopulateUpdatableData(index); + swapData(monitor, shared_register, handles, max_wait); return EXIT_SUCCESS; @@ -400,8 +408,8 @@ void Storage::PopulateStaticData(const SharedDataIndex &index) std::string metric_name; // load profile properties { - const auto profile_properties_ptr = index.GetBlockPtr( - "/common/properties"); + const auto profile_properties_ptr = + index.GetBlockPtr("/common/properties"); extractor::files::readProfileProperties(config.GetPath(".osrm.properties"), *profile_properties_ptr); @@ -455,8 +463,8 @@ void Storage::PopulateUpdatableData(const SharedDataIndex &index) } { - const auto datasources_names_ptr = index.GetBlockPtr( - "/common/data_sources_names"); + const auto datasources_names_ptr = + index.GetBlockPtr("/common/data_sources_names"); extractor::files::readDatasources(config.GetPath(".osrm.datasource_names"), *datasources_names_ptr); } diff --git a/src/tools/store.cpp b/src/tools/store.cpp index 9342a10e7..7289eaa22 100644 --- a/src/tools/store.cpp +++ b/src/tools/store.cpp @@ -75,7 +75,8 @@ bool generateDataStoreOptions(const int argc, boost::filesystem::path &base_path, int &max_wait, std::string &dataset_name, - bool &list_datasets) + bool &list_datasets, + bool &only_metric) { // declare a group of options that will be allowed only on command line boost::program_options::options_description generic_options("Options"); @@ -105,7 +106,13 @@ bool generateDataStoreOptions(const int argc, ->default_value(false) ->implicit_value(true), "Name of the dataset to load into memory. This allows having multiple datasets in memory " - "at the same time."); + "at the same time.") // + ("only-metric", + boost::program_options::value(&only_metric) + ->default_value(false) + ->implicit_value(true), + "Only reload the metric data without updating the full dataset. This is an optimization " + "for traffic updates."); // hidden options, will be allowed on command line but will not be shown to the user boost::program_options::options_description hidden_options("Hidden options"); @@ -200,8 +207,9 @@ int main(const int argc, const char *argv[]) try int max_wait = -1; std::string dataset_name; bool list_datasets = false; + bool only_metric = false; if (!generateDataStoreOptions( - argc, argv, verbosity, base_path, max_wait, dataset_name, list_datasets)) + argc, argv, verbosity, base_path, max_wait, dataset_name, list_datasets, only_metric)) { return EXIT_SUCCESS; } @@ -222,7 +230,7 @@ int main(const int argc, const char *argv[]) try } storage::Storage storage(std::move(config)); - return storage.Run(max_wait, dataset_name); + return storage.Run(max_wait, dataset_name, only_metric); } catch (const osrm::RuntimeError &e) {