Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add support for naming the dataset

Browse Source
This commit is contained in:
Patrick Niklaus 2018-04-04 12:51:46 +00:00 committed by Patrick Niklaus
parent 666ce46d36
commit 2c80f76004
11 changed files with 82 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -16,8 +16,12 @@
- ADDED: Maneuver relation now supports `straight` as a direction [#4995](https://github.com/Project-OSRM/osrm-backend/pull/4995)
- Tools:
- ADDED: `osrm-routed` accepts a new property `--memory_file` to store memory in a file on disk. [#4881](https://github.com/Project-OSRM/osrm-backend/pull/4881)
- ADDED: `osrm-datastore` accepts a new parameter `--dataset-name` to select the name of the dataset. [#4982](https://github.com/Project-OSRM/osrm-backend/pull/4982)
- ADDED: `osrm-datastore` accepts a new parameter `--list` to list all datasets loaded into memory. [#4982](https://github.com/Project-OSRM/osrm-backend/pull/4982)
- ADDED: `osrm-routed` accepts a new parameter `--dataset-name` to select the shared-memory dataset to use. [#4982](https://github.com/Project-OSRM/osrm-backend/pull/4982)
- NodeJS:
- ADDED: `OSRM` object accepts a new option `memory_file` that stores the memory in a file on disk. [#4881](https://github.com/Project-OSRM/osrm-backend/pull/4881)
- ADDED: `OSRM` object accepts a new option `dataset_name` to select the shared-memory dataset. [#4982](https://github.com/Project-OSRM/osrm-backend/pull/4982)
- Internals
- CHANGED: Updated segregated intersection identification [#4845](https://github.com/Project-OSRM/osrm-backend/pull/4845) [#4968](https://github.com/Project-OSRM/osrm-backend/pull/4968)
- REMOVED: Remove `.timestamp` file since it was unused [#4960](https://github.com/Project-OSRM/osrm-backend/pull/4960)

10
include/engine/data_watchdog.hpp
View File

@ -36,17 +36,18 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
using Facade = datafacade::ContiguousInternalMemoryDataFacade<AlgorithmT>;
public:
DataWatchdogImpl() : active(true)
DataWatchdogImpl(const std::string &dataset_name) : dataset_name(dataset_name), active(true)
{
// create the initial facade before launching the watchdog thread
{
boost::interprocess::scoped_lock<mutex_type> current_region_lock(barrier.get_mutex());
auto& shared_register = barrier.data();
auto region_id = shared_register.Find("data");
auto &shared_register = barrier.data();
auto region_id = shared_register.Find(dataset_name + "/data");
if (region_id == storage::SharedRegionRegister::INVALID_REGION_ID)
{
throw util::exception("Could not find shared memory region. Did you run osrm-datastore?");
throw util::exception(
"Could not find shared memory region. Did you run osrm-datastore?");
}
shared_region = &shared_register.GetRegion(region_id);
region = *shared_region;
@ -101,6 +102,7 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
util::Log() << "DataWatchdog thread stopped";
}
const std::string dataset_name;
storage::SharedMonitor<storage::SharedRegionRegister> barrier;
std::thread watcher;
bool active;

2
include/engine/datafacade_provider.hpp
View File

@ -83,6 +83,8 @@ class WatchingProvider : public DataFacadeProvider<AlgorithmT, FacadeT>
public:
using Facade = typename DataFacadeProvider<AlgorithmT, FacadeT>::Facade;
WatchingProvider(const std::string &dataset_name) : watchdog(dataset_name) {}
std::shared_ptr<const Facade> Get(const api::TileParameters &params) const override final
{
return watchdog.Get(params);

2
include/engine/engine.hpp
View File

@ -61,7 +61,7 @@ template <typename Algorithm> class Engine final : public EngineInterface
{
util::Log(logDEBUG) << "Using shared memory with algorithm "
<< routing_algorithms::name<Algorithm>();
facade_provider = std::make_unique<WatchingProvider<Algorithm>>();
facade_provider = std::make_unique<WatchingProvider<Algorithm>>(config.dataset_name);
}
else if (!config.memory_file.empty())
{

1
include/engine/engine_config.hpp
View File

@ -91,6 +91,7 @@ struct EngineConfig final
boost::filesystem::path memory_file;
Algorithm algorithm = Algorithm::CH;
std::string verbosity;
std::string dataset_name;
};
}
}

12
include/storage/shared_datatype.hpp
View File

@ -221,6 +221,18 @@ struct SharedRegionRegister
}
}
template<typename OutIter>
void List(OutIter out) const
{
for (const auto& region : regions)
{
if (!region.IsEmpty())
{
*out++ = region.name;
}
}
}
void Deregister(const RegionID key) { regions[key] = SharedRegion{}; }
const auto &GetRegion(const RegionID key) const { return regions[key]; }

7
include/storage/shared_monitor.hpp
View File

@ -126,7 +126,6 @@ template <typename Data> struct SharedMonitor
bi::interprocess_condition condition;
};
#else
// Implement a conditional variable using a queue of semaphores.
// OSX checks the virtual address of a mutex in pthread_cond_wait and fails with EINVAL
@ -207,8 +206,10 @@ template <typename Data> struct SharedMonitor
static_assert(buffer_size >= 2, "buffer size is too small");
#endif
static constexpr int rounded_internal_size = ((sizeof(InternalData) + alignof(Data) - 1) / alignof(Data)) * alignof(Data);
static_assert(rounded_internal_size < sizeof(InternalData) + sizeof(Data), "Data and internal data need to fit into shared memory");
static constexpr int rounded_internal_size =
((sizeof(InternalData) + alignof(Data) - 1) / alignof(Data)) * alignof(Data);
static_assert(rounded_internal_size < sizeof(InternalData) + sizeof(Data),
"Data and internal data need to fit into shared memory");
InternalData &internal() const
{

2
include/storage/storage.hpp
View File

@ -44,7 +44,7 @@ class Storage
public:
Storage(StorageConfig config);
int Run(int max_wait);
int Run(int max_wait, const std::string &name);
void PopulateLayout(DataLayout &layout);
void PopulateData(const DataLayout &layout, char *memory_ptr);

8
src/storage/storage.cpp
View File

@ -65,7 +65,7 @@ using Monitor = SharedMonitor<SharedRegionRegister>;
Storage::Storage(StorageConfig config_) : config(std::move(config_)) {}
int Storage::Run(int max_wait)
int Storage::Run(int max_wait, const std::string &dataset_name)
{
BOOST_ASSERT_MSG(config.IsValid(), "Invalid storage config");
@ -161,14 +161,14 @@ int Storage::Run(int max_wait)
lock.lock();
}
auto region_id = shared_register.Find("data");
auto region_id = shared_register.Find(dataset_name + "/data");
if (region_id == SharedRegionRegister::INVALID_REGION_ID)
{
region_id = shared_register.Register("data", shm_key);
region_id = shared_register.Register(dataset_name + "/data", shm_key);
}
else
{
auto& shared_region = shared_register.GetRegion(region_id);
auto &shared_region = shared_register.GetRegion(region_id);
next_timestamp = shared_region.timestamp + 1;
in_use_key = shared_region.shm_key;
shared_region.shm_key = shm_key;

3
src/tools/routed.cpp
View File

@ -115,6 +115,9 @@ inline unsigned generateServerProgramOptions(const int argc,
("memory_file",
value<boost::filesystem::path>(&config.memory_file),
"Store data in a memory mapped file rather than in process memory.") //
("dataset-name",
value<std::string>(&config.dataset_name),
"Name of the shared memory dataset to connect to.") //
("algorithm,a",
value<EngineConfig::Algorithm>(&config.algorithm)
->default_value(EngineConfig::Algorithm::CH, "CH"),

51
src/tools/store.cpp
View File

@ -25,6 +25,23 @@ void deleteRegion(const storage::SharedRegionRegister::ShmKey key)
}
}
void listRegions()
{
storage::SharedMonitor<storage::SharedRegionRegister> monitor;
std::vector<std::string> names;
const auto &shared_register = monitor.data();
shared_register.List(std::back_inserter(names));
osrm::util::Log() << "name\tshm key\ttimestamp";
for (const auto &name : names)
{
auto id = shared_register.Find(name);
auto region = shared_register.GetRegion(id);
osrm::util::Log() << name << "\t"
<< (int) region.shm_key << "\t" << region.timestamp;
}
}
void springClean()
{
osrm::util::Log() << "Releasing all locks";
@ -55,7 +72,9 @@ bool generateDataStoreOptions(const int argc,
const char *argv[],
std::string &verbosity,
boost::filesystem::path &base_path,
int &max_wait)
int &max_wait,
std::string &dataset_name,
bool &list_datasets)
{
// declare a group of options that will be allowed only on command line
boost::program_options::options_description generic_options("Options");
@ -69,10 +88,19 @@ bool generateDataStoreOptions(const int argc,
// declare a group of options that will be allowed both on command line
// as well as in a config file
boost::program_options::options_description config_options("Configuration");
config_options.add_options()("max-wait",
boost::program_options::value<int>(&max_wait)->default_value(-1),
"Maximum number of seconds to wait on a running data update "
"before aquiring the lock by force.");
config_options.add_options() //
("max-wait",
boost::program_options::value<int>(&max_wait)->default_value(-1),
"Maximum number of seconds to wait on a running data update "
"before aquiring the lock by force.") //
("dataset-name",
boost::program_options::value<std::string>(&dataset_name)->default_value(""),
"Name of the dataset to load into memory. This allows having multiple datasets in memory "
"at the same time.") //
("list",
boost::program_options::value<bool>(&list_datasets)->default_value(false)->implicit_value(true),
"Name of the dataset to load into memory. This allows having multiple datasets in memory "
"at the same time.");
// hidden options, will be allowed on command line but will not be shown to the user
boost::program_options::options_description hidden_options("Hidden options");
@ -165,13 +193,22 @@ int main(const int argc, const char *argv[]) try
std::string verbosity;
boost::filesystem::path base_path;
int max_wait = -1;
if (!generateDataStoreOptions(argc, argv, verbosity, base_path, max_wait))
std::string dataset_name;
bool list_datasets = false;
if (!generateDataStoreOptions(
argc, argv, verbosity, base_path, max_wait, dataset_name, list_datasets))
{
return EXIT_SUCCESS;
}
util::LogPolicy::GetInstance().SetLevel(verbosity);
if (list_datasets)
{
listRegions();
return EXIT_SUCCESS;
}
storage::StorageConfig config(base_path);
if (!config.IsValid())
{
@ -180,7 +217,7 @@ int main(const int argc, const char *argv[]) try
}
storage::Storage storage(std::move(config));
return storage.Run(max_wait);
return storage.Run(max_wait, dataset_name);
}
catch (const osrm::RuntimeError &e)
{