Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

uncouple configuration of Extractor into compile unit

Browse Source
This commit is contained in:
Dennis Luxen 2014-08-29 15:45:12 +02:00
parent 59f1679d37
commit ea1a89290a
4 changed files with 264 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

220
Extractor/Extractor.cpp
View File

@ -31,6 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "ExtractionNode.h"
#include "ExtractionWay.h"
#include "ExtractorCallbacks.h"
#include "ExtractorOptions.h"
#include "RestrictionParser.h"
#include "ScriptingEnvironment.h"
@ -42,14 +43,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../Util/make_unique.hpp"
#include "../typedefs.h"
#include <boost/program_options.hpp>
#include <luabind/luabind.hpp>
#include <tbb/task_scheduler_init.h>
#include <osmium/io/any_input.hpp>
#include <tbb/task_scheduler_init.h>
#include <cstdlib>
#include <chrono>
@ -58,11 +57,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <thread>
#include <unordered_map>
Extractor::Extractor() : requested_num_threads(0), file_has_pbf_format(false)
namespace
{
}
namespace {
int lua_error_callback(lua_State *L) // This is so I can use my own function as an
// exception handler, pcall_log()
{
@ -73,199 +69,66 @@ int lua_error_callback(lua_State *L) // This is so I can use my own function as
}
}
bool Extractor::ParseArguments(int argc, char *argv[])
{
// declare a group of options that will be allowed only on command line
boost::program_options::options_description generic_options("Options");
generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message")(
"config,c",
boost::program_options::value<boost::filesystem::path>(&config_file_path)
->default_value("extractor.ini"),
"Path to a configuration file.");
// declare a group of options that will be allowed both on command line and in config file
boost::program_options::options_description config_options("Configuration");
config_options.add_options()("profile,p",
boost::program_options::value<boost::filesystem::path>(
&profile_path)->default_value("profile.lua"),
"Path to LUA routing profile")(
"threads,t",
boost::program_options::value<unsigned int>(&requested_num_threads)
->default_value(tbb::task_scheduler_init::default_num_threads()),
"Number of threads to use");
// hidden options, will be allowed both on command line and in config file, but will not be
// shown to the user
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()(
"input,i",
boost::program_options::value<boost::filesystem::path>(&input_path),
"Input file in .osm, .osm.bz2 or .osm.pbf format");
// positional option
boost::program_options::positional_options_description positional_options;
positional_options.add("input", 1);
// combine above options for parsing
boost::program_options::options_description cmdline_options;
cmdline_options.add(generic_options).add(config_options).add(hidden_options);
boost::program_options::options_description config_file_options;
config_file_options.add(config_options).add(hidden_options);
boost::program_options::options_description visible_options(
boost::filesystem::basename(argv[0]) + " <input.osm/.osm.bz2/.osm.pbf> [options]");
visible_options.add(generic_options).add(config_options);
// parse command line options
boost::program_options::variables_map option_variables;
boost::program_options::store(boost::program_options::command_line_parser(argc, argv)
.options(cmdline_options)
.positional(positional_options)
.run(),
option_variables);
if (option_variables.count("version"))
{
SimpleLogger().Write() << g_GIT_DESCRIPTION;
return false;
}
if (option_variables.count("help"))
{
SimpleLogger().Write() << visible_options;
return false;
}
boost::program_options::notify(option_variables);
// parse config file
if (boost::filesystem::is_regular_file(config_file_path))
{
SimpleLogger().Write() << "Reading options from: " << config_file_path.string();
std::string ini_file_contents = ReadIniFileAndLowerContents(config_file_path);
std::stringstream config_stream(ini_file_contents);
boost::program_options::store(parse_config_file(config_stream, config_file_options),
option_variables);
boost::program_options::notify(option_variables);
}
if (!option_variables.count("input"))
{
SimpleLogger().Write() << visible_options;
return false;
}
return true;
}
void Extractor::GenerateOutputFilesNames()
{
output_file_name = input_path.string();
restriction_file_name = input_path.string();
timestamp_file_name = input_path.string();
std::string::size_type pos = output_file_name.find(".osm.bz2");
if (pos == std::string::npos)
{
pos = output_file_name.find(".osm.pbf");
if (pos != std::string::npos)
{
file_has_pbf_format = true;
}
else
{
pos = output_file_name.find(".osm.xml");
}
}
if (pos == std::string::npos)
{
pos = output_file_name.find(".pbf");
if (pos != std::string::npos)
{
file_has_pbf_format = true;
}
}
if (pos == std::string::npos)
{
pos = output_file_name.find(".osm");
if (pos == std::string::npos)
{
output_file_name.append(".osrm");
restriction_file_name.append(".osrm.restrictions");
timestamp_file_name.append(".osrm.timestamp");
}
else
{
output_file_name.replace(pos, 5, ".osrm");
restriction_file_name.replace(pos, 5, ".osrm.restrictions");
timestamp_file_name.replace(pos, 5, ".osrm.timestamp");
}
}
else
{
output_file_name.replace(pos, 8, ".osrm");
restriction_file_name.replace(pos, 8, ".osrm.restrictions");
timestamp_file_name.replace(pos, 8, ".osrm.timestamp");
}
}
int Extractor::Run(int argc, char *argv[])
{
ExtractorConfig extractor_config;
try
{
LogPolicy::GetInstance().Unmute();
TIMER_START(extracting);
if (!ParseArguments(argc, argv))
if (!ExtractorOptions::ParseArguments(argc, argv, extractor_config))
{
return 0;
}
ExtractorOptions::GenerateOutputFilesNames(extractor_config);
if (1 > requested_num_threads)
if (1 > extractor_config.requested_num_threads)
{
SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger";
return 1;
}
if (!boost::filesystem::is_regular_file(input_path))
if (!boost::filesystem::is_regular_file(extractor_config.input_path))
{
SimpleLogger().Write(logWARNING) << "Input file " << input_path.string()
<< " not found!";
SimpleLogger().Write(logWARNING)
<< "Input file " << extractor_config.input_path.string() << " not found!";
return 1;
}
if (!boost::filesystem::is_regular_file(profile_path))
if (!boost::filesystem::is_regular_file(extractor_config.profile_path))
{
SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string()
SimpleLogger().Write(logWARNING) << "Profile " << extractor_config.profile_path.string()
<< " not found!";
return 1;
}
const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
SimpleLogger().Write() << "Input file: " << input_path.filename().string();
SimpleLogger().Write() << "Profile: " << profile_path.filename().string();
SimpleLogger().Write() << "Threads: " << requested_num_threads;
if (recommended_num_threads != requested_num_threads)
SimpleLogger().Write() << "Input file: " << extractor_config.input_path.filename().string();
SimpleLogger().Write() << "Profile: " << extractor_config.profile_path.filename().string();
SimpleLogger().Write() << "Threads: " << extractor_config.requested_num_threads;
if (recommended_num_threads != extractor_config.requested_num_threads)
{
SimpleLogger().Write(logWARNING) << "The recommended number of threads is "
<< recommended_num_threads
<< "! This setting may have performance side-effects.";
}
tbb::task_scheduler_init init(requested_num_threads);
tbb::task_scheduler_init init(extractor_config.requested_num_threads);
/*** Setup Scripting Environment ***/
ScriptingEnvironment scripting_environment(profile_path.string().c_str());
GenerateOutputFilesNames();
ScriptingEnvironment scripting_environment(extractor_config.profile_path.string().c_str());
std::unordered_map<std::string, NodeID> string_map;
ExtractionContainers extraction_containers;
string_map[""] = 0;
auto extractor_callbacks = osrm::make_unique<ExtractorCallbacks>(extraction_containers, string_map);
auto extractor_callbacks =
osrm::make_unique<ExtractorCallbacks>(extraction_containers, string_map);
osmium::io::File infile(input_path.string());
osmium::io::File infile(extractor_config.input_path.string());
osmium::io::Reader reader(infile);
osmium::io::Header header = reader.header();
@ -292,7 +155,7 @@ int Extractor::Run(int argc, char *argv[])
}
SimpleLogger().Write() << "timestamp: " << timestamp;
boost::filesystem::ofstream timestamp_out(timestamp_file_name);
boost::filesystem::ofstream timestamp_out(extractor_config.timestamp_file_name);
timestamp_out.write(timestamp.c_str(), timestamp.length());
timestamp_out.close();
@ -313,24 +176,21 @@ int Extractor::Run(int argc, char *argv[])
case osmium::item_type::node:
++number_of_nodes;
result_node.Clear();
luabind::call_function<void>(
lua_state,
"node_function",
boost::cref(static_cast<osmium::Node &>(entity)),
boost::ref(result_node));
luabind::call_function<void>(lua_state,
"node_function",
boost::cref(static_cast<osmium::Node &>(entity)),
boost::ref(result_node));
extractor_callbacks->ProcessNode(static_cast<osmium::Node &>(entity),
result_node);
break;
case osmium::item_type::way:
++number_of_ways;
result_way.Clear();
luabind::call_function<void>(
lua_state,
"way_function",
boost::cref(static_cast<osmium::Way &>(entity)),
boost::ref(result_way));
extractor_callbacks->ProcessWay(static_cast<osmium::Way &>(entity),
result_way);
luabind::call_function<void>(lua_state,
"way_function",
boost::cref(static_cast<osmium::Way &>(entity)),
boost::ref(result_way));
extractor_callbacks->ProcessWay(static_cast<osmium::Way &>(entity), result_way);
break;
case osmium::item_type::relation:
++number_of_relations;
@ -345,7 +205,9 @@ int Extractor::Run(int argc, char *argv[])
}
TIMER_STOP(parsing);
SimpleLogger().Write() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds";
SimpleLogger().Write() << "Raw input contains " << number_of_nodes << " nodes, " << number_of_ways << " ways, and " << number_of_relations << " relations";
SimpleLogger().Write() << "Raw input contains " << number_of_nodes << " nodes, "
<< number_of_ways << " ways, and " << number_of_relations
<< " relations";
extractor_callbacks.reset();
@ -355,12 +217,14 @@ int Extractor::Run(int argc, char *argv[])
return 1;
}
extraction_containers.PrepareData(output_file_name, restriction_file_name);
extraction_containers.PrepareData(extractor_config.output_file_name,
extractor_config.restriction_file_name);
TIMER_STOP(extracting);
SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s";
SimpleLogger().Write() << "To prepare the data for routing, run: "
<< "./osrm-prepare " << output_file_name << std::endl;
<< "./osrm-prepare " << extractor_config.output_file_name
<< std::endl;
}
catch (boost::program_options::too_many_positional_options_error &)
{

28
Extractor/Extractor.h
View File

@ -1,37 +1,13 @@
#ifndef EXTRACTOR_H_
#define EXTRACTOR_H_
#include <boost/filesystem.hpp>
#include <string>
class ExtractorCallbacks;
#include <boost/filesystem.hpp>
/** \brief Class of 'extract' utility. */
class Extractor
struct Extractor
{
protected:
unsigned requested_num_threads;
boost::filesystem::path config_file_path;
boost::filesystem::path input_path;
boost::filesystem::path profile_path;
std::string output_file_name;
std::string restriction_file_name;
std::string timestamp_file_name;
bool file_has_pbf_format;
/** \brief Parses "extractor's" command line arguments */
bool ParseArguments(int argc, char *argv[]);
/** \brief Parses config file, if present in options */
void GenerateOutputFilesNames();
public:
explicit Extractor();
Extractor(const Extractor &) = delete;
virtual ~Extractor() = default;
int Run(int argc, char *argv[]);
};
#endif /* EXTRACTOR_H_ */

167
Extractor/ExtractorOptions.cpp Normal file
View File

@ -0,0 +1,167 @@
/*
Copyright (c) 2013, Project OSRM, Dennis Luxen, others
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this list
of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "ExtractorOptions.h"
#include "../Util/GitDescription.h"
#include "../Util/IniFileUtil.h"
#include "../Util/SimpleLogger.h"
#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>
#include <tbb/task_scheduler_init.h>
bool ExtractorOptions::ParseArguments(int argc, char *argv[], ExtractorConfig &extractor_config)
{
// declare a group of options that will be allowed only on command line
boost::program_options::options_description generic_options("Options");
generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message")(
"config,c",
boost::program_options::value<boost::filesystem::path>(&extractor_config.config_file_path)
->default_value("extractor.ini"),
"Path to a configuration file.");
// declare a group of options that will be allowed both on command line and in config file
boost::program_options::options_description config_options("Configuration");
config_options.add_options()("profile,p",
boost::program_options::value<boost::filesystem::path>(
&extractor_config.profile_path)->default_value("profile.lua"),
"Path to LUA routing profile")(
"threads,t",
boost::program_options::value<unsigned int>(&extractor_config.requested_num_threads)
->default_value(tbb::task_scheduler_init::default_num_threads()),
"Number of threads to use");
// hidden options, will be allowed both on command line and in config file, but will not be
// shown to the user
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()(
"input,i",
boost::program_options::value<boost::filesystem::path>(&extractor_config.input_path),
"Input file in .osm, .osm.bz2 or .osm.pbf format");
// positional option
boost::program_options::positional_options_description positional_options;
positional_options.add("input", 1);
// combine above options for parsing
boost::program_options::options_description cmdline_options;
cmdline_options.add(generic_options).add(config_options).add(hidden_options);
boost::program_options::options_description config_file_options;
config_file_options.add(config_options).add(hidden_options);
boost::program_options::options_description visible_options(
boost::filesystem::basename(argv[0]) + " <input.osm/.osm.bz2/.osm.pbf> [options]");
visible_options.add(generic_options).add(config_options);
// parse command line options
boost::program_options::variables_map option_variables;
boost::program_options::store(boost::program_options::command_line_parser(argc, argv)
.options(cmdline_options)
.positional(positional_options)
.run(),
option_variables);
if (option_variables.count("version"))
{
SimpleLogger().Write() << g_GIT_DESCRIPTION;
return false;
}
if (option_variables.count("help"))
{
SimpleLogger().Write() << visible_options;
return false;
}
boost::program_options::notify(option_variables);
// parse config file
if (boost::filesystem::is_regular_file(extractor_config.config_file_path))
{
SimpleLogger().Write() << "Reading options from: "
<< extractor_config.config_file_path.string();
std::string ini_file_contents =
ReadIniFileAndLowerContents(extractor_config.config_file_path);
std::stringstream config_stream(ini_file_contents);
boost::program_options::store(parse_config_file(config_stream, config_file_options),
option_variables);
boost::program_options::notify(option_variables);
}
if (!option_variables.count("input"))
{
SimpleLogger().Write() << visible_options;
return false;
}
return true;
}
void ExtractorOptions::GenerateOutputFilesNames(ExtractorConfig &extractor_config)
{
boost::filesystem::path &input_path = extractor_config.input_path;
extractor_config.output_file_name = input_path.string();
extractor_config.restriction_file_name = input_path.string();
extractor_config.timestamp_file_name = input_path.string();
std::string::size_type pos = extractor_config.output_file_name.find(".osm.bz2");
if (pos == std::string::npos)
{
pos = extractor_config.output_file_name.find(".osm.pbf");
if (pos == std::string::npos)
{
pos = extractor_config.output_file_name.find(".osm.xml");
}
}
if (pos == std::string::npos)
{
pos = extractor_config.output_file_name.find(".pbf");
}
if (pos == std::string::npos)
{
pos = extractor_config.output_file_name.find(".osm");
if (pos == std::string::npos)
{
extractor_config.output_file_name.append(".osrm");
extractor_config.restriction_file_name.append(".osrm.restrictions");
extractor_config.timestamp_file_name.append(".osrm.timestamp");
}
else
{
extractor_config.output_file_name.replace(pos, 5, ".osrm");
extractor_config.restriction_file_name.replace(pos, 5, ".osrm.restrictions");
extractor_config.timestamp_file_name.replace(pos, 5, ".osrm.timestamp");
}
}
else
{
extractor_config.output_file_name.replace(pos, 8, ".osrm");
extractor_config.restriction_file_name.replace(pos, 8, ".osrm.restrictions");
extractor_config.timestamp_file_name.replace(pos, 8, ".osrm.timestamp");
}
}

53
Extractor/ExtractorOptions.h Normal file
View File

@ -0,0 +1,53 @@
/*
Copyright (c) 2013, Project OSRM, Dennis Luxen, others
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this list
of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef EXTRACTOR_OPTIONS_H__
#define EXTRACTOR_OPTIONS_H__
#include "Extractor.h"
struct ExtractorConfig
{
ExtractorConfig() noexcept : requested_num_threads(0) {}
unsigned requested_num_threads;
boost::filesystem::path config_file_path;
boost::filesystem::path input_path;
boost::filesystem::path profile_path;
std::string output_file_name;
std::string restriction_file_name;
std::string timestamp_file_name;
};
struct ExtractorOptions
{
static bool ParseArguments(int argc, char *argv[], ExtractorConfig &extractor_config);
static void GenerateOutputFilesNames(ExtractorConfig &extractor_config);
};
#endif // EXTRACTOR_OPTIONS_H__