Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

use boost program_options in extractor

Browse Source
This commit is contained in:
Emil Tin 2013-08-10 13:29:24 +02:00
parent 4748bca8a4
commit e42b8fc37d
12 changed files with 362 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -37,6 +37,7 @@ Thumbs.db
#######################
/build/
/Util/UUID.cpp
/Util/GitDescription.cpp
# Eclipse related files #
#########################

2
.travis.yml
View File

@ -5,7 +5,7 @@ compiler:
# Make sure CMake is installed
install:
- sudo apt-get update >/dev/null
- sudo apt-get -q install build-essential git cmake pkg-config libprotoc-dev libprotobuf7 protobuf-compiler libprotobuf-dev libosmpbf-dev libpng12-dev libbz2-dev libstxxl-dev libstxxl-doc libstxxl1 libxml2-dev libzip-dev libboost-thread-dev libboost-system-dev libboost-regex-dev libboost-filesystem-dev lua5.1 liblua5.1-0-dev libluabind-dev rubygems osmosis
- sudo apt-get -q install build-essential git cmake pkg-config libprotoc-dev libprotobuf7 protobuf-compiler libprotobuf-dev libosmpbf-dev libpng12-dev libbz2-dev libstxxl-dev libstxxl-doc libstxxl1 libxml2-dev libzip-dev libboost-thread-dev libboost-system-dev libboost-regex-dev libboost-filesystem-dev libboost-program-options-dev lua5.1 liblua5.1-0-dev libluabind-dev rubygems osmosis
before_script:
- sudo gem install bundler
- bundle install

9
CMakeLists.txt
View File

@ -3,6 +3,10 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
project(OSRM)
include(FindPackageHandleStandardArgs)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
include(GetGitRevisionDescription)
git_describe(GIT_DESCRIPTION)
TRY_RUN(SHARED_LIBRARY_PATH_TYPE SHARED_LIBRARY_PATH_INFO_COMPILED ${PROJECT_BINARY_DIR}/CMakeTmp ${PROJECT_SOURCE_DIR}/cmake/size.cpp OUTPUT_VARIABLE IS_64_SYSTEM)
if(IS_64_SYSTEM)
message(STATUS "System supports 64 bits.")
@ -25,10 +29,11 @@ add_custom_command(OUTPUT ${CMAKE_SOURCE_DIR}/Util/UUID.cpp UUID.cpp.alwaysbuild
add_custom_target(UUIDConfigure DEPENDS ${CMAKE_SOURCE_DIR}/Util/UUID.cpp )
set(BOOST_COMPONENTS filesystem regex system thread)
set(BOOST_COMPONENTS filesystem regex system thread program_options)
configure_file(Util/GitDescription.cpp.in ${CMAKE_SOURCE_DIR}/Util/GitDescription.cpp)
file(GLOB ExtractorGlob Extractor/*.cpp)
set(ExtractorSources extractor.cpp ${ExtractorGlob})
set(ExtractorSources extractor.cpp ${ExtractorGlob} Util/GitDescription.cpp)
add_executable(osrm-extract ${ExtractorSources} )
file(GLOB PrepareGlob Contractor/*.cpp)

22
Util/GitDescription.cpp.in Normal file
View File

@ -0,0 +1,22 @@
/*
open source routing machine
Copyright (C) Dennis Luxen, others 2010
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
or see http://www.gnu.org/licenses/agpl.txt.
*/
#define GIT_DESCRIPTION "${GIT_DESCRIPTION}"
char g_GIT_DESCRIPTION[] = GIT_DESCRIPTION;

21
Util/GitDescription.h Normal file
View File

@ -0,0 +1,21 @@
/*
open source routing machine
Copyright (C) Dennis Luxen, others 2010
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
or see http://www.gnu.org/licenses/agpl.txt.
*/
extern char g_GIT_DESCRIPTION[];

56
Util/ProgramOptions.h Normal file
View File

@ -0,0 +1,56 @@
/*
open source routing machine
Copyright (C) Dennis Luxen, others 2010
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
or see http://www.gnu.org/licenses/agpl.txt.
Custom validators for use with boost::program_options.
*/
#include <boost/program_options.hpp>
namespace boost {
namespace program_options {
// Class for reporting missing files
class missing_file_error : public boost::program_options::invalid_option_value {
public:
missing_file_error(const std::string& path) : invalid_option_value(path), path(path) {};
~missing_file_error() throw() {};
const char* what() const throw() {
std::string err = get_option_name() + " file not found: " + path;
return err.c_str();
};
protected:
std::string path;
};
}
namespace filesystem {
// Validator for boost::filesystem::path, that verifies that the file exists.
// The validate() function must be defined in the same namespace as the target type,
// (boost::filesystem::path in this case), otherwise it will not be called
void validate(boost::any& v, const std::vector<std::string>& values, boost::filesystem::path*, int) {
boost::program_options::validators::check_first_occurrence(v); // avoid previous assignments
const std::string& input_string = boost::program_options::validators::get_single_string(values); // avoid multiple assignments
if(boost::filesystem::is_regular_file(input_string)) {
v = boost::any(boost::filesystem::path(input_string));
} else {
throw boost::program_options::missing_file_error(input_string);
}
}
}
}

123
cmake/GetGitRevisionDescription.cmake Normal file
View File

@ -0,0 +1,123 @@
# - Returns a version string from Git
#
# These functions force a re-configure on each git commit so that you can
# trust the values of the variables in your build system.
#
# get_git_head_revision(<refspecvar> <hashvar> [<additional arguments to git describe> ...])
#
# Returns the refspec and sha hash of the current head revision
#
# git_describe(<var> [<additional arguments to git describe> ...])
#
# Returns the results of git describe on the source tree, and adjusting
# the output so that it tests false if an error occurs.
#
# git_get_exact_tag(<var> [<additional arguments to git describe> ...])
#
# Returns the results of git describe --exact-match on the source tree,
# and adjusting the output so that it tests false if there was no exact
# matching tag.
#
# Requires CMake 2.6 or newer (uses the 'function' command)
#
# Original Author:
# 2009-2010 Ryan Pavlik <rpavlik@iastate.edu> <abiryan@ryand.net>
# http://academic.cleardefinition.com
# Iowa State University HCI Graduate Program/VRAC
#
# Copyright Iowa State University 2009-2010.
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE_1_0.txt or copy at
# http://www.boost.org/LICENSE_1_0.txt)
if(__get_git_revision_description)
return()
endif()
set(__get_git_revision_description YES)
# We must run the following at "include" time, not at function call time,
# to find the path to this module rather than the path to a calling list file
get_filename_component(_gitdescmoddir ${CMAKE_CURRENT_LIST_FILE} PATH)
function(get_git_head_revision _refspecvar _hashvar)
set(GIT_PARENT_DIR "${CMAKE_SOURCE_DIR}")
set(GIT_DIR "${GIT_PARENT_DIR}/.git")
while(NOT EXISTS "${GIT_DIR}") # .git dir not found, search parent directories
set(GIT_PREVIOUS_PARENT "${GIT_PARENT_DIR}")
get_filename_component(GIT_PARENT_DIR ${GIT_PARENT_DIR} PATH)
if(GIT_PARENT_DIR STREQUAL GIT_PREVIOUS_PARENT)
# We have reached the root directory, we are not in git
set(${_refspecvar} "GITDIR-NOTFOUND" PARENT_SCOPE)
set(${_hashvar} "GITDIR-NOTFOUND" PARENT_SCOPE)
return()
endif()
set(GIT_DIR "${GIT_PARENT_DIR}/.git")
endwhile()
set(GIT_DATA "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/git-data")
if(NOT EXISTS "${GIT_DATA}")
file(MAKE_DIRECTORY "${GIT_DATA}")
endif()
if(NOT EXISTS "${GIT_DIR}/HEAD")
return()
endif()
set(HEAD_FILE "${GIT_DATA}/HEAD")
configure_file("${GIT_DIR}/HEAD" "${HEAD_FILE}" COPYONLY)
configure_file("${_gitdescmoddir}/GetGitRevisionDescription.cmake.in"
"${GIT_DATA}/grabRef.cmake"
@ONLY)
include("${GIT_DATA}/grabRef.cmake")
set(${_refspecvar} "${HEAD_REF}" PARENT_SCOPE)
set(${_hashvar} "${HEAD_HASH}" PARENT_SCOPE)
endfunction()
function(git_describe _var)
if(NOT GIT_FOUND)
find_package(Git QUIET)
endif()
get_git_head_revision(refspec hash)
if(NOT GIT_FOUND)
set(${_var} "GIT-NOTFOUND" PARENT_SCOPE)
return()
endif()
if(NOT hash)
set(${_var} "HEAD-HASH-NOTFOUND" PARENT_SCOPE)
return()
endif()
# TODO sanitize
#if((${ARGN}" MATCHES "&&") OR
# (ARGN MATCHES "||") OR
# (ARGN MATCHES "\\;"))
# message("Please report the following error to the project!")
# message(FATAL_ERROR "Looks like someone's doing something nefarious with git_describe! Passed arguments ${ARGN}")
#endif()
#message(STATUS "Arguments to execute_process: ${ARGN}")
execute_process(COMMAND
"${GIT_EXECUTABLE}"
describe
${hash}
${ARGN}
WORKING_DIRECTORY
"${CMAKE_SOURCE_DIR}"
RESULT_VARIABLE
res
OUTPUT_VARIABLE
out
ERROR_QUIET
OUTPUT_STRIP_TRAILING_WHITESPACE)
if(NOT res EQUAL 0)
set(out "${out}-${res}-NOTFOUND")
endif()
set(${_var} "${out}" PARENT_SCOPE)
endfunction()
function(git_get_exact_tag _var)
git_describe(out --exact-match ${ARGN})
set(${_var} "${out}" PARENT_SCOPE)
endfunction()

38
cmake/GetGitRevisionDescription.cmake.in Normal file
View File

@ -0,0 +1,38 @@
#
# Internal file for GetGitRevisionDescription.cmake
#
# Requires CMake 2.6 or newer (uses the 'function' command)
#
# Original Author:
# 2009-2010 Ryan Pavlik <rpavlik@iastate.edu> <abiryan@ryand.net>
# http://academic.cleardefinition.com
# Iowa State University HCI Graduate Program/VRAC
#
# Copyright Iowa State University 2009-2010.
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE_1_0.txt or copy at
# http://www.boost.org/LICENSE_1_0.txt)
set(HEAD_HASH)
file(READ "@HEAD_FILE@" HEAD_CONTENTS LIMIT 1024)
string(STRIP "${HEAD_CONTENTS}" HEAD_CONTENTS)
if(HEAD_CONTENTS MATCHES "ref")
# named branch
string(REPLACE "ref: " "" HEAD_REF "${HEAD_CONTENTS}")
if(EXISTS "@GIT_DIR@/${HEAD_REF}")
configure_file("@GIT_DIR@/${HEAD_REF}" "@GIT_DATA@/head-ref" COPYONLY)
elseif(EXISTS "@GIT_DIR@/logs/${HEAD_REF}")
configure_file("@GIT_DIR@/logs/${HEAD_REF}" "@GIT_DATA@/head-ref" COPYONLY)
set(HEAD_HASH "${HEAD_REF}")
endif()
else()
# detached HEAD
configure_file("@GIT_DIR@/HEAD" "@GIT_DATA@/head-ref" COPYONLY)
endif()
if(NOT HEAD_HASH)
file(READ "@GIT_DATA@/head-ref" HEAD_HASH LIMIT 1024)
string(STRIP "${HEAD_HASH}" HEAD_HASH)
endif()

117
extractor.cpp
View File

@ -23,11 +23,11 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "Extractor/ScriptingEnvironment.h"
#include "Extractor/PBFParser.h"
#include "Extractor/XMLParser.h"
#include "Util/IniFile.h"
#include "Util/InputFileUtil.h"
#include "Util/GitDescription.h"
#include "Util/MachineInfo.h"
#include "Util/OpenMPWrapper.h"
#include "Util/OSRMException.h"
#include "Util/ProgramOptions.h"
#include "Util/SimpleLogger.h"
#include "Util/StringUtil.h"
#include "Util/UUID.h"
@ -46,32 +46,93 @@ int main (int argc, char *argv[]) {
try {
LogPolicy::GetInstance().Unmute();
double startup_time = get_timestamp();
if(argc < 2) {
SimpleLogger().Write(logWARNING) <<
"usage: \n" <<
argv[0] <<
" <file.osm/.osm.bz2/.osm.pbf> [<profile.lua>]";
boost::filesystem::path config_file_path, input_path, profile_path;
int requested_num_threads;
// declare a group of options that will be allowed only on command line
boost::program_options::options_description generic_options("Options");
generic_options.add_options()
("version,v", "Show version")
("help,h", "Show this help message")
("config,c", boost::program_options::value<boost::filesystem::path>(&config_file_path)->default_value("extractor.ini"),
"Path to a configuration file.");
// declare a group of options that will be allowed both on command line and in config file
boost::program_options::options_description config_options("Configuration");
config_options.add_options()
("profile,p", boost::program_options::value<boost::filesystem::path>(&profile_path)->default_value("profile.lua"),
"Path to LUA routing profile")
("threads,t", boost::program_options::value<int>(&requested_num_threads)->default_value(10),
"Number of threads to use");
// hidden options, will be allowed both on command line and in config file, but will not be shown to the user
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()
("input,i", boost::program_options::value<boost::filesystem::path>(&input_path),
"Input file in .osm, .osm.bz2 or .osm.pbf format");
// positional option
boost::program_options::positional_options_description positional_options;
positional_options.add("input", 1);
// combine above options for parsing
boost::program_options::options_description cmdline_options;
cmdline_options.add(generic_options).add(config_options).add(hidden_options);
boost::program_options::options_description config_file_options;
config_file_options.add(config_options).add(hidden_options);
boost::program_options::options_description visible_options(boost::filesystem::basename(argv[0]) + " <input.osm/.osm.bz2/.osm.pbf> [<profile.lua>]");
visible_options.add(generic_options).add(config_options);
// parse command line options
boost::program_options::variables_map option_variables;
boost::program_options::store(boost::program_options::command_line_parser(argc, argv).
options(cmdline_options).positional(positional_options).run(), option_variables);
if(option_variables.count("version")) {
SimpleLogger().Write() << g_GIT_DESCRIPTION;
return 0;
}
if(option_variables.count("help")) {
SimpleLogger().Write() << visible_options;
return 0;
}
boost::program_options::notify(option_variables);
// parse config file
if(boost::filesystem::is_regular_file(config_file_path)) {
std::ifstream ifs(config_file_path.c_str());
SimpleLogger().Write() << "Reading options from: " << config_file_path.filename().string();
boost::program_options::store(parse_config_file(ifs, config_file_options), option_variables);
boost::program_options::notify(option_variables);
}
if(!option_variables.count("input")) {
SimpleLogger().Write(logWARNING) << "No input file specified";
return -1;
}
/*** Setup Scripting Environment ***/
ScriptingEnvironment scriptingEnvironment((argc > 2 ? argv[2] : "profile.lua"));
unsigned number_of_threads = omp_get_num_procs();
if(testDataFile("extractor.ini")) {
IniFile extractorConfig("extractor.ini");
unsigned rawNumber = stringToInt(extractorConfig.GetParameter("Threads"));
if( rawNumber != 0 && rawNumber <= number_of_threads) {
number_of_threads = rawNumber;
}
if(1 > requested_num_threads) {
SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger";
return -1;
}
omp_set_num_threads(number_of_threads);
SimpleLogger().Write() << "extracting data from input file " << argv[1];
SimpleLogger().Write() << "Input file: " << input_path.filename().string();
SimpleLogger().Write() << "Profile: " << profile_path.filename().string();
SimpleLogger().Write() << "Threads: " << requested_num_threads;
/*** Setup Scripting Environment ***/
ScriptingEnvironment scriptingEnvironment(profile_path.c_str());
omp_set_num_threads( std::min( omp_get_num_procs(), requested_num_threads) );
bool file_has_pbf_format(false);
std::string output_file_name(argv[1]);
std::string restrictionsFileName(argv[1]);
std::string output_file_name(input_path.c_str());
std::string restrictionsFileName(input_path.c_str());
std::string::size_type pos = output_file_name.find(".osm.bz2");
if(pos==std::string::npos) {
pos = output_file_name.find(".osm.pbf");
@ -112,9 +173,9 @@ int main (int argc, char *argv[]) {
extractCallBacks = new ExtractorCallbacks(&externalMemory, &stringMap);
BaseParser* parser;
if(file_has_pbf_format) {
parser = new PBFParser(argv[1], extractCallBacks, scriptingEnvironment);
parser = new PBFParser(input_path.c_str(), extractCallBacks, scriptingEnvironment);
} else {
parser = new XMLParser(argv[1], extractCallBacks, scriptingEnvironment);
parser = new XMLParser(input_path.c_str(), extractCallBacks, scriptingEnvironment);
}
if(!parser->ReadHeader()) {
@ -141,8 +202,14 @@ int main (int argc, char *argv[]) {
" " <<
restrictionsFileName <<
std::endl;
} catch(boost::program_options::too_many_positional_options_error& e) {
SimpleLogger().Write(logWARNING) << "Only one input file can be specified";
return -1;
} catch(boost::program_options::error& e) {
SimpleLogger().Write(logWARNING) << e.what();
return -1;
} catch(std::exception & e) {
SimpleLogger().Write(logWARNING) << "unhandled exception: " << e.what();
SimpleLogger().Write(logWARNING) << "Unhandled exception: " << e.what();
return -1;
}
return 0;

2
extractor.ini
View File

@ -1,2 +0,0 @@
Memory = 2
Threads = 10

2
features/support/data.rb
View File

@ -250,7 +250,7 @@ def reprocess
unless extracted?
log_preprocess_info
log "== Extracting #{@osm_file}.osm...", :preprocess
unless system "#{BIN_PATH}/osrm-extract #{@osm_file}.osm#{'.pbf' if use_pbf} 1>>#{PREPROCESS_LOG_FILE} 2>>#{PREPROCESS_LOG_FILE} #{PROFILES_PATH}/#{@profile}.lua"
unless system "#{BIN_PATH}/osrm-extract #{@osm_file}.osm#{'.pbf' if use_pbf} --profile #{PROFILES_PATH}/#{@profile}.lua 1>>#{PREPROCESS_LOG_FILE} 2>>#{PREPROCESS_LOG_FILE}"
log "*** Exited with code #{$?.exitstatus}.", :preprocess
raise ExtractError.new $?.exitstatus, "osrm-extract exited with code #{$?.exitstatus}."
end

1
test/extractor.ini
View File

@ -1 +0,0 @@
Memory = 1