diff --git a/.gitignore b/.gitignore index dbee9e07a..d87653237 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ Thumbs.db ####################### /build/ /Util/UUID.cpp +/Util/GitDescription.cpp # Eclipse related files # ######################### diff --git a/.travis.yml b/.travis.yml index 4894bdf4d..d28518fb8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ compiler: # Make sure CMake is installed install: - sudo apt-get update >/dev/null - - sudo apt-get -q install build-essential git cmake pkg-config libprotoc-dev libprotobuf7 protobuf-compiler libprotobuf-dev libosmpbf-dev libpng12-dev libbz2-dev libstxxl-dev libstxxl-doc libstxxl1 libxml2-dev libzip-dev libboost-thread-dev libboost-system-dev libboost-regex-dev libboost-filesystem-dev lua5.1 liblua5.1-0-dev libluabind-dev rubygems osmosis + - sudo apt-get -q install build-essential git cmake pkg-config libprotoc-dev libprotobuf7 protobuf-compiler libprotobuf-dev libosmpbf-dev libpng12-dev libbz2-dev libstxxl-dev libstxxl-doc libstxxl1 libxml2-dev libzip-dev libboost-thread-dev libboost-system-dev libboost-regex-dev libboost-filesystem-dev libboost-program-options-dev lua5.1 liblua5.1-0-dev libluabind-dev rubygems osmosis before_script: - sudo gem install bundler - bundle install diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d047faa6..67031eafb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,10 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) project(OSRM) include(FindPackageHandleStandardArgs) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") +include(GetGitRevisionDescription) +git_describe(GIT_DESCRIPTION) + TRY_RUN(SHARED_LIBRARY_PATH_TYPE SHARED_LIBRARY_PATH_INFO_COMPILED ${PROJECT_BINARY_DIR}/CMakeTmp ${PROJECT_SOURCE_DIR}/cmake/size.cpp OUTPUT_VARIABLE IS_64_SYSTEM) if(IS_64_SYSTEM) message(STATUS "System supports 64 bits.") @@ -25,10 +29,11 @@ add_custom_command(OUTPUT ${CMAKE_SOURCE_DIR}/Util/UUID.cpp UUID.cpp.alwaysbuild add_custom_target(UUIDConfigure DEPENDS ${CMAKE_SOURCE_DIR}/Util/UUID.cpp ) -set(BOOST_COMPONENTS filesystem regex system thread) +set(BOOST_COMPONENTS filesystem regex system thread program_options) +configure_file(Util/GitDescription.cpp.in ${CMAKE_SOURCE_DIR}/Util/GitDescription.cpp) file(GLOB ExtractorGlob Extractor/*.cpp) -set(ExtractorSources extractor.cpp ${ExtractorGlob}) +set(ExtractorSources extractor.cpp ${ExtractorGlob} Util/GitDescription.cpp) add_executable(osrm-extract ${ExtractorSources} ) file(GLOB PrepareGlob Contractor/*.cpp) diff --git a/Util/GitDescription.cpp.in b/Util/GitDescription.cpp.in new file mode 100644 index 000000000..c89beb093 --- /dev/null +++ b/Util/GitDescription.cpp.in @@ -0,0 +1,22 @@ +/* + open source routing machine + Copyright (C) Dennis Luxen, others 2010 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU AFFERO General Public License as published by +the Free Software Foundation; either version 3 of the License, or +any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +or see http://www.gnu.org/licenses/agpl.txt. +*/ + +#define GIT_DESCRIPTION "${GIT_DESCRIPTION}" +char g_GIT_DESCRIPTION[] = GIT_DESCRIPTION; \ No newline at end of file diff --git a/Util/GitDescription.h b/Util/GitDescription.h new file mode 100644 index 000000000..4df06f338 --- /dev/null +++ b/Util/GitDescription.h @@ -0,0 +1,21 @@ +/* + open source routing machine + Copyright (C) Dennis Luxen, others 2010 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU AFFERO General Public License as published by +the Free Software Foundation; either version 3 of the License, or +any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +or see http://www.gnu.org/licenses/agpl.txt. +*/ + +extern char g_GIT_DESCRIPTION[]; \ No newline at end of file diff --git a/Util/ProgramOptions.h b/Util/ProgramOptions.h new file mode 100644 index 000000000..c08e1b185 --- /dev/null +++ b/Util/ProgramOptions.h @@ -0,0 +1,56 @@ +/* + open source routing machine + Copyright (C) Dennis Luxen, others 2010 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU AFFERO General Public License as published by +the Free Software Foundation; either version 3 of the License, or +any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +or see http://www.gnu.org/licenses/agpl.txt. + + +Custom validators for use with boost::program_options. +*/ + +#include + +namespace boost { + namespace program_options { + // Class for reporting missing files + class missing_file_error : public boost::program_options::invalid_option_value { + public: + missing_file_error(const std::string& path) : invalid_option_value(path), path(path) {}; + ~missing_file_error() throw() {}; + const char* what() const throw() { + std::string err = get_option_name() + " file not found: " + path; + return err.c_str(); + }; + protected: + std::string path; + }; + } + + namespace filesystem { + // Validator for boost::filesystem::path, that verifies that the file exists. + // The validate() function must be defined in the same namespace as the target type, + // (boost::filesystem::path in this case), otherwise it will not be called + void validate(boost::any& v, const std::vector& values, boost::filesystem::path*, int) { + boost::program_options::validators::check_first_occurrence(v); // avoid previous assignments + const std::string& input_string = boost::program_options::validators::get_single_string(values); // avoid multiple assignments + if(boost::filesystem::is_regular_file(input_string)) { + v = boost::any(boost::filesystem::path(input_string)); + } else { + throw boost::program_options::missing_file_error(input_string); + } + } + } +} diff --git a/cmake/GetGitRevisionDescription.cmake b/cmake/GetGitRevisionDescription.cmake new file mode 100644 index 000000000..1bf023008 --- /dev/null +++ b/cmake/GetGitRevisionDescription.cmake @@ -0,0 +1,123 @@ +# - Returns a version string from Git +# +# These functions force a re-configure on each git commit so that you can +# trust the values of the variables in your build system. +# +# get_git_head_revision( [ ...]) +# +# Returns the refspec and sha hash of the current head revision +# +# git_describe( [ ...]) +# +# Returns the results of git describe on the source tree, and adjusting +# the output so that it tests false if an error occurs. +# +# git_get_exact_tag( [ ...]) +# +# Returns the results of git describe --exact-match on the source tree, +# and adjusting the output so that it tests false if there was no exact +# matching tag. +# +# Requires CMake 2.6 or newer (uses the 'function' command) +# +# Original Author: +# 2009-2010 Ryan Pavlik +# http://academic.cleardefinition.com +# Iowa State University HCI Graduate Program/VRAC +# +# Copyright Iowa State University 2009-2010. +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) + +if(__get_git_revision_description) + return() +endif() +set(__get_git_revision_description YES) + +# We must run the following at "include" time, not at function call time, +# to find the path to this module rather than the path to a calling list file +get_filename_component(_gitdescmoddir ${CMAKE_CURRENT_LIST_FILE} PATH) + +function(get_git_head_revision _refspecvar _hashvar) + set(GIT_PARENT_DIR "${CMAKE_SOURCE_DIR}") + set(GIT_DIR "${GIT_PARENT_DIR}/.git") + while(NOT EXISTS "${GIT_DIR}") # .git dir not found, search parent directories + set(GIT_PREVIOUS_PARENT "${GIT_PARENT_DIR}") + get_filename_component(GIT_PARENT_DIR ${GIT_PARENT_DIR} PATH) + if(GIT_PARENT_DIR STREQUAL GIT_PREVIOUS_PARENT) + # We have reached the root directory, we are not in git + set(${_refspecvar} "GITDIR-NOTFOUND" PARENT_SCOPE) + set(${_hashvar} "GITDIR-NOTFOUND" PARENT_SCOPE) + return() + endif() + set(GIT_DIR "${GIT_PARENT_DIR}/.git") + endwhile() + set(GIT_DATA "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/git-data") + if(NOT EXISTS "${GIT_DATA}") + file(MAKE_DIRECTORY "${GIT_DATA}") + endif() + + if(NOT EXISTS "${GIT_DIR}/HEAD") + return() + endif() + set(HEAD_FILE "${GIT_DATA}/HEAD") + configure_file("${GIT_DIR}/HEAD" "${HEAD_FILE}" COPYONLY) + + configure_file("${_gitdescmoddir}/GetGitRevisionDescription.cmake.in" + "${GIT_DATA}/grabRef.cmake" + @ONLY) + include("${GIT_DATA}/grabRef.cmake") + + set(${_refspecvar} "${HEAD_REF}" PARENT_SCOPE) + set(${_hashvar} "${HEAD_HASH}" PARENT_SCOPE) +endfunction() + +function(git_describe _var) + if(NOT GIT_FOUND) + find_package(Git QUIET) + endif() + get_git_head_revision(refspec hash) + if(NOT GIT_FOUND) + set(${_var} "GIT-NOTFOUND" PARENT_SCOPE) + return() + endif() + if(NOT hash) + set(${_var} "HEAD-HASH-NOTFOUND" PARENT_SCOPE) + return() + endif() + + # TODO sanitize + #if((${ARGN}" MATCHES "&&") OR + # (ARGN MATCHES "||") OR + # (ARGN MATCHES "\\;")) + # message("Please report the following error to the project!") + # message(FATAL_ERROR "Looks like someone's doing something nefarious with git_describe! Passed arguments ${ARGN}") + #endif() + + #message(STATUS "Arguments to execute_process: ${ARGN}") + + execute_process(COMMAND + "${GIT_EXECUTABLE}" + describe + ${hash} + ${ARGN} + WORKING_DIRECTORY + "${CMAKE_SOURCE_DIR}" + RESULT_VARIABLE + res + OUTPUT_VARIABLE + out + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(NOT res EQUAL 0) + set(out "${out}-${res}-NOTFOUND") + endif() + + set(${_var} "${out}" PARENT_SCOPE) +endfunction() + +function(git_get_exact_tag _var) + git_describe(out --exact-match ${ARGN}) + set(${_var} "${out}" PARENT_SCOPE) +endfunction() diff --git a/cmake/GetGitRevisionDescription.cmake.in b/cmake/GetGitRevisionDescription.cmake.in new file mode 100644 index 000000000..888ce13aa --- /dev/null +++ b/cmake/GetGitRevisionDescription.cmake.in @@ -0,0 +1,38 @@ +# +# Internal file for GetGitRevisionDescription.cmake +# +# Requires CMake 2.6 or newer (uses the 'function' command) +# +# Original Author: +# 2009-2010 Ryan Pavlik +# http://academic.cleardefinition.com +# Iowa State University HCI Graduate Program/VRAC +# +# Copyright Iowa State University 2009-2010. +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) + +set(HEAD_HASH) + +file(READ "@HEAD_FILE@" HEAD_CONTENTS LIMIT 1024) + +string(STRIP "${HEAD_CONTENTS}" HEAD_CONTENTS) +if(HEAD_CONTENTS MATCHES "ref") + # named branch + string(REPLACE "ref: " "" HEAD_REF "${HEAD_CONTENTS}") + if(EXISTS "@GIT_DIR@/${HEAD_REF}") + configure_file("@GIT_DIR@/${HEAD_REF}" "@GIT_DATA@/head-ref" COPYONLY) + elseif(EXISTS "@GIT_DIR@/logs/${HEAD_REF}") + configure_file("@GIT_DIR@/logs/${HEAD_REF}" "@GIT_DATA@/head-ref" COPYONLY) + set(HEAD_HASH "${HEAD_REF}") + endif() +else() + # detached HEAD + configure_file("@GIT_DIR@/HEAD" "@GIT_DATA@/head-ref" COPYONLY) +endif() + +if(NOT HEAD_HASH) + file(READ "@GIT_DATA@/head-ref" HEAD_HASH LIMIT 1024) + string(STRIP "${HEAD_HASH}" HEAD_HASH) +endif() diff --git a/extractor.cpp b/extractor.cpp index d39c00ab6..4a380e18b 100644 --- a/extractor.cpp +++ b/extractor.cpp @@ -23,11 +23,11 @@ or see http://www.gnu.org/licenses/agpl.txt. #include "Extractor/ScriptingEnvironment.h" #include "Extractor/PBFParser.h" #include "Extractor/XMLParser.h" -#include "Util/IniFile.h" -#include "Util/InputFileUtil.h" +#include "Util/GitDescription.h" #include "Util/MachineInfo.h" #include "Util/OpenMPWrapper.h" #include "Util/OSRMException.h" +#include "Util/ProgramOptions.h" #include "Util/SimpleLogger.h" #include "Util/StringUtil.h" #include "Util/UUID.h" @@ -46,32 +46,93 @@ int main (int argc, char *argv[]) { try { LogPolicy::GetInstance().Unmute(); double startup_time = get_timestamp(); - if(argc < 2) { - SimpleLogger().Write(logWARNING) << - "usage: \n" << - argv[0] << - " []"; + + boost::filesystem::path config_file_path, input_path, profile_path; + int requested_num_threads; + + // declare a group of options that will be allowed only on command line + boost::program_options::options_description generic_options("Options"); + generic_options.add_options() + ("version,v", "Show version") + ("help,h", "Show this help message") + ("config,c", boost::program_options::value(&config_file_path)->default_value("extractor.ini"), + "Path to a configuration file."); + + // declare a group of options that will be allowed both on command line and in config file + boost::program_options::options_description config_options("Configuration"); + config_options.add_options() + ("profile,p", boost::program_options::value(&profile_path)->default_value("profile.lua"), + "Path to LUA routing profile") + ("threads,t", boost::program_options::value(&requested_num_threads)->default_value(10), + "Number of threads to use"); + + // hidden options, will be allowed both on command line and in config file, but will not be shown to the user + boost::program_options::options_description hidden_options("Hidden options"); + hidden_options.add_options() + ("input,i", boost::program_options::value(&input_path), + "Input file in .osm, .osm.bz2 or .osm.pbf format"); + + // positional option + boost::program_options::positional_options_description positional_options; + positional_options.add("input", 1); + + // combine above options for parsing + boost::program_options::options_description cmdline_options; + cmdline_options.add(generic_options).add(config_options).add(hidden_options); + + boost::program_options::options_description config_file_options; + config_file_options.add(config_options).add(hidden_options); + + boost::program_options::options_description visible_options(boost::filesystem::basename(argv[0]) + " []"); + visible_options.add(generic_options).add(config_options); + + // parse command line options + boost::program_options::variables_map option_variables; + boost::program_options::store(boost::program_options::command_line_parser(argc, argv). + options(cmdline_options).positional(positional_options).run(), option_variables); + + if(option_variables.count("version")) { + SimpleLogger().Write() << g_GIT_DESCRIPTION; + return 0; + } + + if(option_variables.count("help")) { + SimpleLogger().Write() << visible_options; + return 0; + } + + boost::program_options::notify(option_variables); + + // parse config file + if(boost::filesystem::is_regular_file(config_file_path)) { + std::ifstream ifs(config_file_path.c_str()); + SimpleLogger().Write() << "Reading options from: " << config_file_path.filename().string(); + boost::program_options::store(parse_config_file(ifs, config_file_options), option_variables); + boost::program_options::notify(option_variables); + } + + if(!option_variables.count("input")) { + SimpleLogger().Write(logWARNING) << "No input file specified"; return -1; } - /*** Setup Scripting Environment ***/ - ScriptingEnvironment scriptingEnvironment((argc > 2 ? argv[2] : "profile.lua")); - - unsigned number_of_threads = omp_get_num_procs(); - - if(testDataFile("extractor.ini")) { - IniFile extractorConfig("extractor.ini"); - unsigned rawNumber = stringToInt(extractorConfig.GetParameter("Threads")); - if( rawNumber != 0 && rawNumber <= number_of_threads) { - number_of_threads = rawNumber; - } + if(1 > requested_num_threads) { + SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger"; + return -1; } - omp_set_num_threads(number_of_threads); - SimpleLogger().Write() << "extracting data from input file " << argv[1]; + SimpleLogger().Write() << "Input file: " << input_path.filename().string(); + SimpleLogger().Write() << "Profile: " << profile_path.filename().string(); + SimpleLogger().Write() << "Threads: " << requested_num_threads; + + /*** Setup Scripting Environment ***/ + ScriptingEnvironment scriptingEnvironment(profile_path.c_str()); + + omp_set_num_threads( std::min( omp_get_num_procs(), requested_num_threads) ); + bool file_has_pbf_format(false); - std::string output_file_name(argv[1]); - std::string restrictionsFileName(argv[1]); + std::string output_file_name(input_path.c_str()); + std::string restrictionsFileName(input_path.c_str()); std::string::size_type pos = output_file_name.find(".osm.bz2"); if(pos==std::string::npos) { pos = output_file_name.find(".osm.pbf"); @@ -112,9 +173,9 @@ int main (int argc, char *argv[]) { extractCallBacks = new ExtractorCallbacks(&externalMemory, &stringMap); BaseParser* parser; if(file_has_pbf_format) { - parser = new PBFParser(argv[1], extractCallBacks, scriptingEnvironment); + parser = new PBFParser(input_path.c_str(), extractCallBacks, scriptingEnvironment); } else { - parser = new XMLParser(argv[1], extractCallBacks, scriptingEnvironment); + parser = new XMLParser(input_path.c_str(), extractCallBacks, scriptingEnvironment); } if(!parser->ReadHeader()) { @@ -141,8 +202,14 @@ int main (int argc, char *argv[]) { " " << restrictionsFileName << std::endl; + } catch(boost::program_options::too_many_positional_options_error& e) { + SimpleLogger().Write(logWARNING) << "Only one input file can be specified"; + return -1; + } catch(boost::program_options::error& e) { + SimpleLogger().Write(logWARNING) << e.what(); + return -1; } catch(std::exception & e) { - SimpleLogger().Write(logWARNING) << "unhandled exception: " << e.what(); + SimpleLogger().Write(logWARNING) << "Unhandled exception: " << e.what(); return -1; } return 0; diff --git a/extractor.ini b/extractor.ini deleted file mode 100644 index 8dd11de65..000000000 --- a/extractor.ini +++ /dev/null @@ -1,2 +0,0 @@ -Memory = 2 -Threads = 10 diff --git a/features/support/data.rb b/features/support/data.rb index dbb0398b8..9b04f083e 100644 --- a/features/support/data.rb +++ b/features/support/data.rb @@ -250,7 +250,7 @@ def reprocess unless extracted? log_preprocess_info log "== Extracting #{@osm_file}.osm...", :preprocess - unless system "#{BIN_PATH}/osrm-extract #{@osm_file}.osm#{'.pbf' if use_pbf} 1>>#{PREPROCESS_LOG_FILE} 2>>#{PREPROCESS_LOG_FILE} #{PROFILES_PATH}/#{@profile}.lua" + unless system "#{BIN_PATH}/osrm-extract #{@osm_file}.osm#{'.pbf' if use_pbf} --profile #{PROFILES_PATH}/#{@profile}.lua 1>>#{PREPROCESS_LOG_FILE} 2>>#{PREPROCESS_LOG_FILE}" log "*** Exited with code #{$?.exitstatus}.", :preprocess raise ExtractError.new $?.exitstatus, "osrm-extract exited with code #{$?.exitstatus}." end diff --git a/test/extractor.ini b/test/extractor.ini deleted file mode 100644 index eca8b179d..000000000 --- a/test/extractor.ini +++ /dev/null @@ -1 +0,0 @@ -Memory = 1