use boost program_options in extractor

.gitignore

@@ -37,6 +37,7 @@ Thumbs.db
 #######################
 /build/
 /Util/UUID.cpp
+/Util/GitDescription.cpp
 
 # Eclipse related files #
 #########################

.travis.yml

@@ -5,7 +5,7 @@ compiler:
 # Make sure CMake is installed
 install:
  - sudo apt-get update >/dev/null
- - sudo apt-get -q install build-essential git cmake pkg-config libprotoc-dev libprotobuf7 protobuf-compiler libprotobuf-dev libosmpbf-dev libpng12-dev libbz2-dev libstxxl-dev libstxxl-doc libstxxl1 libxml2-dev libzip-dev libboost-thread-dev libboost-system-dev libboost-regex-dev libboost-filesystem-dev lua5.1 liblua5.1-0-dev libluabind-dev rubygems osmosis
+ - sudo apt-get -q install build-essential git cmake pkg-config libprotoc-dev libprotobuf7 protobuf-compiler libprotobuf-dev libosmpbf-dev libpng12-dev libbz2-dev libstxxl-dev libstxxl-doc libstxxl1 libxml2-dev libzip-dev libboost-thread-dev libboost-system-dev libboost-regex-dev libboost-filesystem-dev libboost-program-options-dev lua5.1 liblua5.1-0-dev libluabind-dev rubygems osmosis
 before_script:
  - sudo gem install bundler
  - bundle install

CMakeLists.txt

@@ -3,6 +3,10 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 project(OSRM)
 include(FindPackageHandleStandardArgs)
 
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
+include(GetGitRevisionDescription)
+git_describe(GIT_DESCRIPTION)
+
 TRY_RUN(SHARED_LIBRARY_PATH_TYPE SHARED_LIBRARY_PATH_INFO_COMPILED ${PROJECT_BINARY_DIR}/CMakeTmp ${PROJECT_SOURCE_DIR}/cmake/size.cpp OUTPUT_VARIABLE IS_64_SYSTEM)
 if(IS_64_SYSTEM)
 	message(STATUS "System supports 64 bits.")
@@ -25,10 +29,11 @@ add_custom_command(OUTPUT ${CMAKE_SOURCE_DIR}/Util/UUID.cpp UUID.cpp.alwaysbuild
 
 add_custom_target(UUIDConfigure DEPENDS ${CMAKE_SOURCE_DIR}/Util/UUID.cpp )
 
-set(BOOST_COMPONENTS filesystem regex system thread)
+set(BOOST_COMPONENTS filesystem regex system thread program_options)
 
+configure_file(Util/GitDescription.cpp.in ${CMAKE_SOURCE_DIR}/Util/GitDescription.cpp)
 file(GLOB ExtractorGlob Extractor/*.cpp)
-set(ExtractorSources extractor.cpp ${ExtractorGlob})
+set(ExtractorSources extractor.cpp ${ExtractorGlob} Util/GitDescription.cpp)
 add_executable(osrm-extract ${ExtractorSources} )
 
 file(GLOB PrepareGlob Contractor/*.cpp)

Util/GitDescription.cpp.in

@@ -0,0 +1,22 @@
+/*
+    open source routing machine
+    Copyright (C) Dennis Luxen, others 2010
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU AFFERO General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+or see http://www.gnu.org/licenses/agpl.txt.
+*/
+
+#define GIT_DESCRIPTION "${GIT_DESCRIPTION}"
+char g_GIT_DESCRIPTION[] = GIT_DESCRIPTION;

Util/GitDescription.h

@@ -0,0 +1,21 @@
+/*
+    open source routing machine
+    Copyright (C) Dennis Luxen, others 2010
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU AFFERO General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+or see http://www.gnu.org/licenses/agpl.txt.
+*/
+
+extern char g_GIT_DESCRIPTION[];

Util/ProgramOptions.h

@@ -0,0 +1,56 @@
+/*
+    open source routing machine
+    Copyright (C) Dennis Luxen, others 2010
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU AFFERO General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+or see http://www.gnu.org/licenses/agpl.txt.
+
+
+Custom validators for use with boost::program_options.
+*/
+
+#include <boost/program_options.hpp>
+
+namespace boost {
+    namespace program_options {
+        // Class for reporting missing files
+        class missing_file_error : public boost::program_options::invalid_option_value {
+        public:
+            missing_file_error(const std::string& path) : invalid_option_value(path), path(path) {};
+            ~missing_file_error() throw() {};
+            const char* what() const throw() {
+                std::string err = get_option_name() + " file not found: " + path;
+                return err.c_str();
+            };
+        protected:
+            std::string path;
+        };
+    }
+
+    namespace filesystem {
+        // Validator for boost::filesystem::path, that verifies that the file exists.
+        // The validate() function must be defined in the same namespace as the target type,
+        // (boost::filesystem::path in this case), otherwise it will not be called
+        void validate(boost::any& v, const std::vector<std::string>& values, boost::filesystem::path*, int) {
+            boost::program_options::validators::check_first_occurrence(v);  // avoid previous assignments
+            const std::string& input_string = boost::program_options::validators::get_single_string(values); // avoid multiple assignments
+            if(boost::filesystem::is_regular_file(input_string)) {
+                v = boost::any(boost::filesystem::path(input_string));
+            } else {
+                throw boost::program_options::missing_file_error(input_string);
+            }
+        }
+    }
+}

cmake/GetGitRevisionDescription.cmake

@@ -0,0 +1,123 @@
+# - Returns a version string from Git
+#
+# These functions force a re-configure on each git commit so that you can
+# trust the values of the variables in your build system.
+#
+#  get_git_head_revision(<refspecvar> <hashvar> [<additional arguments to git describe> ...])
+#
+# Returns the refspec and sha hash of the current head revision
+#
+#  git_describe(<var> [<additional arguments to git describe> ...])
+#
+# Returns the results of git describe on the source tree, and adjusting
+# the output so that it tests false if an error occurs.
+#
+#  git_get_exact_tag(<var> [<additional arguments to git describe> ...])
+#
+# Returns the results of git describe --exact-match on the source tree,
+# and adjusting the output so that it tests false if there was no exact
+# matching tag.
+#
+# Requires CMake 2.6 or newer (uses the 'function' command)
+#
+# Original Author:
+# 2009-2010 Ryan Pavlik <rpavlik@iastate.edu> <abiryan@ryand.net>
+# http://academic.cleardefinition.com
+# Iowa State University HCI Graduate Program/VRAC
+#
+# Copyright Iowa State University 2009-2010.
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at
+# http://www.boost.org/LICENSE_1_0.txt)
+
+if(__get_git_revision_description)
+	return()
+endif()
+set(__get_git_revision_description YES)
+
+# We must run the following at "include" time, not at function call time,
+# to find the path to this module rather than the path to a calling list file
+get_filename_component(_gitdescmoddir ${CMAKE_CURRENT_LIST_FILE} PATH)
+
+function(get_git_head_revision _refspecvar _hashvar)
+	set(GIT_PARENT_DIR "${CMAKE_SOURCE_DIR}")
+	set(GIT_DIR "${GIT_PARENT_DIR}/.git")
+	while(NOT EXISTS "${GIT_DIR}")	# .git dir not found, search parent directories
+		set(GIT_PREVIOUS_PARENT "${GIT_PARENT_DIR}")
+		get_filename_component(GIT_PARENT_DIR ${GIT_PARENT_DIR} PATH)
+		if(GIT_PARENT_DIR STREQUAL GIT_PREVIOUS_PARENT)
+			# We have reached the root directory, we are not in git
+			set(${_refspecvar} "GITDIR-NOTFOUND" PARENT_SCOPE)
+			set(${_hashvar} "GITDIR-NOTFOUND" PARENT_SCOPE)
+			return()
+		endif()
+		set(GIT_DIR "${GIT_PARENT_DIR}/.git")
+	endwhile()
+	set(GIT_DATA "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/git-data")
+	if(NOT EXISTS "${GIT_DATA}")
+		file(MAKE_DIRECTORY "${GIT_DATA}")
+	endif()
+
+	if(NOT EXISTS "${GIT_DIR}/HEAD")
+		return()
+	endif()
+	set(HEAD_FILE "${GIT_DATA}/HEAD")
+	configure_file("${GIT_DIR}/HEAD" "${HEAD_FILE}" COPYONLY)
+
+	configure_file("${_gitdescmoddir}/GetGitRevisionDescription.cmake.in"
+		"${GIT_DATA}/grabRef.cmake"
+		@ONLY)
+	include("${GIT_DATA}/grabRef.cmake")
+
+	set(${_refspecvar} "${HEAD_REF}" PARENT_SCOPE)
+	set(${_hashvar} "${HEAD_HASH}" PARENT_SCOPE)
+endfunction()
+
+function(git_describe _var)
+	if(NOT GIT_FOUND)
+		find_package(Git QUIET)
+	endif()
+	get_git_head_revision(refspec hash)
+	if(NOT GIT_FOUND)
+		set(${_var} "GIT-NOTFOUND" PARENT_SCOPE)
+		return()
+	endif()
+	if(NOT hash)
+		set(${_var} "HEAD-HASH-NOTFOUND" PARENT_SCOPE)
+		return()
+	endif()
+
+	# TODO sanitize
+	#if((${ARGN}" MATCHES "&&") OR
+	#	(ARGN MATCHES "||") OR
+	#	(ARGN MATCHES "\\;"))
+	#	message("Please report the following error to the project!")
+	#	message(FATAL_ERROR "Looks like someone's doing something nefarious with git_describe! Passed arguments ${ARGN}")
+	#endif()
+
+	#message(STATUS "Arguments to execute_process: ${ARGN}")
+
+	execute_process(COMMAND
+		"${GIT_EXECUTABLE}"
+		describe
+		${hash}
+		${ARGN}
+		WORKING_DIRECTORY
+		"${CMAKE_SOURCE_DIR}"
+		RESULT_VARIABLE
+		res
+		OUTPUT_VARIABLE
+		out
+		ERROR_QUIET
+		OUTPUT_STRIP_TRAILING_WHITESPACE)
+	if(NOT res EQUAL 0)
+		set(out "${out}-${res}-NOTFOUND")
+	endif()
+
+	set(${_var} "${out}" PARENT_SCOPE)
+endfunction()
+
+function(git_get_exact_tag _var)
+	git_describe(out --exact-match ${ARGN})
+	set(${_var} "${out}" PARENT_SCOPE)
+endfunction()

cmake/GetGitRevisionDescription.cmake.in

@@ -0,0 +1,38 @@
+# 
+# Internal file for GetGitRevisionDescription.cmake
+#
+# Requires CMake 2.6 or newer (uses the 'function' command)
+#
+# Original Author:
+# 2009-2010 Ryan Pavlik <rpavlik@iastate.edu> <abiryan@ryand.net>
+# http://academic.cleardefinition.com
+# Iowa State University HCI Graduate Program/VRAC
+#
+# Copyright Iowa State University 2009-2010.
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at
+# http://www.boost.org/LICENSE_1_0.txt)
+
+set(HEAD_HASH)
+
+file(READ "@HEAD_FILE@" HEAD_CONTENTS LIMIT 1024)
+
+string(STRIP "${HEAD_CONTENTS}" HEAD_CONTENTS)
+if(HEAD_CONTENTS MATCHES "ref")
+	# named branch
+	string(REPLACE "ref: " "" HEAD_REF "${HEAD_CONTENTS}")
+	if(EXISTS "@GIT_DIR@/${HEAD_REF}")
+		configure_file("@GIT_DIR@/${HEAD_REF}" "@GIT_DATA@/head-ref" COPYONLY)
+	elseif(EXISTS "@GIT_DIR@/logs/${HEAD_REF}")
+		configure_file("@GIT_DIR@/logs/${HEAD_REF}" "@GIT_DATA@/head-ref" COPYONLY)
+		set(HEAD_HASH "${HEAD_REF}")
+	endif()
+else()
+	# detached HEAD
+	configure_file("@GIT_DIR@/HEAD" "@GIT_DATA@/head-ref" COPYONLY)
+endif()
+
+if(NOT HEAD_HASH)
+	file(READ "@GIT_DATA@/head-ref" HEAD_HASH LIMIT 1024)
+	string(STRIP "${HEAD_HASH}" HEAD_HASH)
+endif()

extractor.cpp

@@ -23,11 +23,11 @@ or see http://www.gnu.org/licenses/agpl.txt.
 #include "Extractor/ScriptingEnvironment.h"
 #include "Extractor/PBFParser.h"
 #include "Extractor/XMLParser.h"
-#include "Util/IniFile.h"
+#include "Util/GitDescription.h"
-#include "Util/InputFileUtil.h"
 #include "Util/MachineInfo.h"
 #include "Util/OpenMPWrapper.h"
 #include "Util/OSRMException.h"
+#include "Util/ProgramOptions.h"
 #include "Util/SimpleLogger.h"
 #include "Util/StringUtil.h"
 #include "Util/UUID.h"
@@ -46,32 +46,93 @@ int main (int argc, char *argv[]) {
     try {
         LogPolicy::GetInstance().Unmute();
         double startup_time = get_timestamp();
-        if(argc < 2) {
+
-            SimpleLogger().Write(logWARNING) <<
+        boost::filesystem::path config_file_path, input_path, profile_path;
-                "usage: \n" <<
+        int requested_num_threads;
-                argv[0] <<
+
-                " <file.osm/.osm.bz2/.osm.pbf> [<profile.lua>]";
+        // declare a group of options that will be allowed only on command line
+        boost::program_options::options_description generic_options("Options");
+        generic_options.add_options()
+            ("version,v", "Show version")
+            ("help,h", "Show this help message")
+            ("config,c", boost::program_options::value<boost::filesystem::path>(&config_file_path)->default_value("extractor.ini"),
+                  "Path to a configuration file.");
+
+        // declare a group of options that will be allowed both on command line and in config file
+        boost::program_options::options_description config_options("Configuration");
+        config_options.add_options()
+            ("profile,p", boost::program_options::value<boost::filesystem::path>(&profile_path)->default_value("profile.lua"),
+                "Path to LUA routing profile")
+            ("threads,t", boost::program_options::value<int>(&requested_num_threads)->default_value(10), 
+                "Number of threads to use");
+
+        // hidden options, will be allowed both on command line and in config file, but will not be shown to the user
+        boost::program_options::options_description hidden_options("Hidden options");
+        hidden_options.add_options()
+            ("input,i", boost::program_options::value<boost::filesystem::path>(&input_path),
+                "Input file in .osm, .osm.bz2 or .osm.pbf format");
+
+        // positional option
+        boost::program_options::positional_options_description positional_options;
+        positional_options.add("input", 1);
+
+        // combine above options for parsing
+        boost::program_options::options_description cmdline_options;
+        cmdline_options.add(generic_options).add(config_options).add(hidden_options);
+
+        boost::program_options::options_description config_file_options;
+        config_file_options.add(config_options).add(hidden_options);
+
+        boost::program_options::options_description visible_options(boost::filesystem::basename(argv[0]) + " <input.osm/.osm.bz2/.osm.pbf> [<profile.lua>]");
+        visible_options.add(generic_options).add(config_options);
+
+        // parse command line options
+        boost::program_options::variables_map option_variables;
+        boost::program_options::store(boost::program_options::command_line_parser(argc, argv).
+            options(cmdline_options).positional(positional_options).run(), option_variables);
+
+        if(option_variables.count("version")) {
+            SimpleLogger().Write() << g_GIT_DESCRIPTION;
+            return 0;
+        }
+
+        if(option_variables.count("help")) {
+            SimpleLogger().Write() << visible_options;
+            return 0;
+        }
+
+        boost::program_options::notify(option_variables);
+
+        // parse config file
+        if(boost::filesystem::is_regular_file(config_file_path)) {
+            std::ifstream ifs(config_file_path.c_str());
+            SimpleLogger().Write() << "Reading options from: " << config_file_path.filename().string();
+            boost::program_options::store(parse_config_file(ifs, config_file_options), option_variables);
+            boost::program_options::notify(option_variables);
+        }
+
+        if(!option_variables.count("input")) {
+            SimpleLogger().Write(logWARNING) << "No input file specified";
             return -1;
         }
 
+        if(1 > requested_num_threads) {
+            SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger";
+            return -1;
+        }
+
+        SimpleLogger().Write() << "Input file: " << input_path.filename().string();
+        SimpleLogger().Write() << "Profile: " << profile_path.filename().string();
+        SimpleLogger().Write() << "Threads: " << requested_num_threads;
+
         /*** Setup Scripting Environment ***/
-        ScriptingEnvironment scriptingEnvironment((argc > 2 ? argv[2] : "profile.lua"));
+        ScriptingEnvironment scriptingEnvironment(profile_path.c_str());
 
-        unsigned number_of_threads = omp_get_num_procs();
+        omp_set_num_threads( std::min( omp_get_num_procs(), requested_num_threads) );
 
-        if(testDataFile("extractor.ini")) {
-            IniFile extractorConfig("extractor.ini");
-            unsigned rawNumber = stringToInt(extractorConfig.GetParameter("Threads"));
-            if( rawNumber != 0 && rawNumber <= number_of_threads) {
-                number_of_threads = rawNumber;
-            }
-        }
-        omp_set_num_threads(number_of_threads);
-
-        SimpleLogger().Write() << "extracting data from input file " << argv[1];
         bool file_has_pbf_format(false);
-        std::string output_file_name(argv[1]);
+        std::string output_file_name(input_path.c_str());
-        std::string restrictionsFileName(argv[1]);
+        std::string restrictionsFileName(input_path.c_str());
         std::string::size_type pos = output_file_name.find(".osm.bz2");
         if(pos==std::string::npos) {
             pos = output_file_name.find(".osm.pbf");
@@ -112,9 +173,9 @@ int main (int argc, char *argv[]) {
         extractCallBacks = new ExtractorCallbacks(&externalMemory, &stringMap);
         BaseParser* parser;
         if(file_has_pbf_format) {
-            parser = new PBFParser(argv[1], extractCallBacks, scriptingEnvironment);
+            parser = new PBFParser(input_path.c_str(), extractCallBacks, scriptingEnvironment);
         } else {
-            parser = new XMLParser(argv[1], extractCallBacks, scriptingEnvironment);
+            parser = new XMLParser(input_path.c_str(), extractCallBacks, scriptingEnvironment);
         }
 
         if(!parser->ReadHeader()) {
@@ -141,8 +202,14 @@ int main (int argc, char *argv[]) {
             " " <<
             restrictionsFileName <<
             std::endl;
+    } catch(boost::program_options::too_many_positional_options_error& e) {
+        SimpleLogger().Write(logWARNING) << "Only one input file can be specified";
+        return -1;
+    } catch(boost::program_options::error& e) {
+        SimpleLogger().Write(logWARNING) << e.what();
+        return -1;
     } catch(std::exception & e) {
-        SimpleLogger().Write(logWARNING) << "unhandled exception: " << e.what();
+        SimpleLogger().Write(logWARNING) << "Unhandled exception: " << e.what();
         return -1;
     }
     return 0;

extractor.ini

@@ -1,2 +0,0 @@
-Memory = 2
-Threads = 10

features/support/data.rb

@@ -250,7 +250,7 @@ def reprocess
     unless extracted?
       log_preprocess_info
       log "== Extracting #{@osm_file}.osm...", :preprocess
-      unless system "#{BIN_PATH}/osrm-extract #{@osm_file}.osm#{'.pbf' if use_pbf} 1>>#{PREPROCESS_LOG_FILE} 2>>#{PREPROCESS_LOG_FILE} #{PROFILES_PATH}/#{@profile}.lua"
+      unless system "#{BIN_PATH}/osrm-extract #{@osm_file}.osm#{'.pbf' if use_pbf} --profile #{PROFILES_PATH}/#{@profile}.lua 1>>#{PREPROCESS_LOG_FILE} 2>>#{PREPROCESS_LOG_FILE}"
         log "*** Exited with code #{$?.exitstatus}.", :preprocess
         raise ExtractError.new $?.exitstatus, "osrm-extract exited with code #{$?.exitstatus}."
       end

test/extractor.ini

@@ -1 +0,0 @@
-Memory = 1