From 7ba8e51fa9ef52a1743b7b0a61b37f96d3d50813 Mon Sep 17 00:00:00 2001 From: Emil Tin Date: Fri, 24 Jan 2014 21:14:38 +0100 Subject: [PATCH] use osrm-database during testing --- features/datastore/datastore.feature | 72 ++++++++++++++++++ features/step_definitions/locate.rb | 2 +- features/step_definitions/nearest.rb | 2 +- features/step_definitions/options.rb | 6 +- features/step_definitions/requests.rb | 4 +- features/step_definitions/routability.rb | 2 +- features/step_definitions/routing.rb | 2 +- features/support/data.rb | 12 ++- features/support/env.rb | 18 +++++ features/support/launch.rb | 94 +++++++++++++----------- features/support/route.rb | 2 + features/testbot/datastore.feature | 31 ++++++++ 12 files changed, 192 insertions(+), 55 deletions(-) create mode 100644 features/datastore/datastore.feature create mode 100644 features/testbot/datastore.feature diff --git a/features/datastore/datastore.feature b/features/datastore/datastore.feature new file mode 100644 index 000000000..2e37f0256 --- /dev/null +++ b/features/datastore/datastore.feature @@ -0,0 +1,72 @@ +@datastore +Feature: Loading data through datastore +# Normally when launching osrm-routed, it will keep running as a server until it's shut down. +# For testing program options, the --trial option is used, which causes osrm-routed to quit +# immediately after initialization. This makes testing easier and faster. +# +# The {base} part of the options to osrm-routed will be expanded to the actual base path of +# the preprocessed file. + + Background: + Given the profile "testbot" + + Scenario: Load data with osrm-datastore - medium size + Given the node map + | a | b | c | + | d | e | f | + And the ways + | nodes | + | abc | + | def | + | ad | + | be | + | cf | + And the data has been prepared + + When I run "osrm-datastore --springclean" + Then stderr should be empty + And stdout should contain /^\[info\] spring-cleaning all shared memory regions/ + And it should exit with code 0 + + When I run "osrm-datastore {base}.osrm" + Then stderr should be empty + And stdout should contain /^\[info\] all data loaded/ + And it should exit with code 0 + + When I run "osrm-routed --sharedmemory --trial" + Then stderr should be empty + And stdout should contain /^\[info\] starting up engines/ + And stdout should contain /\d{1,2}\.\d{1,2}\.\d{1,2}/ + And stdout should contain /compiled at/ + And stdout should contain /^\[info\] loaded plugin: viaroute/ + And stdout should contain /^\[info\] trial run/ + And stdout should contain /^\[info\] shutdown completed/ + And it should exit with code 0 + + Scenario: Load data with osrm-datastore - small size + Given the node map + | a | b | + And the ways + | nodes | + | ab | + And the data has been prepared + + When I run "osrm-datastore --springclean" + Then stderr should be empty + And stdout should contain /^\[info\] spring-cleaning all shared memory regions/ + And it should exit with code 0 + + When I run "osrm-datastore {base}.osrm" + Then stderr should be empty + And stdout should contain /^\[info\] all data loaded/ + And it should exit with code 0 + + When I run "osrm-routed --sharedmemory --trial" + Then stderr should be empty + And stdout should contain /^\[info\] starting up engines/ + And stdout should contain /\d{1,2}\.\d{1,2}\.\d{1,2}/ + And stdout should contain /compiled at/ + And stdout should contain /^\[info\] loaded plugin: viaroute/ + And stdout should contain /^\[info\] trial run/ + And stdout should contain /^\[info\] shutdown completed/ + And it should exit with code 0 \ No newline at end of file diff --git a/features/step_definitions/locate.rb b/features/step_definitions/locate.rb index 9d3d74f65..26c52e2d0 100644 --- a/features/step_definitions/locate.rb +++ b/features/step_definitions/locate.rb @@ -1,7 +1,7 @@ When /^I request locate I should get$/ do |table| reprocess actual = [] - OSRMBackgroundLauncher.new("#{@osm_file}.osrm") do + OSRMLoader.load("#{@osm_file}.osrm") do table.hashes.each_with_index do |row,ri| in_node = find_node_by_name row['in'] raise "*** unknown in-node '#{row['in']}" unless in_node diff --git a/features/step_definitions/nearest.rb b/features/step_definitions/nearest.rb index ae5a79cf5..030a8e5bc 100644 --- a/features/step_definitions/nearest.rb +++ b/features/step_definitions/nearest.rb @@ -1,7 +1,7 @@ When /^I request nearest I should get$/ do |table| reprocess actual = [] - OSRMBackgroundLauncher.new("#{@osm_file}.osrm") do + OSRMLoader.load("#{@osm_file}.osrm") do table.hashes.each_with_index do |row,ri| in_node = find_node_by_name row['in'] raise "*** unknown in-node '#{row['in']}" unless in_node diff --git a/features/step_definitions/options.rb b/features/step_definitions/options.rb index 38c9ea256..e2e50661e 100644 --- a/features/step_definitions/options.rb +++ b/features/step_definitions/options.rb @@ -1,6 +1,6 @@ When(/^I run "osrm\-routed\s?(.*?)"$/) do |options| begin - Timeout.timeout(1) { run_bin 'osrm-routed', options } + Timeout.timeout(SHUTDOWN_TIMEOUT) { run_bin 'osrm-routed', options } rescue Timeout::Error raise "*** osrm-routed didn't quit. Maybe the --trial option wasn't used?" end @@ -14,6 +14,10 @@ When(/^I run "osrm\-prepare\s?(.*?)"$/) do |options| run_bin 'osrm-prepare', options end +When(/^I run "osrm\-datastore\s?(.*?)"$/) do |options| + run_bin 'osrm-datastore', options +end + Then /^it should exit with code (\d+)$/ do |code| expect(@exit_code).to eq( code.to_i ) end diff --git a/features/step_definitions/requests.rb b/features/step_definitions/requests.rb index 7002cd9e3..b9f4d98f6 100644 --- a/features/step_definitions/requests.rb +++ b/features/step_definitions/requests.rb @@ -1,6 +1,6 @@ When /^I request \/(.*)$/ do |path| reprocess - OSRMBackgroundLauncher.new("#{@osm_file}.osrm") do + OSRMLoader.load("#{@osm_file}.osrm") do @response = request_path path end end @@ -43,4 +43,4 @@ Then /^"([^"]*)" should return code (\d+)$/ do |binary, code| expect(@process_error.is_a?(OSRMError)).to eq(true) expect(@process_error.process).to eq(binary) expect(@process_error.code.to_i).to eq(code.to_i) -end \ No newline at end of file +end diff --git a/features/step_definitions/routability.rb b/features/step_definitions/routability.rb index 48f7f24de..e46d0d9ff 100644 --- a/features/step_definitions/routability.rb +++ b/features/step_definitions/routability.rb @@ -44,7 +44,7 @@ Then /^routability should be$/ do |table| if table.headers&["forw","backw","bothw"] == [] raise "*** routability tabel must contain either 'forw', 'backw' or 'bothw' column" end - OSRMBackgroundLauncher.new("#{@osm_file}.osrm") do + OSRMLoader.load("#{@osm_file}.osrm") do table.hashes.each_with_index do |row,i| output_row = row.dup attempts = [] diff --git a/features/step_definitions/routing.rb b/features/step_definitions/routing.rb index 4d944f32b..53a1b6fec 100644 --- a/features/step_definitions/routing.rb +++ b/features/step_definitions/routing.rb @@ -1,7 +1,7 @@ When /^I route I should get$/ do |table| reprocess actual = [] - OSRMBackgroundLauncher.new("#{@osm_file}.osrm") do + OSRMLoader.load("#{@osm_file}.osrm") do table.hashes.each_with_index do |row,ri| if row['request'] got = {'request' => row['request'] } diff --git a/features/support/data.rb b/features/support/data.rb index 7c1ba887a..c09b763bd 100644 --- a/features/support/data.rb +++ b/features/support/data.rb @@ -225,13 +225,17 @@ def convert_osm_to_pbf end def extracted? - File.exist?("#{@osm_file}.osrm") && - File.exist?("#{@osm_file}.osrm.names") && - File.exist?("#{@osm_file}.osrm.restrictions") + Dir.chdir TEST_FOLDER do + File.exist?("#{@osm_file}.osrm") && + File.exist?("#{@osm_file}.osrm.names") && + File.exist?("#{@osm_file}.osrm.restrictions") + end end def prepared? - File.exist?("#{@osm_file}.osrm.hsgr") + Dir.chdir TEST_FOLDER do + File.exist?("#{@osm_file}.osrm.hsgr") + end end def write_timestamp diff --git a/features/support/env.rb b/features/support/env.rb index 369705ab0..cb328d12b 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -17,6 +17,20 @@ PROFILES_PATH = File.join ROOT_FOLDER, 'profiles' BIN_PATH = File.join ROOT_FOLDER, 'build' DEFAULT_INPUT_FORMAT = 'osm' DEFAULT_ORIGIN = [1,1] +LAUNCH_TIMEOUT = 1 +SHUTDOWN_TIMEOUT = 10 + + +def log_time_and_run cmd + log_time cmd + `#{cmd}` +end + +def log_time cmd + puts "[#{Time.now.strftime('%Y-%m-%d %H:%M:%S:%L')}] #{cmd}" +end + + puts "Ruby version #{RUBY_VERSION}" @@ -55,3 +69,7 @@ end AfterConfiguration do |config| clear_log_files end + +at_exit do + OSRMLoader.shutdown +end diff --git a/features/support/launch.rb b/features/support/launch.rb index e55867a67..8a458da17 100644 --- a/features/support/launch.rb +++ b/features/support/launch.rb @@ -9,76 +9,82 @@ end OSRM_ROUTED_LOG_FILE = 'osrm-routed.log' -class OSRMBackgroundLauncher - def initialize input_file, &block +class OSRMLoader + + @@pid = nil + + def self.load input_file, &block @input_file = input_file Dir.chdir TEST_FOLDER do - begin - launch - yield - ensure - shutdown - end + self.load_data + self.launch unless @@pid + yield end end + + def self.load_data + puts "=== loading data with osrm-datastore" + log_time_and_run "#{BIN_PATH}/osrm-datastore --springclean" + log_time_and_run "#{BIN_PATH}/osrm-datastore #{@input_file}" + end - private - - def launch - Timeout.timeout(OSRM_TIMEOUT) do - osrm_up - wait_for_connection + def self.launch + Timeout.timeout(LAUNCH_TIMEOUT) do + self.osrm_up + self.wait_for_connection end rescue Timeout::Error raise RoutedError.new "Launching osrm-routed timed out." end - def shutdown - Timeout.timeout(OSRM_TIMEOUT) do - osrm_down + def self.shutdown + Timeout.timeout(SHUTDOWN_TIMEOUT) do + self.osrm_down end rescue Timeout::Error - kill + self.kill raise RoutedError.new "Shutting down osrm-routed timed out." end - - def osrm_up? - if @pid - begin - if Process.waitpid(@pid, Process::WNOHANG) then - false - else - true - end - rescue Errno::ESRCH, Errno::ECHILD - false - end + def self.osrm_up? + if @@pid + s = `ps -o state -p #{@@pid}`.split[1].to_s.strip + up = (s =~ /^[DRST]/) != nil + # puts "=== osrm-routed, status pid #{@@pid}: #{s} (#{up ? 'up' : 'down'})" + up + else + false end end - def osrm_up - return if osrm_up? - @pid = Process.spawn("#{BIN_PATH}/osrm-routed #{@input_file} --port #{OSRM_PORT}",:out=>OSRM_ROUTED_LOG_FILE, :err=>OSRM_ROUTED_LOG_FILE) - Process.detach(@pid) # avoid zombie processes + def self.osrm_up + return if self.osrm_up? + puts "=== launching osrm-routed" + log %[Process.spawn("#{BIN_PATH}/osrm-routed --sharedmemory=1 --port #{OSRM_PORT}",:out=>'#{OSRM_ROUTED_LOG_FILE}', :err=>'#{OSRM_ROUTED_LOG_FILE}')] + @@pid = Process.spawn("#{BIN_PATH}/osrm-routed --sharedmemory=1 --port #{OSRM_PORT}",:out=>OSRM_ROUTED_LOG_FILE, :err=>OSRM_ROUTED_LOG_FILE) end - def osrm_down - if @pid - Process.kill TERMSIGNAL, @pid - wait_for_shutdown + def self.osrm_down + if @@pid + puts '=== shutting down osrm' + log_time "Process.kill 'TERM', #{@@pid}" + Process.kill 'TERM', @@pid + self.wait_for_shutdown end end - def kill - if @pid - Process.kill 'KILL', @pid + def self.kill + if @@pid + puts '=== killing osrm' + log_time "Process.kill 'KILL', @@pid" + Process.kill 'KILL', @@pid end end - def wait_for_connection + def self.wait_for_connection while true begin + log_time "TCPSocket.new('127.0.0.1', OSRM_PORT)" socket = TCPSocket.new('127.0.0.1', OSRM_PORT) return rescue Errno::ECONNREFUSED @@ -87,8 +93,8 @@ class OSRMBackgroundLauncher end end - def wait_for_shutdown - while osrm_up? + def self.wait_for_shutdown + while self.osrm_up? sleep 0.1 end end diff --git a/features/support/route.rb b/features/support/route.rb index 745edc0fd..cf81d9e37 100644 --- a/features/support/route.rb +++ b/features/support/route.rb @@ -18,6 +18,7 @@ def request_path path, waypoints=[], options={} uri = URI.parse ["#{HOST}/#{path}", params].compact.join('?') @query = uri.to_s Timeout.timeout(OSRM_TIMEOUT) do + log_time "Net::HTTP.get_response #{uri}" Net::HTTP.get_response uri end rescue Errno::ECONNREFUSED => e @@ -30,6 +31,7 @@ def request_url path uri = URI.parse"#{HOST}/#{path}" @query = uri.to_s Timeout.timeout(OSRM_TIMEOUT) do + log_time "Net::HTTP.get_response #{uri}" Net::HTTP.get_response uri end rescue Errno::ECONNREFUSED => e diff --git a/features/testbot/datastore.feature b/features/testbot/datastore.feature new file mode 100644 index 000000000..7814f3b47 --- /dev/null +++ b/features/testbot/datastore.feature @@ -0,0 +1,31 @@ +@routing @datastore +Feature: Temporary tests related to osrm-datastore + + Background: + Given the profile "testbot" + + Scenario: Scenario ab + Given the node map + | a | b | + + And the ways + | nodes | + | ab | + + When I route I should get + | from | to | route | + | a | b | ab | + | b | a | ab | + + Scenario: Scenaria xy + Given the node map + | x | y | + + And the ways + | nodes | + | xy | + + When I route I should get + | from | to | route | + | x | y | xy | + | y | x | xy |