use osrm-database during testing

features/datastore/datastore.feature

@@ -0,0 +1,72 @@
+@datastore
+Feature: Loading data through datastore
+# Normally when launching osrm-routed, it will keep running as a server until it's shut down.
+# For testing program options, the --trial option is used, which causes osrm-routed to quit
+# immediately after initialization. This makes testing easier and faster.
+# 
+# The {base} part of the options to osrm-routed will be expanded to the actual base path of
+# the preprocessed file.
+
+    Background:
+        Given the profile "testbot"
+
+    Scenario: Load data with osrm-datastore - medium size
+        Given the node map
+            | a | b | c |
+            | d | e | f |
+        And the ways
+            | nodes |
+            | abc   |
+            | def   |
+            | ad    |
+            | be    |
+            | cf    |
+        And the data has been prepared
+        
+        When I run "osrm-datastore --springclean"
+        Then stderr should be empty
+        And stdout should contain /^\[info\] spring-cleaning all shared memory regions/
+        And it should exit with code 0
+        
+        When I run "osrm-datastore {base}.osrm"
+        Then stderr should be empty
+        And stdout should contain /^\[info\] all data loaded/
+        And it should exit with code 0
+
+        When I run "osrm-routed --sharedmemory --trial"
+        Then stderr should be empty
+        And stdout should contain /^\[info\] starting up engines/
+        And stdout should contain /\d{1,2}\.\d{1,2}\.\d{1,2}/
+        And stdout should contain /compiled at/
+        And stdout should contain /^\[info\] loaded plugin: viaroute/
+        And stdout should contain /^\[info\] trial run/
+        And stdout should contain /^\[info\] shutdown completed/
+        And it should exit with code 0
+
+    Scenario: Load data with osrm-datastore - small size
+        Given the node map
+            | a | b |
+        And the ways
+            | nodes |
+            | ab    |
+        And the data has been prepared
+
+        When I run "osrm-datastore --springclean"
+        Then stderr should be empty
+        And stdout should contain /^\[info\] spring-cleaning all shared memory regions/
+        And it should exit with code 0
+        
+        When I run "osrm-datastore {base}.osrm"
+        Then stderr should be empty
+        And stdout should contain /^\[info\] all data loaded/
+        And it should exit with code 0
+
+        When I run "osrm-routed --sharedmemory --trial"
+        Then stderr should be empty
+        And stdout should contain /^\[info\] starting up engines/
+        And stdout should contain /\d{1,2}\.\d{1,2}\.\d{1,2}/
+        And stdout should contain /compiled at/
+        And stdout should contain /^\[info\] loaded plugin: viaroute/
+        And stdout should contain /^\[info\] trial run/
+        And stdout should contain /^\[info\] shutdown completed/
+        And it should exit with code 0

features/step_definitions/locate.rb

@@ -1,7 +1,7 @@
 When /^I request locate I should get$/ do |table|
   reprocess
   actual = []
-  OSRMBackgroundLauncher.new("#{@osm_file}.osrm") do
+  OSRMLoader.load("#{@osm_file}.osrm") do
     table.hashes.each_with_index do |row,ri|
       in_node = find_node_by_name row['in']
       raise "*** unknown in-node '#{row['in']}" unless in_node

features/step_definitions/nearest.rb

@@ -1,7 +1,7 @@
 When /^I request nearest I should get$/ do |table|
   reprocess
   actual = []
-  OSRMBackgroundLauncher.new("#{@osm_file}.osrm") do
+  OSRMLoader.load("#{@osm_file}.osrm") do
     table.hashes.each_with_index do |row,ri|
       in_node = find_node_by_name row['in']
       raise "*** unknown in-node '#{row['in']}" unless in_node

features/step_definitions/options.rb

@@ -1,6 +1,6 @@
 When(/^I run "osrm\-routed\s?(.*?)"$/) do |options|
   begin
-    Timeout.timeout(1) { run_bin 'osrm-routed', options }
+    Timeout.timeout(SHUTDOWN_TIMEOUT) { run_bin 'osrm-routed', options }
   rescue Timeout::Error
     raise "*** osrm-routed didn't quit. Maybe the --trial option wasn't used?"
   end
@@ -14,6 +14,10 @@ When(/^I run "osrm\-prepare\s?(.*?)"$/) do |options|
   run_bin 'osrm-prepare', options
 end
 
+When(/^I run "osrm\-datastore\s?(.*?)"$/) do |options|
+  run_bin 'osrm-datastore', options
+end
+
 Then /^it should exit with code (\d+)$/ do |code|
   expect(@exit_code).to eq( code.to_i )
 end

features/step_definitions/requests.rb

@@ -1,6 +1,6 @@
 When /^I request \/(.*)$/ do |path|
   reprocess
-  OSRMBackgroundLauncher.new("#{@osm_file}.osrm") do
+  OSRMLoader.load("#{@osm_file}.osrm") do
     @response = request_path path
   end
 end
@@ -43,4 +43,4 @@ Then /^"([^"]*)" should return code (\d+)$/ do |binary, code|
   expect(@process_error.is_a?(OSRMError)).to eq(true)
   expect(@process_error.process).to eq(binary)
   expect(@process_error.code.to_i).to eq(code.to_i)
-end
+end

features/step_definitions/routability.rb

@@ -44,7 +44,7 @@ Then /^routability should be$/ do |table|
   if table.headers&["forw","backw","bothw"] == []
     raise "*** routability tabel must contain either 'forw', 'backw' or 'bothw' column"
   end
-  OSRMBackgroundLauncher.new("#{@osm_file}.osrm") do
+  OSRMLoader.load("#{@osm_file}.osrm") do
     table.hashes.each_with_index do |row,i|
       output_row = row.dup
       attempts = []

features/step_definitions/routing.rb

@@ -1,7 +1,7 @@
 When /^I route I should get$/ do |table|
   reprocess
   actual = []
-  OSRMBackgroundLauncher.new("#{@osm_file}.osrm") do
+  OSRMLoader.load("#{@osm_file}.osrm") do
     table.hashes.each_with_index do |row,ri|
       if row['request']
         got = {'request' => row['request'] }

features/support/data.rb

@@ -225,13 +225,17 @@ def convert_osm_to_pbf
 end
 
 def extracted?
-  File.exist?("#{@osm_file}.osrm") &&
+  Dir.chdir TEST_FOLDER do
-  File.exist?("#{@osm_file}.osrm.names") &&
+    File.exist?("#{@osm_file}.osrm") &&
-  File.exist?("#{@osm_file}.osrm.restrictions")
+    File.exist?("#{@osm_file}.osrm.names") &&
+    File.exist?("#{@osm_file}.osrm.restrictions")
+  end
 end
 
 def prepared?
-  File.exist?("#{@osm_file}.osrm.hsgr")
+  Dir.chdir TEST_FOLDER do
+    File.exist?("#{@osm_file}.osrm.hsgr")
+  end
 end
 
 def write_timestamp

features/support/env.rb

@@ -17,6 +17,20 @@ PROFILES_PATH = File.join ROOT_FOLDER, 'profiles'
 BIN_PATH = File.join ROOT_FOLDER, 'build'
 DEFAULT_INPUT_FORMAT = 'osm'
 DEFAULT_ORIGIN = [1,1]
+LAUNCH_TIMEOUT = 1
+SHUTDOWN_TIMEOUT = 10
+
+
+def log_time_and_run cmd
+  log_time cmd
+  `#{cmd}`
+end
+
+def log_time cmd
+  puts "[#{Time.now.strftime('%Y-%m-%d %H:%M:%S:%L')}] #{cmd}"
+end
+
+
 
 
 puts "Ruby version #{RUBY_VERSION}"
@@ -55,3 +69,7 @@ end
 AfterConfiguration do |config|
   clear_log_files
 end
+
+at_exit do
+  OSRMLoader.shutdown
+end

features/support/launch.rb

@@ -9,76 +9,82 @@ end
 
 OSRM_ROUTED_LOG_FILE = 'osrm-routed.log'
 
-class OSRMBackgroundLauncher
+class OSRMLoader
-  def initialize input_file, &block
+  
+  @@pid = nil
+    
+  def self.load input_file, &block
     @input_file = input_file
     Dir.chdir TEST_FOLDER do
-      begin
+      self.load_data
-        launch
+      self.launch unless @@pid
-        yield
+      yield
-      ensure
-        shutdown
-      end
     end
   end
+  
+  def self.load_data
+    puts "=== loading data with osrm-datastore"
+    log_time_and_run "#{BIN_PATH}/osrm-datastore --springclean"
+    log_time_and_run "#{BIN_PATH}/osrm-datastore #{@input_file}"
+  end
 
-  private
+  def self.launch
-
+    Timeout.timeout(LAUNCH_TIMEOUT) do
-  def launch
+      self.osrm_up
-    Timeout.timeout(OSRM_TIMEOUT) do
+      self.wait_for_connection
-      osrm_up
-      wait_for_connection
     end
   rescue Timeout::Error
     raise RoutedError.new "Launching osrm-routed timed out."
   end
 
-  def shutdown
+  def self.shutdown
-    Timeout.timeout(OSRM_TIMEOUT) do
+    Timeout.timeout(SHUTDOWN_TIMEOUT) do
-      osrm_down
+      self.osrm_down
     end
   rescue Timeout::Error
-    kill
+    self.kill
     raise RoutedError.new "Shutting down osrm-routed timed out."
   end
 
-
+  def self.osrm_up?
-  def osrm_up?
+    if @@pid
-    if @pid
+      s = `ps -o state -p #{@@pid}`.split[1].to_s.strip
-      begin
+      up = (s =~ /^[DRST]/) != nil
-        if Process.waitpid(@pid, Process::WNOHANG) then
+ #     puts "=== osrm-routed, status pid #{@@pid}: #{s} (#{up ? 'up' : 'down'})"
-           false
+      up
-        else
+    else
-           true
+      false
-        end
-      rescue Errno::ESRCH, Errno::ECHILD
-        false
-      end
     end
   end
 
-  def osrm_up
+  def self.osrm_up
-    return if osrm_up?
+    return if self.osrm_up?
-    @pid = Process.spawn("#{BIN_PATH}/osrm-routed #{@input_file} --port #{OSRM_PORT}",:out=>OSRM_ROUTED_LOG_FILE, :err=>OSRM_ROUTED_LOG_FILE)
+    puts "=== launching osrm-routed"
-    Process.detach(@pid)    # avoid zombie processes
+    log %[Process.spawn("#{BIN_PATH}/osrm-routed --sharedmemory=1 --port #{OSRM_PORT}",:out=>'#{OSRM_ROUTED_LOG_FILE}', :err=>'#{OSRM_ROUTED_LOG_FILE}')]
+    @@pid = Process.spawn("#{BIN_PATH}/osrm-routed --sharedmemory=1 --port #{OSRM_PORT}",:out=>OSRM_ROUTED_LOG_FILE, :err=>OSRM_ROUTED_LOG_FILE)
   end
 
-  def osrm_down
+  def self.osrm_down
-    if @pid
+    if @@pid
-      Process.kill TERMSIGNAL, @pid
+    puts '=== shutting down osrm'
-      wait_for_shutdown
+      log_time "Process.kill 'TERM', #{@@pid}"
+      Process.kill 'TERM', @@pid
+      self.wait_for_shutdown
     end
   end
 
-  def kill
+  def self.kill
-    if @pid
+    if @@pid
-      Process.kill 'KILL', @pid
+      puts '=== killing osrm'
+      log_time "Process.kill 'KILL', @@pid"
+      Process.kill 'KILL', @@pid
     end
   end
 
-  def wait_for_connection
+  def self.wait_for_connection
     while true
       begin
+        log_time "TCPSocket.new('127.0.0.1', OSRM_PORT)"
         socket = TCPSocket.new('127.0.0.1', OSRM_PORT)
         return
       rescue Errno::ECONNREFUSED
@@ -87,8 +93,8 @@ class OSRMBackgroundLauncher
     end
   end
 
-  def wait_for_shutdown
+  def self.wait_for_shutdown
-    while osrm_up?
+    while self.osrm_up?
       sleep 0.1
     end
   end

features/support/route.rb

@@ -18,6 +18,7 @@ def request_path path, waypoints=[], options={}
   uri = URI.parse ["#{HOST}/#{path}", params].compact.join('?')
   @query = uri.to_s
   Timeout.timeout(OSRM_TIMEOUT) do
+    log_time "Net::HTTP.get_response #{uri}"
     Net::HTTP.get_response uri
   end
 rescue Errno::ECONNREFUSED => e
@@ -30,6 +31,7 @@ def request_url path
   uri = URI.parse"#{HOST}/#{path}"
   @query = uri.to_s
   Timeout.timeout(OSRM_TIMEOUT) do
+    log_time "Net::HTTP.get_response #{uri}"
     Net::HTTP.get_response uri
   end
 rescue Errno::ECONNREFUSED => e

features/testbot/datastore.feature

@@ -0,0 +1,31 @@
+@routing @datastore
+Feature: Temporary tests related to osrm-datastore
+
+    Background:
+        Given the profile "testbot"
+
+    Scenario: Scenario ab
+        Given the node map
+            | a | b |
+
+        And the ways
+            | nodes |
+            | ab    |
+
+        When I route I should get
+            | from | to | route |
+            | a    | b  | ab    |
+            | b    | a  | ab    |
+
+    Scenario: Scenaria xy
+        Given the node map
+            | x | y |
+
+        And the ways
+            | nodes |
+            | xy    |
+
+        When I route I should get
+            | from | to | route |
+            | x    | y  | xy    |
+            | y    | x  | xy    |