Merge pull request #1231 from Project-OSRM/testing/smarter_caching

smarter caching of test files

Rakefile

@@ -181,3 +181,10 @@ end
 desc "Stop, reprocess and restart."
 task :update => [:down,:process,:up] do
 end
+
+
+desc "Remove test cache files."
+task :sweep do
+  system "rm test/cache/*"
+end
+

features/options/extract/files.feature

@@ -1,7 +1,7 @@
 @extract @options @files
 Feature: osrm-extract command line options: files
 # expansions:
-# {base} => path to current input file
+# {osm_base} => path to current input file
 # {profile} => path to current profile script
 
     Background:
@@ -14,12 +14,12 @@ Feature: osrm-extract command line options: files
         And the data has been saved to disk
 
     Scenario: osrm-extract - Passing base file
-        When I run "osrm-extract {base}.osm --profile {profile}"
+        When I run "osrm-extract {osm_base}.osm --profile {profile}"
         Then stderr should be empty
         And it should exit with code 0
 
     Scenario: osrm-extract - Order of options should not matter
-        When I run "osrm-extract --profile {profile} {base}.osm"
+        When I run "osrm-extract --profile {profile} {osm_base}.osm"
         Then stderr should be empty
         And it should exit with code 0
 

features/options/prepare/files.feature

@@ -1,7 +1,7 @@
 @prepare @options @files
 Feature: osrm-prepare command line options: files
 # expansions:
-# {base} => path to current input file
+# {extracted_base} => path to current extracted input file
 # {profile} => path to current profile script
 
     Background:
@@ -14,12 +14,12 @@ Feature: osrm-prepare command line options: files
         And the data has been extracted
 
     Scenario: osrm-prepare - Passing base file
-        When I run "osrm-prepare {base}.osrm --profile {profile}"
+        When I run "osrm-prepare {extracted_base}.osrm --profile {profile}"
         Then stderr should be empty
         And it should exit with code 0
 
     Scenario: osrm-prepare - Order of options should not matter
-        When I run "osrm-prepare --profile {profile} {base}.osrm"
+        When I run "osrm-prepare --profile {profile} {extracted_base}.osrm"
         Then stderr should be empty
         And it should exit with code 0
 

features/options/routed/files.feature

@@ -4,8 +4,8 @@ Feature: osrm-routed command line options: files
 # For testing program options, the --trial option is used, which causes osrm-routed to quit
 # immediately after initialization. This makes testing easier and faster.
 # 
-# The {base} part of the options to osrm-routed will be expanded to the actual base path of
+# The {prepared_base} part of the options to osrm-routed will be expanded to the actual base path of
-# the preprocessed file.
+# the prepared input file.
 
 # TODO
 # Since we're not using osmr-datastore for all testing, osrm-routed is kept running.
@@ -22,7 +22,7 @@ Feature: osrm-routed command line options: files
         And the data has been prepared
 
     Scenario: osrm-routed - Passing base file
-        When I run "osrm-routed {base}.osrm --trial"
+        When I run "osrm-routed {prepared_base}.osrm --trial"
         Then stdout should contain /^\[info\] starting up engines/
         And stdout should contain /\d{1,2}\.\d{1,2}\.\d{1,2}/
         And stdout should contain /compiled at/

features/step_definitions/locate.rb

@@ -1,7 +1,7 @@
 When /^I request locate I should get$/ do |table|
   reprocess
   actual = []
-  OSRMLoader.load(self,"#{@osm_file}.osrm") do
+  OSRMLoader.load(self,"#{prepared_file}.osrm") do
     table.hashes.each_with_index do |row,ri|
       in_node = find_node_by_name row['in']
       raise "*** unknown in-node '#{row['in']}" unless in_node

features/step_definitions/nearest.rb

@@ -1,7 +1,7 @@
 When /^I request nearest I should get$/ do |table|
   reprocess
   actual = []
-  OSRMLoader.load(self,"#{@osm_file}.osrm") do
+  OSRMLoader.load(self,"#{prepared_file}.osrm") do
     table.hashes.each_with_index do |row,ri|
       in_node = find_node_by_name row['in']
       raise "*** unknown in-node '#{row['in']}" unless in_node

features/step_definitions/requests.rb

@@ -1,6 +1,6 @@
 When /^I request \/(.*)$/ do |path|
   reprocess
-  OSRMLoader.load(self,"#{@osm_file}.osrm") do
+  OSRMLoader.load(self,"#{prepared_file}.osrm") do
     @response = request_path path
   end
 end

features/step_definitions/routability.rb

@@ -44,7 +44,7 @@ Then /^routability should be$/ do |table|
   if table.headers&["forw","backw","bothw"] == []
     raise "*** routability tabel must contain either 'forw', 'backw' or 'bothw' column"
   end
-  OSRMLoader.load(self,"#{@osm_file}.osrm") do
+  OSRMLoader.load(self,"#{prepared_file}.osrm") do
     table.hashes.each_with_index do |row,i|
       output_row = row.dup
       attempts = []

features/step_definitions/routing.rb

@@ -1,7 +1,7 @@
 When /^I route I should get$/ do |table|
   reprocess
   actual = []
-  OSRMLoader.load(self,"#{@osm_file}.osrm") do
+  OSRMLoader.load(self,"#{prepared_file}.osrm") do
     table.hashes.each_with_index do |row,ri|
       if row['request']
         got = {'request' => row['request'] }

features/support/data.rb

@@ -1,6 +1,7 @@
 require 'OSM/objects'       #osmlib gem
 require 'OSM/Database'
 require 'builder'
+require 'fileutils'
 
 class Location
   attr_accessor :lon,:lat
@@ -155,7 +156,10 @@ def reset_data
   end
   reset_profile
   reset_osm
-  @fingerprint = nil
+  @fingerprint_osm = nil
+  @fingerprint_extract = nil
+  @fingerprint_prepare = nil
+  @fingerprint_route = nil
 end
 
 def make_osm_id
@@ -205,20 +209,30 @@ def osm_str
   @osm_str
 end
 
+def osm_file
+  @osm_file ||= "#{DATA_FOLDER}/#{fingerprint_osm}"
+end
+
+def extracted_file
+  @extracted_file ||= "#{osm_file}_#{fingerprint_extract}"
+end
+
+def prepared_file
+  @prepared_file ||= "#{osm_file}_#{fingerprint_extract}_#{fingerprint_prepare}"
+end
+
 def write_osm
-  #write .oms file if needed
   Dir.mkdir DATA_FOLDER unless File.exist? DATA_FOLDER
-  @osm_file = "#{DATA_FOLDER}/#{sanitized_scenario_title}_#{fingerprint}"
+  unless File.exist?("#{osm_file}.osm")
-  unless File.exist?("#{@osm_file}.osm")
+    File.open( "#{osm_file}.osm", 'w') {|f| f.write(osm_str) }
-    File.open( "#{@osm_file}.osm", 'w') {|f| f.write(osm_str) }
   end
 end
 
 def convert_osm_to_pbf
-  unless File.exist?("#{@osm_file}.osm.pbf")
+  unless File.exist?("#{osm_file}.osm.pbf")
     log_preprocess_info
-    log "== Converting #{@osm_file}.osm to protobuffer format...", :preprocess
+    log "== Converting #{osm_file}.osm to protobuffer format...", :preprocess
-    unless system "osmosis --read-xml #{@osm_file}.osm --write-pbf #{@osm_file}.osm.pbf omitmetadata=true >>#{PREPROCESS_LOG_FILE} 2>&1"
+    unless system "osmosis --read-xml #{osm_file}.osm --write-pbf #{osm_file}.osm.pbf omitmetadata=true >>#{PREPROCESS_LOG_FILE} 2>&1"
       raise OsmosisError.new $?, "osmosis exited with code #{$?.exitstatus}"
     end
     log '', :preprocess
@@ -227,25 +241,26 @@ end
 
 def extracted?
   Dir.chdir TEST_FOLDER do
-    File.exist?("#{@osm_file}.osrm") &&
+    File.exist?("#{extracted_file}.osrm") &&
-    File.exist?("#{@osm_file}.osrm.names") &&
+    File.exist?("#{extracted_file}.osrm.names") &&
-    File.exist?("#{@osm_file}.osrm.restrictions")
+    File.exist?("#{extracted_file}.osrm.restrictions")
   end
 end
 
 def prepared?
   Dir.chdir TEST_FOLDER do
-    File.exist?("#{@osm_file}.osrm.hsgr")
+    File.exist?("#{prepared_file}.osrm.hsgr")
   end
 end
 
 def write_timestamp
-  File.open( "#{@osm_file}.osrm.timestamp", 'w') {|f| f.write(OSM_TIMESTAMP) }
+  File.open( "#{prepared_file}.osrm.timestamp", 'w') {|f| f.write(OSM_TIMESTAMP) }
 end
 
 def pbf?
   input_format=='pbf'
 end
+
 def write_input_data
   Dir.chdir TEST_FOLDER do
     write_osm
@@ -257,23 +272,43 @@ end
 def extract_data
   Dir.chdir TEST_FOLDER do
     log_preprocess_info
-    log "== Extracting #{@osm_file}.osm...", :preprocess
+    log "== Extracting #{osm_file}.osm...", :preprocess
-    unless system "#{BIN_PATH}/osrm-extract #{@osm_file}.osm#{'.pbf' if pbf?} --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1"
+    unless system "#{BIN_PATH}/osrm-extract #{osm_file}.osm#{'.pbf' if pbf?} --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1"
       log "*** Exited with code #{$?.exitstatus}.", :preprocess
       raise ExtractError.new $?.exitstatus, "osrm-extract exited with code #{$?.exitstatus}."
     end
-    log '', :preprocess
+    begin
+      ["osrm","osrm.names","osrm.restrictions"].each do |file|
+        File.rename "#{osm_file}.#{file}", "#{extracted_file}.#{file}"
+      end
+    rescue Exception => e
+      raise FileError.new nil, "failed to rename data file after extracting."
+    end
   end
 end
 
 def prepare_data
   Dir.chdir TEST_FOLDER do
     log_preprocess_info
-    log "== Preparing #{@osm_file}.osm...", :preprocess
+    log "== Preparing #{extracted_file}.osm...", :preprocess
-    unless system "#{BIN_PATH}/osrm-prepare #{@osm_file}.osrm  --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1"
+    unless system "#{BIN_PATH}/osrm-prepare #{extracted_file}.osrm  --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1"
       log "*** Exited with code #{$?.exitstatus}.", :preprocess
       raise PrepareError.new $?.exitstatus, "osrm-prepare exited with code #{$?.exitstatus}."
     end
+    begin
+      ["osrm.hsgr","osrm.fileIndex","osrm.geometry","osrm.nodes","osrm.ramIndex"].each do |file|
+        File.rename "#{extracted_file}.#{file}", "#{prepared_file}.#{file}"
+      end
+    rescue Exception => e
+      raise FileError.new nil, "failed to rename data file after preparing."
+    end
+    begin
+      ["osrm.names","osrm.edges","osrm.restrictions"].each do |file|
+        FileUtils.cp "#{extracted_file}.#{file}", "#{prepared_file}.#{file}"
+      end
+    rescue Exception => e
+      raise FileError.new nil, "failed to copy data file after preparing."
+    end
     log '', :preprocess
   end
 end

features/support/exceptions.rb

@@ -31,6 +31,12 @@ class OSRMError < StandardError
   end
 end
 
+class FileError < OSRMError
+  def initialize code, msg
+    super 'fileutil', code, msg, PREPROCESS_LOG_FILE, 5
+  end
+end
+
 class OsmosisError < OSRMError
   def initialize code, msg
     super 'osmosis', code, msg, PREPROCESS_LOG_FILE, 40

features/support/hash.rb

@@ -32,18 +32,32 @@ def lua_lib_hash
 end
 
 def bin_extract_hash
-  bin_extract_hash ||= hash_of_files "#{BIN_PATH}/osrm-extract#{EXE}"
+  @bin_extract_hash ||= hash_of_files "#{BIN_PATH}/osrm-extract#{EXE}"
+  @bin_extract_hash
 end
 
 def bin_prepare_hash
-  bin_prepare_hash ||= hash_of_files "#{BIN_PATH}/osrm-prepare#{EXE}"
+  @bin_prepare_hash ||= hash_of_files "#{BIN_PATH}/osrm-prepare#{EXE}"
 end
 
 def bin_routed_hash
-  bin_routed_hash ||= hash_of_files "#{BIN_PATH}/osrm-routed#{EXE}"
+  @bin_routed_hash ||= hash_of_files "#{BIN_PATH}/osrm-routed#{EXE}"
 end
 
-#combine state of data, profile and binaries into a hash that identifies the exact test scenario
+# combine state of data, profile and binaries into a hashes that identifies
-def fingerprint
+# the exact test situation at different stages, so we can later skip steps when possible.
-  @fingerprint ||= Digest::SHA1.hexdigest "#{bin_extract_hash}-#{bin_prepare_hash}-#{bin_routed_hash}-#{profile_hash}-#{lua_lib_hash}-#{osm_hash}"
+def fingerprint_osm
+  @fingerprint_osm ||= Digest::SHA1.hexdigest "#{osm_hash}"
+end
+
+def fingerprint_extract
+  @fingerprint_extract ||= Digest::SHA1.hexdigest "#{profile_hash}-#{lua_lib_hash}-#{bin_extract_hash}"
+end
+
+def fingerprint_prepare
+  @fingerprint_prepare ||= Digest::SHA1.hexdigest "#{bin_prepare_hash}"
+end
+
+def fingerprint_route
+  @fingerprint_route ||= Digest::SHA1.hexdigest "#{bin_routed_hash}"
 end

features/support/log.rb

@@ -29,7 +29,10 @@ def log_scenario_fail_info
   log "========================================="
   log "Failed scenario: #{@scenario_title}"
   log "Time: #{@scenario_time}"
-  log "Fingerprint: #{@fingerprint}"
+  log "Fingerprint osm stage: #{@fingerprint_osm}"
+  log "Fingerprint extract stage: #{@fingerprint_extract}"
+  log "Fingerprint prepare stage: #{@fingerprint_prepare}"
+  log "Fingerprint route stage: #{@fingerprint_route}"
   log "Profile: #{@profile}"
   log
   log '```xml' #so output can be posted directly to github comment fields
@@ -41,6 +44,7 @@ def log_scenario_fail_info
 end
 
 def log_fail expected,got,attempts
+  return
   log_scenario_fail_info
   log "== "
   log "Expected: #{expected}"

features/support/run.rb

@@ -2,11 +2,20 @@ def run_bin bin, options
   Dir.chdir TEST_FOLDER do
     opt = options.dup
 
-    if opt.include? '{base}'
+    if opt.include? '{osm_base}'
-      raise "*** {base} is missing" unless @osm_file
+      raise "*** {osm_base} is missing" unless osm_file
-      opt.gsub! "{base}", "#{@osm_file}" 
+      opt.gsub! "{osm_base}", "#{osm_file}" 
     end
 
+    if opt.include? '{extracted_base}'
+      raise "*** {extracted_base} is missing" unless extracted_file
+      opt.gsub! "{extracted_base}", "#{extracted_file}" 
+    end
+
+    if opt.include? '{prepared_base}'
+      raise "*** {prepared_base} is missing" unless prepared_file
+      opt.gsub! "{prepared_base}", "#{prepared_file}" 
+    end
     if opt.include? '{profile}'
       opt.gsub! "{profile}", "#{PROFILES_PATH}/#{@profile}.lua" 
     end