smarter caching of test files

features/step_definitions/locate.rb

@@ -1,7 +1,7 @@
 When /^I request locate I should get$/ do |table|
   reprocess
   actual = []
-  OSRMLoader.load(self,"#{@osm_file}.osrm") do
+  OSRMLoader.load(self,"#{prepared_file}.osrm") do
     table.hashes.each_with_index do |row,ri|
       in_node = find_node_by_name row['in']
       raise "*** unknown in-node '#{row['in']}" unless in_node

features/step_definitions/nearest.rb

@@ -1,7 +1,7 @@
 When /^I request nearest I should get$/ do |table|
   reprocess
   actual = []
-  OSRMLoader.load(self,"#{@osm_file}.osrm") do
+  OSRMLoader.load(self,"#{prepared_file}.osrm") do
     table.hashes.each_with_index do |row,ri|
       in_node = find_node_by_name row['in']
       raise "*** unknown in-node '#{row['in']}" unless in_node

features/step_definitions/requests.rb

@@ -1,6 +1,6 @@
 When /^I request \/(.*)$/ do |path|
   reprocess
-  OSRMLoader.load(self,"#{@osm_file}.osrm") do
+  OSRMLoader.load(self,"#{prepared_file}.osrm") do
     @response = request_path path
   end
 end

features/step_definitions/routability.rb

@@ -44,7 +44,7 @@ Then /^routability should be$/ do |table|
   if table.headers&["forw","backw","bothw"] == []
     raise "*** routability tabel must contain either 'forw', 'backw' or 'bothw' column"
   end
-  OSRMLoader.load(self,"#{@osm_file}.osrm") do
+  OSRMLoader.load(self,"#{prepared_file}.osrm") do
     table.hashes.each_with_index do |row,i|
       output_row = row.dup
       attempts = []

features/step_definitions/routing.rb

@@ -1,7 +1,7 @@
 When /^I route I should get$/ do |table|
   reprocess
   actual = []
-  OSRMLoader.load(self,"#{@osm_file}.osrm") do
+  OSRMLoader.load(self,"#{prepared_file}.osrm") do
     table.hashes.each_with_index do |row,ri|
       if row['request']
         got = {'request' => row['request'] }

features/support/data.rb

@@ -1,6 +1,7 @@
 require 'OSM/objects'       #osmlib gem
 require 'OSM/Database'
 require 'builder'
+require 'fileutils'
 
 class Location
   attr_accessor :lon,:lat
@@ -155,7 +156,10 @@ def reset_data
   end
   reset_profile
   reset_osm
-  @fingerprint = nil
+  @fingerprint_osm = nil
+  @fingerprint_extract = nil
+  @fingerprint_prepare = nil
+  @fingerprint_route = nil
 end
 
 def make_osm_id
@@ -205,20 +209,31 @@ def osm_str
   @osm_str
 end
 
+def osm_file
+  @osm_file ||= "#{DATA_FOLDER}/#{fingerprint_osm}"
+end
+
+def extracted_file
+  @extracted_file ||= "#{osm_file}_#{fingerprint_extract}"
+end
+
+def prepared_file
+  @prepared_file ||= "#{osm_file}_#{fingerprint_extract}_#{fingerprint_prepare}"
+end
+
 def write_osm
-  #write .oms file if needed
   Dir.mkdir DATA_FOLDER unless File.exist? DATA_FOLDER
-  @osm_file = "#{DATA_FOLDER}/#{sanitized_scenario_title}_#{fingerprint}"
+  unless File.exist?("#{osm_file}.osm")
-  unless File.exist?("#{@osm_file}.osm")
+    puts "write osm data"
-    File.open( "#{@osm_file}.osm", 'w') {|f| f.write(osm_str) }
+    File.open( "#{osm_file}.osm", 'w') {|f| f.write(osm_str) }
   end
 end
 
 def convert_osm_to_pbf
-  unless File.exist?("#{@osm_file}.osm.pbf")
+  unless File.exist?("#{osm_file}.osm.pbf")
     log_preprocess_info
-    log "== Converting #{@osm_file}.osm to protobuffer format...", :preprocess
+    log "== Converting #{osm_file}.osm to protobuffer format...", :preprocess
-    unless system "osmosis --read-xml #{@osm_file}.osm --write-pbf #{@osm_file}.osm.pbf omitmetadata=true >>#{PREPROCESS_LOG_FILE} 2>&1"
+    unless system "osmosis --read-xml #{osm_file}.osm --write-pbf #{osm_file}.osm.pbf omitmetadata=true >>#{PREPROCESS_LOG_FILE} 2>&1"
       raise OsmosisError.new $?, "osmosis exited with code #{$?.exitstatus}"
     end
     log '', :preprocess
@@ -227,25 +242,26 @@ end
 
 def extracted?
   Dir.chdir TEST_FOLDER do
-    File.exist?("#{@osm_file}.osrm") &&
+    File.exist?("#{extracted_file}.osrm") &&
-    File.exist?("#{@osm_file}.osrm.names") &&
+    File.exist?("#{extracted_file}.osrm.names") &&
-    File.exist?("#{@osm_file}.osrm.restrictions")
+    File.exist?("#{extracted_file}.osrm.restrictions")
   end
 end
 
 def prepared?
   Dir.chdir TEST_FOLDER do
-    File.exist?("#{@osm_file}.osrm.hsgr")
+    File.exist?("#{prepared_file}.osrm.hsgr")
   end
 end
 
 def write_timestamp
-  File.open( "#{@osm_file}.osrm.timestamp", 'w') {|f| f.write(OSM_TIMESTAMP) }
+  File.open( "#{prepared_file}.osrm.timestamp", 'w') {|f| f.write(OSM_TIMESTAMP) }
 end
 
 def pbf?
   input_format=='pbf'
 end
+
 def write_input_data
   Dir.chdir TEST_FOLDER do
     write_osm
@@ -255,10 +271,16 @@ def write_input_data
 end
 
 def extract_data
+  puts "extract"
   Dir.chdir TEST_FOLDER do
     log_preprocess_info
-    log "== Extracting #{@osm_file}.osm...", :preprocess
+    log "== Extracting #{osm_file}.osm...", :preprocess
-    unless system "#{BIN_PATH}/osrm-extract #{@osm_file}.osm#{'.pbf' if pbf?} --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1"
+    begin
+     FileUtils.cp "#{osm_file}.osm#{'.pbf' if pbf?}", "#{extracted_file}.osm#{'.pbf' if pbf?}"   
+    rescue Exception => e
+      raise FileError.new nil, "failed to copy data file from #{osm_file}.osm#{'.pbf' if pbf?} to #{extracted_file}.osm#{'.pbf' if pbf?}."
+    end
+    unless system "#{BIN_PATH}/osrm-extract #{extracted_file}.osm#{'.pbf' if pbf?} --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1"
       log "*** Exited with code #{$?.exitstatus}.", :preprocess
       raise ExtractError.new $?.exitstatus, "osrm-extract exited with code #{$?.exitstatus}."
     end
@@ -267,10 +289,19 @@ def extract_data
 end
 
 def prepare_data
+  puts "prepare"
   Dir.chdir TEST_FOLDER do
     log_preprocess_info
-    log "== Preparing #{@osm_file}.osm...", :preprocess
+    log "== Preparing #{extracted_file}.osm...", :preprocess
-    unless system "#{BIN_PATH}/osrm-prepare #{@osm_file}.osrm  --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1"
+    begin
+     FileUtils.cp "#{extracted_file}.osm#{'.pbf' if pbf?}", "#{prepared_file}.osm#{'.pbf' if pbf?}"   
+     FileUtils.cp "#{extracted_file}.osrm", "#{prepared_file}.osrm"   
+     FileUtils.cp "#{extracted_file}.osrm.names", "#{prepared_file}.osrm.names"   
+     FileUtils.cp "#{extracted_file}.osrm.restrictions", "#{prepared_file}.osrm.restrictions"   
+    rescue Exception => e
+      raise FileError.new nil, "failed to copy data file."
+    end
+    unless system "#{BIN_PATH}/osrm-prepare #{prepared_file}.osrm  --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1"
       log "*** Exited with code #{$?.exitstatus}.", :preprocess
       raise PrepareError.new $?.exitstatus, "osrm-prepare exited with code #{$?.exitstatus}."
     end
@@ -279,6 +310,17 @@ def prepare_data
 end
 
 def reprocess
+  if false
+    puts fingerprint_osm
+    puts fingerprint_extract
+    puts fingerprint_prepare
+    puts fingerprint_route
+    
+    puts osm_file
+    puts extracted_file
+    puts prepared_file
+  end
+
   write_input_data
   extract_data unless extracted?
   prepare_data unless prepared?

features/support/exceptions.rb

@@ -31,6 +31,12 @@ class OSRMError < StandardError
   end
 end
 
+class FileError < OSRMError
+  def initialize code, msg
+    super 'fileutil', code, msg, PREPROCESS_LOG_FILE, 5
+  end
+end
+
 class OsmosisError < OSRMError
   def initialize code, msg
     super 'osmosis', code, msg, PREPROCESS_LOG_FILE, 40

features/support/hash.rb

@@ -32,18 +32,32 @@ def lua_lib_hash
 end
 
 def bin_extract_hash
-  bin_extract_hash ||= hash_of_files "#{BIN_PATH}/osrm-extract#{EXE}"
+  @bin_extract_hash ||= hash_of_files "#{BIN_PATH}/osrm-extract#{EXE}"
+  @bin_extract_hash
 end
 
 def bin_prepare_hash
-  bin_prepare_hash ||= hash_of_files "#{BIN_PATH}/osrm-prepare#{EXE}"
+  @bin_prepare_hash ||= hash_of_files "#{BIN_PATH}/osrm-prepare#{EXE}"
 end
 
 def bin_routed_hash
-  bin_routed_hash ||= hash_of_files "#{BIN_PATH}/osrm-routed#{EXE}"
+  @bin_routed_hash ||= hash_of_files "#{BIN_PATH}/osrm-routed#{EXE}"
 end
 
-#combine state of data, profile and binaries into a hash that identifies the exact test scenario
+# combine state of data, profile and binaries into a hashes that identifies
-def fingerprint
+# the exact test situation at different stages, so we can later skip steps when possible.
-  @fingerprint ||= Digest::SHA1.hexdigest "#{bin_extract_hash}-#{bin_prepare_hash}-#{bin_routed_hash}-#{profile_hash}-#{lua_lib_hash}-#{osm_hash}"
+def fingerprint_osm
+  @fingerprint_osm ||= Digest::SHA1.hexdigest "#{osm_hash}"
+end
+
+def fingerprint_extract
+  @fingerprint_extract ||= Digest::SHA1.hexdigest "#{profile_hash}-#{lua_lib_hash}-#{bin_extract_hash}"
+end
+
+def fingerprint_prepare
+  @fingerprint_prepare ||= Digest::SHA1.hexdigest "#{bin_prepare_hash}"
+end
+
+def fingerprint_route
+  @fingerprint_route ||= Digest::SHA1.hexdigest "#{bin_routed_hash}"
 end

features/support/log.rb

@@ -29,7 +29,10 @@ def log_scenario_fail_info
   log "========================================="
   log "Failed scenario: #{@scenario_title}"
   log "Time: #{@scenario_time}"
-  log "Fingerprint: #{@fingerprint}"
+  log "Fingerprint osm stage: #{@fingerprint_osm}"
+  log "Fingerprint extract stage: #{@fingerprint_extract}"
+  log "Fingerprint prepare stage: #{@fingerprint_prepare}"
+  log "Fingerprint route stage: #{@fingerprint_route}"
   log "Profile: #{@profile}"
   log
   log '```xml' #so output can be posted directly to github comment fields
@@ -41,12 +44,15 @@ def log_scenario_fail_info
 end
 
 def log_fail expected,got,attempts
+  return
   log_scenario_fail_info
   log "== "
   log "Expected: #{expected}"
   log "Got:      #{got}"
   log
   ['route','forw','backw'].each do |direction|
+    p attempts
+    p direction
     if attempts[direction]
       attempts[direction]
       log "Direction: #{direction}"

features/support/run.rb

@@ -3,8 +3,8 @@ def run_bin bin, options
     opt = options.dup
     
     if opt.include? '{base}'
-      raise "*** {base} is missing" unless @osm_file
+      raise "*** {base} is missing" unless prepared_file
-      opt.gsub! "{base}", "#{@osm_file}" 
+      opt.gsub! "{base}", "#{prepared_file}" 
     end
 
     if opt.include? '{profile}'