diff --git a/features/step_definitions/locate.rb b/features/step_definitions/locate.rb index ca04b7742..3e67be3c9 100644 --- a/features/step_definitions/locate.rb +++ b/features/step_definitions/locate.rb @@ -1,7 +1,7 @@ When /^I request locate I should get$/ do |table| reprocess actual = [] - OSRMLoader.load(self,"#{@osm_file}.osrm") do + OSRMLoader.load(self,"#{prepared_file}.osrm") do table.hashes.each_with_index do |row,ri| in_node = find_node_by_name row['in'] raise "*** unknown in-node '#{row['in']}" unless in_node diff --git a/features/step_definitions/nearest.rb b/features/step_definitions/nearest.rb index 3dd90dd55..532c1f0d7 100644 --- a/features/step_definitions/nearest.rb +++ b/features/step_definitions/nearest.rb @@ -1,7 +1,7 @@ When /^I request nearest I should get$/ do |table| reprocess actual = [] - OSRMLoader.load(self,"#{@osm_file}.osrm") do + OSRMLoader.load(self,"#{prepared_file}.osrm") do table.hashes.each_with_index do |row,ri| in_node = find_node_by_name row['in'] raise "*** unknown in-node '#{row['in']}" unless in_node diff --git a/features/step_definitions/requests.rb b/features/step_definitions/requests.rb index aec6ebbab..41257e5b9 100644 --- a/features/step_definitions/requests.rb +++ b/features/step_definitions/requests.rb @@ -1,6 +1,6 @@ When /^I request \/(.*)$/ do |path| reprocess - OSRMLoader.load(self,"#{@osm_file}.osrm") do + OSRMLoader.load(self,"#{prepared_file}.osrm") do @response = request_path path end end diff --git a/features/step_definitions/routability.rb b/features/step_definitions/routability.rb index 1496ad01f..dcbde5afa 100644 --- a/features/step_definitions/routability.rb +++ b/features/step_definitions/routability.rb @@ -44,7 +44,7 @@ Then /^routability should be$/ do |table| if table.headers&["forw","backw","bothw"] == [] raise "*** routability tabel must contain either 'forw', 'backw' or 'bothw' column" end - OSRMLoader.load(self,"#{@osm_file}.osrm") do + OSRMLoader.load(self,"#{prepared_file}.osrm") do table.hashes.each_with_index do |row,i| output_row = row.dup attempts = [] diff --git a/features/step_definitions/routing.rb b/features/step_definitions/routing.rb index adf2f7a7a..c25f5b7c8 100644 --- a/features/step_definitions/routing.rb +++ b/features/step_definitions/routing.rb @@ -1,7 +1,7 @@ When /^I route I should get$/ do |table| reprocess actual = [] - OSRMLoader.load(self,"#{@osm_file}.osrm") do + OSRMLoader.load(self,"#{prepared_file}.osrm") do table.hashes.each_with_index do |row,ri| if row['request'] got = {'request' => row['request'] } diff --git a/features/support/data.rb b/features/support/data.rb index adb619204..8bd2e5f3c 100644 --- a/features/support/data.rb +++ b/features/support/data.rb @@ -1,6 +1,7 @@ require 'OSM/objects' #osmlib gem require 'OSM/Database' require 'builder' +require 'fileutils' class Location attr_accessor :lon,:lat @@ -155,7 +156,10 @@ def reset_data end reset_profile reset_osm - @fingerprint = nil + @fingerprint_osm = nil + @fingerprint_extract = nil + @fingerprint_prepare = nil + @fingerprint_route = nil end def make_osm_id @@ -205,20 +209,31 @@ def osm_str @osm_str end +def osm_file + @osm_file ||= "#{DATA_FOLDER}/#{fingerprint_osm}" +end + +def extracted_file + @extracted_file ||= "#{osm_file}_#{fingerprint_extract}" +end + +def prepared_file + @prepared_file ||= "#{osm_file}_#{fingerprint_extract}_#{fingerprint_prepare}" +end + def write_osm - #write .oms file if needed Dir.mkdir DATA_FOLDER unless File.exist? DATA_FOLDER - @osm_file = "#{DATA_FOLDER}/#{sanitized_scenario_title}_#{fingerprint}" - unless File.exist?("#{@osm_file}.osm") - File.open( "#{@osm_file}.osm", 'w') {|f| f.write(osm_str) } + unless File.exist?("#{osm_file}.osm") + puts "write osm data" + File.open( "#{osm_file}.osm", 'w') {|f| f.write(osm_str) } end end def convert_osm_to_pbf - unless File.exist?("#{@osm_file}.osm.pbf") + unless File.exist?("#{osm_file}.osm.pbf") log_preprocess_info - log "== Converting #{@osm_file}.osm to protobuffer format...", :preprocess - unless system "osmosis --read-xml #{@osm_file}.osm --write-pbf #{@osm_file}.osm.pbf omitmetadata=true >>#{PREPROCESS_LOG_FILE} 2>&1" + log "== Converting #{osm_file}.osm to protobuffer format...", :preprocess + unless system "osmosis --read-xml #{osm_file}.osm --write-pbf #{osm_file}.osm.pbf omitmetadata=true >>#{PREPROCESS_LOG_FILE} 2>&1" raise OsmosisError.new $?, "osmosis exited with code #{$?.exitstatus}" end log '', :preprocess @@ -227,25 +242,26 @@ end def extracted? Dir.chdir TEST_FOLDER do - File.exist?("#{@osm_file}.osrm") && - File.exist?("#{@osm_file}.osrm.names") && - File.exist?("#{@osm_file}.osrm.restrictions") + File.exist?("#{extracted_file}.osrm") && + File.exist?("#{extracted_file}.osrm.names") && + File.exist?("#{extracted_file}.osrm.restrictions") end end def prepared? Dir.chdir TEST_FOLDER do - File.exist?("#{@osm_file}.osrm.hsgr") + File.exist?("#{prepared_file}.osrm.hsgr") end end def write_timestamp - File.open( "#{@osm_file}.osrm.timestamp", 'w') {|f| f.write(OSM_TIMESTAMP) } + File.open( "#{prepared_file}.osrm.timestamp", 'w') {|f| f.write(OSM_TIMESTAMP) } end def pbf? input_format=='pbf' end + def write_input_data Dir.chdir TEST_FOLDER do write_osm @@ -255,10 +271,16 @@ def write_input_data end def extract_data + puts "extract" Dir.chdir TEST_FOLDER do log_preprocess_info - log "== Extracting #{@osm_file}.osm...", :preprocess - unless system "#{BIN_PATH}/osrm-extract #{@osm_file}.osm#{'.pbf' if pbf?} --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1" + log "== Extracting #{osm_file}.osm...", :preprocess + begin + FileUtils.cp "#{osm_file}.osm#{'.pbf' if pbf?}", "#{extracted_file}.osm#{'.pbf' if pbf?}" + rescue Exception => e + raise FileError.new nil, "failed to copy data file from #{osm_file}.osm#{'.pbf' if pbf?} to #{extracted_file}.osm#{'.pbf' if pbf?}." + end + unless system "#{BIN_PATH}/osrm-extract #{extracted_file}.osm#{'.pbf' if pbf?} --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1" log "*** Exited with code #{$?.exitstatus}.", :preprocess raise ExtractError.new $?.exitstatus, "osrm-extract exited with code #{$?.exitstatus}." end @@ -267,10 +289,19 @@ def extract_data end def prepare_data + puts "prepare" Dir.chdir TEST_FOLDER do log_preprocess_info - log "== Preparing #{@osm_file}.osm...", :preprocess - unless system "#{BIN_PATH}/osrm-prepare #{@osm_file}.osrm --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1" + log "== Preparing #{extracted_file}.osm...", :preprocess + begin + FileUtils.cp "#{extracted_file}.osm#{'.pbf' if pbf?}", "#{prepared_file}.osm#{'.pbf' if pbf?}" + FileUtils.cp "#{extracted_file}.osrm", "#{prepared_file}.osrm" + FileUtils.cp "#{extracted_file}.osrm.names", "#{prepared_file}.osrm.names" + FileUtils.cp "#{extracted_file}.osrm.restrictions", "#{prepared_file}.osrm.restrictions" + rescue Exception => e + raise FileError.new nil, "failed to copy data file." + end + unless system "#{BIN_PATH}/osrm-prepare #{prepared_file}.osrm --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1" log "*** Exited with code #{$?.exitstatus}.", :preprocess raise PrepareError.new $?.exitstatus, "osrm-prepare exited with code #{$?.exitstatus}." end @@ -279,6 +310,17 @@ def prepare_data end def reprocess + if false + puts fingerprint_osm + puts fingerprint_extract + puts fingerprint_prepare + puts fingerprint_route + + puts osm_file + puts extracted_file + puts prepared_file + end + write_input_data extract_data unless extracted? prepare_data unless prepared? diff --git a/features/support/exceptions.rb b/features/support/exceptions.rb index 2901bfe6a..02f59d388 100644 --- a/features/support/exceptions.rb +++ b/features/support/exceptions.rb @@ -31,6 +31,12 @@ class OSRMError < StandardError end end +class FileError < OSRMError + def initialize code, msg + super 'fileutil', code, msg, PREPROCESS_LOG_FILE, 5 + end +end + class OsmosisError < OSRMError def initialize code, msg super 'osmosis', code, msg, PREPROCESS_LOG_FILE, 40 diff --git a/features/support/hash.rb b/features/support/hash.rb index 197cd426e..6980b6645 100644 --- a/features/support/hash.rb +++ b/features/support/hash.rb @@ -32,18 +32,32 @@ def lua_lib_hash end def bin_extract_hash - bin_extract_hash ||= hash_of_files "#{BIN_PATH}/osrm-extract#{EXE}" + @bin_extract_hash ||= hash_of_files "#{BIN_PATH}/osrm-extract#{EXE}" + @bin_extract_hash end def bin_prepare_hash - bin_prepare_hash ||= hash_of_files "#{BIN_PATH}/osrm-prepare#{EXE}" + @bin_prepare_hash ||= hash_of_files "#{BIN_PATH}/osrm-prepare#{EXE}" end def bin_routed_hash - bin_routed_hash ||= hash_of_files "#{BIN_PATH}/osrm-routed#{EXE}" + @bin_routed_hash ||= hash_of_files "#{BIN_PATH}/osrm-routed#{EXE}" end -#combine state of data, profile and binaries into a hash that identifies the exact test scenario -def fingerprint - @fingerprint ||= Digest::SHA1.hexdigest "#{bin_extract_hash}-#{bin_prepare_hash}-#{bin_routed_hash}-#{profile_hash}-#{lua_lib_hash}-#{osm_hash}" +# combine state of data, profile and binaries into a hashes that identifies +# the exact test situation at different stages, so we can later skip steps when possible. +def fingerprint_osm + @fingerprint_osm ||= Digest::SHA1.hexdigest "#{osm_hash}" end + +def fingerprint_extract + @fingerprint_extract ||= Digest::SHA1.hexdigest "#{profile_hash}-#{lua_lib_hash}-#{bin_extract_hash}" +end + +def fingerprint_prepare + @fingerprint_prepare ||= Digest::SHA1.hexdigest "#{bin_prepare_hash}" +end + +def fingerprint_route + @fingerprint_route ||= Digest::SHA1.hexdigest "#{bin_routed_hash}" +end \ No newline at end of file diff --git a/features/support/log.rb b/features/support/log.rb index e5d34cc26..540a07eda 100644 --- a/features/support/log.rb +++ b/features/support/log.rb @@ -29,7 +29,10 @@ def log_scenario_fail_info log "=========================================" log "Failed scenario: #{@scenario_title}" log "Time: #{@scenario_time}" - log "Fingerprint: #{@fingerprint}" + log "Fingerprint osm stage: #{@fingerprint_osm}" + log "Fingerprint extract stage: #{@fingerprint_extract}" + log "Fingerprint prepare stage: #{@fingerprint_prepare}" + log "Fingerprint route stage: #{@fingerprint_route}" log "Profile: #{@profile}" log log '```xml' #so output can be posted directly to github comment fields @@ -41,12 +44,15 @@ def log_scenario_fail_info end def log_fail expected,got,attempts + return log_scenario_fail_info log "== " log "Expected: #{expected}" log "Got: #{got}" log ['route','forw','backw'].each do |direction| + p attempts + p direction if attempts[direction] attempts[direction] log "Direction: #{direction}" diff --git a/features/support/run.rb b/features/support/run.rb index 1b52d6348..a62e45042 100644 --- a/features/support/run.rb +++ b/features/support/run.rb @@ -3,8 +3,8 @@ def run_bin bin, options opt = options.dup if opt.include? '{base}' - raise "*** {base} is missing" unless @osm_file - opt.gsub! "{base}", "#{@osm_file}" + raise "*** {base} is missing" unless prepared_file + opt.gsub! "{base}", "#{prepared_file}" end if opt.include? '{profile}'