Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

smarter caching of test files

Browse Source
This commit is contained in:
Emil Tin 2014-10-17 10:47:14 +02:00
parent 8eccfaa034
commit 1f4241a63d
10 changed files with 99 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
features/step_definitions/locate.rb
View File

@ -1,7 +1,7 @@
When /^I request locate I should get$/ do |table|
reprocess
actual = []
OSRMLoader.load(self,"#{@osm_file}.osrm") do
OSRMLoader.load(self,"#{prepared_file}.osrm") do
table.hashes.each_with_index do |row,ri|
in_node = find_node_by_name row['in']
raise "*** unknown in-node '#{row['in']}" unless in_node

2
features/step_definitions/nearest.rb
View File

@ -1,7 +1,7 @@
When /^I request nearest I should get$/ do |table|
reprocess
actual = []
OSRMLoader.load(self,"#{@osm_file}.osrm") do
OSRMLoader.load(self,"#{prepared_file}.osrm") do
table.hashes.each_with_index do |row,ri|
in_node = find_node_by_name row['in']
raise "*** unknown in-node '#{row['in']}" unless in_node

2
features/step_definitions/requests.rb
View File

@ -1,6 +1,6 @@
When /^I request \/(.*)$/ do |path|
reprocess
OSRMLoader.load(self,"#{@osm_file}.osrm") do
OSRMLoader.load(self,"#{prepared_file}.osrm") do
@response = request_path path
end
end

2
features/step_definitions/routability.rb
View File

@ -44,7 +44,7 @@ Then /^routability should be$/ do |table|
if table.headers&["forw","backw","bothw"] == []
raise "*** routability tabel must contain either 'forw', 'backw' or 'bothw' column"
end
OSRMLoader.load(self,"#{@osm_file}.osrm") do
OSRMLoader.load(self,"#{prepared_file}.osrm") do
table.hashes.each_with_index do |row,i|
output_row = row.dup
attempts = []

2
features/step_definitions/routing.rb
View File

@ -1,7 +1,7 @@
When /^I route I should get$/ do |table|
reprocess
actual = []
OSRMLoader.load(self,"#{@osm_file}.osrm") do
OSRMLoader.load(self,"#{prepared_file}.osrm") do
table.hashes.each_with_index do |row,ri|
if row['request']
got = {'request' => row['request'] }

76
features/support/data.rb
View File

@ -1,6 +1,7 @@
require 'OSM/objects' #osmlib gem
require 'OSM/Database'
require 'builder'
require 'fileutils'
class Location
attr_accessor :lon,:lat
@ -155,7 +156,10 @@ def reset_data
end
reset_profile
reset_osm
@fingerprint = nil
@fingerprint_osm = nil
@fingerprint_extract = nil
@fingerprint_prepare = nil
@fingerprint_route = nil
end
def make_osm_id
@ -205,20 +209,31 @@ def osm_str
@osm_str
end
def osm_file
@osm_file ||= "#{DATA_FOLDER}/#{fingerprint_osm}"
end
def extracted_file
@extracted_file ||= "#{osm_file}_#{fingerprint_extract}"
end
def prepared_file
@prepared_file ||= "#{osm_file}_#{fingerprint_extract}_#{fingerprint_prepare}"
end
def write_osm
#write .oms file if needed
Dir.mkdir DATA_FOLDER unless File.exist? DATA_FOLDER
@osm_file = "#{DATA_FOLDER}/#{sanitized_scenario_title}_#{fingerprint}"
unless File.exist?("#{@osm_file}.osm")
File.open( "#{@osm_file}.osm", 'w') {|f| f.write(osm_str) }
unless File.exist?("#{osm_file}.osm")
puts "write osm data"
File.open( "#{osm_file}.osm", 'w') {|f| f.write(osm_str) }
end
end
def convert_osm_to_pbf
unless File.exist?("#{@osm_file}.osm.pbf")
unless File.exist?("#{osm_file}.osm.pbf")
log_preprocess_info
log "== Converting #{@osm_file}.osm to protobuffer format...", :preprocess
unless system "osmosis --read-xml #{@osm_file}.osm --write-pbf #{@osm_file}.osm.pbf omitmetadata=true >>#{PREPROCESS_LOG_FILE} 2>&1"
log "== Converting #{osm_file}.osm to protobuffer format...", :preprocess
unless system "osmosis --read-xml #{osm_file}.osm --write-pbf #{osm_file}.osm.pbf omitmetadata=true >>#{PREPROCESS_LOG_FILE} 2>&1"
raise OsmosisError.new $?, "osmosis exited with code #{$?.exitstatus}"
end
log '', :preprocess
@ -227,25 +242,26 @@ end
def extracted?
Dir.chdir TEST_FOLDER do
File.exist?("#{@osm_file}.osrm") &&
File.exist?("#{@osm_file}.osrm.names") &&
File.exist?("#{@osm_file}.osrm.restrictions")
File.exist?("#{extracted_file}.osrm") &&
File.exist?("#{extracted_file}.osrm.names") &&
File.exist?("#{extracted_file}.osrm.restrictions")
end
end
def prepared?
Dir.chdir TEST_FOLDER do
File.exist?("#{@osm_file}.osrm.hsgr")
File.exist?("#{prepared_file}.osrm.hsgr")
end
end
def write_timestamp
File.open( "#{@osm_file}.osrm.timestamp", 'w') {|f| f.write(OSM_TIMESTAMP) }
File.open( "#{prepared_file}.osrm.timestamp", 'w') {|f| f.write(OSM_TIMESTAMP) }
end
def pbf?
input_format=='pbf'
end
def write_input_data
Dir.chdir TEST_FOLDER do
write_osm
@ -255,10 +271,16 @@ def write_input_data
end
def extract_data
puts "extract"
Dir.chdir TEST_FOLDER do
log_preprocess_info
log "== Extracting #{@osm_file}.osm...", :preprocess
unless system "#{BIN_PATH}/osrm-extract #{@osm_file}.osm#{'.pbf' if pbf?} --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1"
log "== Extracting #{osm_file}.osm...", :preprocess
begin
FileUtils.cp "#{osm_file}.osm#{'.pbf' if pbf?}", "#{extracted_file}.osm#{'.pbf' if pbf?}"
rescue Exception => e
raise FileError.new nil, "failed to copy data file from #{osm_file}.osm#{'.pbf' if pbf?} to #{extracted_file}.osm#{'.pbf' if pbf?}."
end
unless system "#{BIN_PATH}/osrm-extract #{extracted_file}.osm#{'.pbf' if pbf?} --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1"
log "*** Exited with code #{$?.exitstatus}.", :preprocess
raise ExtractError.new $?.exitstatus, "osrm-extract exited with code #{$?.exitstatus}."
end
@ -267,10 +289,19 @@ def extract_data
end
def prepare_data
puts "prepare"
Dir.chdir TEST_FOLDER do
log_preprocess_info
log "== Preparing #{@osm_file}.osm...", :preprocess
unless system "#{BIN_PATH}/osrm-prepare #{@osm_file}.osrm --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1"
log "== Preparing #{extracted_file}.osm...", :preprocess
begin
FileUtils.cp "#{extracted_file}.osm#{'.pbf' if pbf?}", "#{prepared_file}.osm#{'.pbf' if pbf?}"
FileUtils.cp "#{extracted_file}.osrm", "#{prepared_file}.osrm"
FileUtils.cp "#{extracted_file}.osrm.names", "#{prepared_file}.osrm.names"
FileUtils.cp "#{extracted_file}.osrm.restrictions", "#{prepared_file}.osrm.restrictions"
rescue Exception => e
raise FileError.new nil, "failed to copy data file."
end
unless system "#{BIN_PATH}/osrm-prepare #{prepared_file}.osrm --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1"
log "*** Exited with code #{$?.exitstatus}.", :preprocess
raise PrepareError.new $?.exitstatus, "osrm-prepare exited with code #{$?.exitstatus}."
end
@ -279,6 +310,17 @@ def prepare_data
end
def reprocess
if false
puts fingerprint_osm
puts fingerprint_extract
puts fingerprint_prepare
puts fingerprint_route
puts osm_file
puts extracted_file
puts prepared_file
end
write_input_data
extract_data unless extracted?
prepare_data unless prepared?

6
features/support/exceptions.rb
View File

@ -31,6 +31,12 @@ class OSRMError < StandardError
end
end
class FileError < OSRMError
def initialize code, msg
super 'fileutil', code, msg, PREPROCESS_LOG_FILE, 5
end
end
class OsmosisError < OSRMError
def initialize code, msg
super 'osmosis', code, msg, PREPROCESS_LOG_FILE, 40

26
features/support/hash.rb
View File

@ -32,18 +32,32 @@ def lua_lib_hash
end
def bin_extract_hash
bin_extract_hash ||= hash_of_files "#{BIN_PATH}/osrm-extract#{EXE}"
@bin_extract_hash ||= hash_of_files "#{BIN_PATH}/osrm-extract#{EXE}"
@bin_extract_hash
end
def bin_prepare_hash
bin_prepare_hash ||= hash_of_files "#{BIN_PATH}/osrm-prepare#{EXE}"
@bin_prepare_hash ||= hash_of_files "#{BIN_PATH}/osrm-prepare#{EXE}"
end
def bin_routed_hash
bin_routed_hash ||= hash_of_files "#{BIN_PATH}/osrm-routed#{EXE}"
@bin_routed_hash ||= hash_of_files "#{BIN_PATH}/osrm-routed#{EXE}"
end
#combine state of data, profile and binaries into a hash that identifies the exact test scenario
def fingerprint
@fingerprint ||= Digest::SHA1.hexdigest "#{bin_extract_hash}-#{bin_prepare_hash}-#{bin_routed_hash}-#{profile_hash}-#{lua_lib_hash}-#{osm_hash}"
# combine state of data, profile and binaries into a hashes that identifies
# the exact test situation at different stages, so we can later skip steps when possible.
def fingerprint_osm
@fingerprint_osm ||= Digest::SHA1.hexdigest "#{osm_hash}"
end
def fingerprint_extract
@fingerprint_extract ||= Digest::SHA1.hexdigest "#{profile_hash}-#{lua_lib_hash}-#{bin_extract_hash}"
end
def fingerprint_prepare
@fingerprint_prepare ||= Digest::SHA1.hexdigest "#{bin_prepare_hash}"
end
def fingerprint_route
@fingerprint_route ||= Digest::SHA1.hexdigest "#{bin_routed_hash}"
end

8
features/support/log.rb
View File

@ -29,7 +29,10 @@ def log_scenario_fail_info
log "========================================="
log "Failed scenario: #{@scenario_title}"
log "Time: #{@scenario_time}"
log "Fingerprint: #{@fingerprint}"
log "Fingerprint osm stage: #{@fingerprint_osm}"
log "Fingerprint extract stage: #{@fingerprint_extract}"
log "Fingerprint prepare stage: #{@fingerprint_prepare}"
log "Fingerprint route stage: #{@fingerprint_route}"
log "Profile: #{@profile}"
log
log '```xml' #so output can be posted directly to github comment fields
@ -41,12 +44,15 @@ def log_scenario_fail_info
end
def log_fail expected,got,attempts
return
log_scenario_fail_info
log "== "
log "Expected: #{expected}"
log "Got: #{got}"
log
['route','forw','backw'].each do |direction|
p attempts
p direction
if attempts[direction]
attempts[direction]
log "Direction: #{direction}"

4
features/support/run.rb
View File

@ -3,8 +3,8 @@ def run_bin bin, options
opt = options.dup
if opt.include? '{base}'
raise "*** {base} is missing" unless @osm_file
opt.gsub! "{base}", "#{@osm_file}"
raise "*** {base} is missing" unless prepared_file
opt.gsub! "{base}", "#{prepared_file}"
end
if opt.include? '{profile}'