Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Fix calculation of best ops/sec in benchmarks (#6973)

Browse Source
This commit is contained in:
Siarhei Fedartsou 2024-06-29 10:31:23 +02:00 committed by GitHub
parent 0e17869e21
commit cacb1b23f9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scripts/ci/e2e_benchmark.py
View File

@ -86,9 +86,9 @@ def bootstrap_confidence_interval(data, num_samples=1000, confidence_level=0.95)
mean = np.mean(means) mean = np.mean(means)
return mean, lower_bound, upper_bound return mean, lower_bound, upper_bound
def calculate_confidence_interval(data): def calculate_confidence_interval(data, min_is_best=True):
mean, lower, upper = bootstrap_confidence_interval(data) mean, lower, upper = bootstrap_confidence_interval(data)
min_value = np.min(data) min_value = np.min(data) if min_is_best else np.max(data)
return mean, (upper - lower) / 2, min_value return mean, (upper - lower) / 2, min_value
@ -117,7 +117,7 @@ def main():
total_time, total_ci, total_best = calculate_confidence_interval(np.sum(all_times, axis=1)) total_time, total_ci, total_best = calculate_confidence_interval(np.sum(all_times, axis=1))
ops_per_sec, ops_per_sec_ci, ops_per_sec_best = calculate_confidence_interval(float(all_times.shape[1]) / np.sum(all_times / 1000, axis=1)) ops_per_sec, ops_per_sec_ci, ops_per_sec_best = calculate_confidence_interval(float(all_times.shape[1]) / np.sum(all_times / 1000, axis=1), min_is_best=False)
min_time, min_ci, _ = calculate_confidence_interval(np.min(all_times, axis=1)) min_time, min_ci, _ = calculate_confidence_interval(np.min(all_times, axis=1))
mean_time, mean_ci, _ = calculate_confidence_interval(np.mean(all_times, axis=1)) mean_time, mean_ci, _ = calculate_confidence_interval(np.mean(all_times, axis=1))
median_time, median_ci, _ = calculate_confidence_interval(np.median(all_times, axis=1)) median_time, median_ci, _ = calculate_confidence_interval(np.median(all_times, axis=1))

9
src/benchmarks/bench.cpp
View File

@ -134,6 +134,7 @@ struct ConfidenceInterval
double mean; double mean;
double confidence; double confidence;
double min; double min;
double max;
}; };
// Helper function to calculate the bootstrap confidence interval // Helper function to calculate the bootstrap confidence interval
@ -161,8 +162,10 @@ ConfidenceInterval confidenceInterval(const std::vector<double> &data,
double upper_bound = means[(int)((1 + confidence_level) / 2 * num_samples)]; double upper_bound = means[(int)((1 + confidence_level) / 2 * num_samples)];
double mean = std::accumulate(means.begin(), means.end(), 0.0) / means.size(); double mean = std::accumulate(means.begin(), means.end(), 0.0) / means.size();
ConfidenceInterval ci = { ConfidenceInterval ci = {mean,
mean, (upper_bound - lower_bound) / 2, *std::min_element(data.begin(), data.end())}; (upper_bound - lower_bound) / 2,
*std::min_element(data.begin(), data.end()),
*std::max_element(data.begin(), data.end())};
return ci; return ci;
} }
@ -260,7 +263,7 @@ std::ostream &operator<<(std::ostream &os, Statistics &statistics)
ConfidenceInterval ops_ci = statistics.ops_per_sec(); ConfidenceInterval ops_ci = statistics.ops_per_sec();
os << "ops: " << ops_ci.mean << " ± " << ops_ci.confidence << " ops/s. " os << "ops: " << ops_ci.mean << " ± " << ops_ci.confidence << " ops/s. "
<< "best: " << ops_ci.min << "ops/s." << std::endl; << "best: " << ops_ci.max << "ops/s." << std::endl;
os << "total: " << total_ci.mean << " ± " << total_ci.confidence << "ms. " os << "total: " << total_ci.mean << " ± " << total_ci.confidence << "ms. "
<< "best: " << total_ci.min << "ms." << std::endl; << "best: " << total_ci.min << "ms." << std::endl;
os << "avg: " << mean_ci.mean << " ± " << mean_ci.confidence << "ms" << std::endl; os << "avg: " << mean_ci.mean << " ± " << mean_ci.confidence << "ms" << std::endl;