diff --git a/src/benchmarks/bench.cpp b/src/benchmarks/bench.cpp index 3114c078d..55737c09c 100644 --- a/src/benchmarks/bench.cpp +++ b/src/benchmarks/bench.cpp @@ -37,6 +37,8 @@ using namespace osrm; namespace { + + class GPSTraces { private: @@ -114,63 +116,119 @@ class GPSTraces } }; +// Struct to hold confidence interval data +struct ConfidenceInterval { + double mean; + double confidence; + double min; +}; + +// Helper function to calculate the bootstrap confidence interval +ConfidenceInterval confidenceInterval(const std::vector& data, int num_samples = 1000, double confidence_level = 0.95) { + std::vector means; + std::default_random_engine generator; + std::uniform_int_distribution distribution(0, data.size() - 1); + + for (int i = 0; i < num_samples; ++i) { + std::vector sample; + for (size_t j = 0; j < data.size(); ++j) { + sample.push_back(data[distribution(generator)]); + } + double sample_mean = std::accumulate(sample.begin(), sample.end(), 0.0) / sample.size(); + means.push_back(sample_mean); + } + + std::sort(means.begin(), means.end()); + double lower_bound = means[(int)((1 - confidence_level) / 2 * num_samples)]; + double upper_bound = means[(int)((1 + confidence_level) / 2 * num_samples)]; + double mean = std::accumulate(means.begin(), means.end(), 0.0) / means.size(); + + ConfidenceInterval ci = {mean, (upper_bound - lower_bound) / 2, *std::min_element(data.begin(), data.end())}; + return ci; +} + + class Statistics { public: - explicit Statistics(int iterations) : times(iterations) {} - void push(double timeMs, int iteration) - { + explicit Statistics(int iterations) : times(iterations) {} + + void push(double timeMs, int iteration) { times[iteration].push_back(timeMs); - sorted = false; } - double mean() { return sum() / times.size(); } - - double sum() - { - double sum = 0; - for (auto time : times[0]) - { - sum += time; + ConfidenceInterval mean() { + std::vector means; + for (const auto& iter_times : times) { + means.push_back(std::accumulate(iter_times.begin(), iter_times.end(), 0.0) / iter_times.size()); } - return sum; + return confidenceInterval(means); } - double min() { return *std::min_element(times[0].begin(), times[0].end()); } - - double max() { return *std::max_element(times[0].begin(), times[0].end()); } - - double percentile(double p) - { - const auto × = getTimes(); - return times[static_cast(p * times.size())]; + ConfidenceInterval total() { + std::vector sums; + for (const auto& iter_times : times) { + sums.push_back(std::accumulate(iter_times.begin(), iter_times.end(), 0.0)); + } + return confidenceInterval(sums); } + ConfidenceInterval min() { + std::vector mins; + for (const auto& iter_times : times) { + mins.push_back(*std::min_element(iter_times.begin(), iter_times.end())); + } + return confidenceInterval(mins); + } + + ConfidenceInterval max() { + std::vector maxs; + for (const auto& iter_times : times) { + maxs.push_back(*std::max_element(iter_times.begin(), iter_times.end())); + } + return confidenceInterval(maxs); + } + + ConfidenceInterval percentile(double p) { + std::vector percentiles; + for (const auto& iter_times : times) { + auto sorted_times = iter_times; + std::sort(sorted_times.begin(), sorted_times.end()); + percentiles.push_back(sorted_times[static_cast(p * sorted_times.size())]); + } + return confidenceInterval(percentiles); + } + + ConfidenceInterval ops_per_sec() { + std::vector ops; + for (const auto& iter_times : times) { + double total_time = std::accumulate(iter_times.begin(), iter_times.end(), 0.0) / 1000.0; + ops.push_back(iter_times.size() / total_time); + } + return confidenceInterval(ops); + } private: - std::vector getTimes() - { - if (!sorted) - { - std::sort(times[0].begin(), times[0].end()); - sorted = true; - } - return times[0]; - } - // vector of times for each iteration std::vector> times; - - bool sorted = false; }; -std::ostream &operator<<(std::ostream &os, Statistics &statistics) -{ +std::ostream& operator<<(std::ostream& os, Statistics& statistics) { os << std::fixed << std::setprecision(2); - os << "total: " << statistics.sum() << "ms" << std::endl; - os << "avg: " << statistics.mean() << "ms" << std::endl; - os << "min: " << statistics.min() << "ms" << std::endl; - os << "max: " << statistics.max() << "ms" << std::endl; - os << "p99: " << statistics.percentile(0.99) << "ms" << std::endl; + + ConfidenceInterval mean_ci = statistics.mean(); + ConfidenceInterval total_ci = statistics.total(); + ConfidenceInterval min_ci = statistics.min(); + ConfidenceInterval max_ci = statistics.max(); + ConfidenceInterval p99_ci = statistics.percentile(0.99); + ConfidenceInterval ops_ci = statistics.ops_per_sec(); + + os << "ops: " << ops_ci.mean << " ± " << ops_ci.confidence << " ops/s. " << "best: " << ops_ci.min << "ops/s." << std::endl; + os << "total: " << total_ci.mean << " ± " << total_ci.confidence << "ms. " << "best: " << total_ci.min << "ms." << std::endl; + os << "avg: " << mean_ci.mean << " ± " << mean_ci.confidence << "ms" << std::endl; + os << "min: " << min_ci.mean << " ± " << min_ci.confidence << "ms" << std::endl; + os << "max: " << max_ci.mean << " ± " << max_ci.confidence << "ms" << std::endl; + os << "p99: " << p99_ci.mean << " ± " << p99_ci.confidence << "ms" << std::endl; + return os; }