Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

check for invalid parameter, coverity issue 1198843, also reformat source

Browse Source
This commit is contained in:
Dennis Luxen 2014-04-09 12:03:19 -04:00
parent 3a1a51ac46
commit 9894f2e053
1 changed files with 142 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

305
Tools/io-benchmark.cpp
View File

@ -48,45 +48,40 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
const unsigned number_of_elements = 268435456; const unsigned number_of_elements = 268435456;
struct Statistics { double min, max, med, mean, dev; }; struct Statistics
{
double min, max, med, mean, dev;
};
void RunStatistics(std::vector<double> & timings_vector, Statistics & stats) { void RunStatistics(std::vector<double> &timings_vector, Statistics &stats)
{
std::sort(timings_vector.begin(), timings_vector.end()); std::sort(timings_vector.begin(), timings_vector.end());
stats.min = timings_vector.front(); stats.min = timings_vector.front();
stats.max = timings_vector.back(); stats.max = timings_vector.back();
stats.med = timings_vector[timings_vector.size()/2]; stats.med = timings_vector[timings_vector.size() / 2];
double primary_sum = std::accumulate( double primary_sum = std::accumulate(timings_vector.begin(), timings_vector.end(), 0.0);
timings_vector.begin(),
timings_vector.end(),
0.0
);
stats.mean = primary_sum / timings_vector.size(); stats.mean = primary_sum / timings_vector.size();
double primary_sq_sum = std::inner_product( timings_vector.begin(), double primary_sq_sum = std::inner_product(
timings_vector.end(), timings_vector.begin(), timings_vector.end(), timings_vector.begin(), 0.0);
timings_vector.begin(), stats.dev = std::sqrt(primary_sq_sum / timings_vector.size() - (stats.mean * stats.mean));
0.0
);
stats.dev = std::sqrt(
primary_sq_sum / timings_vector.size() - (stats.mean * stats.mean)
);
} }
int main (int argc, char * argv[]) { int main(int argc, char *argv[])
{
LogPolicy::GetInstance().Unmute(); LogPolicy::GetInstance().Unmute();
SimpleLogger().Write() << SimpleLogger().Write() << "starting up engines, " << g_GIT_DESCRIPTION << ", "
"starting up engines, " << g_GIT_DESCRIPTION << ", " << << "compiled at " << __DATE__ << ", " __TIME__;
"compiled at " << __DATE__ << ", " __TIME__;
#ifdef __FreeBSD__ #ifdef __FreeBSD__
SimpleLogger().Write() << "Not supported on FreeBSD"; SimpleLogger().Write() << "Not supported on FreeBSD";
return 0; return 0;
#endif #endif
if( 1 == argc ) { if (1 == argc)
SimpleLogger().Write(logWARNING) << {
"usage: " << argv[0] << " /path/on/device"; SimpleLogger().Write(logWARNING) << "usage: " << argv[0] << " /path/on/device";
return -1; return -1;
} }
@ -94,119 +89,105 @@ int main (int argc, char * argv[]) {
test_path /= "osrm.tst"; test_path /= "osrm.tst";
SimpleLogger().Write(logDEBUG) << "temporary file: " << test_path.string(); SimpleLogger().Write(logDEBUG) << "temporary file: " << test_path.string();
try { try
//create files for testing {
if( 2 == argc) { // create files for testing
//create file to test if (2 == argc)
if( boost::filesystem::exists(test_path) ) { {
// create file to test
if (boost::filesystem::exists(test_path))
{
throw OSRMException("Data file already exists"); throw OSRMException("Data file already exists");
} }
double time1, time2; double time1, time2;
int * random_array = new int[number_of_elements]; int *random_array = new int[number_of_elements];
std::generate ( std::generate(random_array, random_array + number_of_elements, std::rand);
random_array,
random_array+number_of_elements,
std::rand
);
#ifdef __APPLE__ #ifdef __APPLE__
FILE * fd = fopen(test_path.string().c_str(), "w"); FILE *fd = fopen(test_path.string().c_str(), "w");
fcntl(fileno(fd), F_NOCACHE, 1); fcntl(fileno(fd), F_NOCACHE, 1);
fcntl(fileno(fd), F_RDAHEAD, 0); fcntl(fileno(fd), F_RDAHEAD, 0);
time1 = get_timestamp(); time1 = get_timestamp();
write( write(fileno(fd), (char *)random_array, number_of_elements * sizeof(unsigned));
fileno(fd),
(char*)random_array,
number_of_elements*sizeof(unsigned)
);
time2 = get_timestamp(); time2 = get_timestamp();
fclose(fd); fclose(fd);
#endif #endif
#ifdef __linux__ #ifdef __linux__
int f = open( int f =
test_path.string().c_str(), open(test_path.string().c_str(), O_CREAT | O_TRUNC | O_WRONLY | O_SYNC, S_IRWXU);
O_CREAT|O_TRUNC|O_WRONLY|O_SYNC,
S_IRWXU
);
time1 = get_timestamp(); time1 = get_timestamp();
int ret = write( int ret = write(f, random_array, number_of_elements * sizeof(unsigned));
f, if (-1 == ret)
random_array, {
number_of_elements*sizeof(unsigned)
);
if(-1 == ret) {
throw OSRMException("could not write random data file"); throw OSRMException("could not write random data file");
} }
time2 = get_timestamp(); time2 = get_timestamp();
close(f); close(f);
#endif #endif
delete[] random_array; delete[] random_array;
SimpleLogger().Write(logDEBUG) << SimpleLogger().Write(logDEBUG) << "writing raw 1GB took " << (time2 - time1) * 1000
"writing raw 1GB took " << (time2-time1)*1000 << "ms"; << "ms";
SimpleLogger().Write() << "raw write performance: " << SimpleLogger().Write() << "raw write performance: " << std::setprecision(5)
std::setprecision(5) << std::fixed << << std::fixed << 1024 * 1024 / ((time2 - time1) * 1000)
1024*1024/((time2-time1)*1000) << "MB/sec"; << "MB/sec";
SimpleLogger().Write(logDEBUG) << SimpleLogger().Write(logDEBUG)
"finished creation of random data. Flush disk cache now!"; << "finished creation of random data. Flush disk cache now!";
}
} else { else
{
// //
// Run Non-Cached I/O benchmarks // Run Non-Cached I/O benchmarks
// //
if( !boost::filesystem::exists(test_path) ) { if (!boost::filesystem::exists(test_path))
{
throw OSRMException("data file does not exist"); throw OSRMException("data file does not exist");
} }
double time1, time2; double time1, time2;
//volatiles do not get optimized // volatiles do not get optimized
Statistics stats; Statistics stats;
#ifdef __APPLE__ #ifdef __APPLE__
volatile unsigned single_block[1024]; volatile unsigned single_block[1024];
char * raw_array = new char[number_of_elements*sizeof(unsigned)]; char *raw_array = new char[number_of_elements * sizeof(unsigned)];
FILE * fd = fopen(test_path.string().c_str(), "r"); FILE *fd = fopen(test_path.string().c_str(), "r");
fcntl(fileno(fd), F_NOCACHE, 1); fcntl(fileno(fd), F_NOCACHE, 1);
fcntl(fileno(fd), F_RDAHEAD, 0); fcntl(fileno(fd), F_RDAHEAD, 0);
#endif #endif
#ifdef __linux__ #ifdef __linux__
char * single_block = (char*) memalign( char *single_block = (char *)memalign(512, 1024 * sizeof(unsigned));
512,
1024*sizeof(unsigned)
);
int f = open(test_path.string().c_str(), O_RDONLY|O_DIRECT|O_SYNC); int f = open(test_path.string().c_str(), O_RDONLY | O_DIRECT | O_SYNC);
SimpleLogger().Write(logDEBUG) << if (-1 == f)
"opened, error: " << strerror(errno); {
char * raw_array = (char*) memalign( SimpleLogger().Write(logDEBUG) << "opened, error: " << strerror(errno);
512, return -1;
number_of_elements*sizeof(unsigned) }
); char *raw_array = (char *)memalign(512, number_of_elements * sizeof(unsigned));
#endif #endif
time1 = get_timestamp(); time1 = get_timestamp();
#ifdef __APPLE__ #ifdef __APPLE__
read(fileno(fd), raw_array, number_of_elements*sizeof(unsigned)); read(fileno(fd), raw_array, number_of_elements * sizeof(unsigned));
close(fileno(fd)); close(fileno(fd));
fd = fopen(test_path.string().c_str(), "r"); fd = fopen(test_path.string().c_str(), "r");
#endif #endif
#ifdef __linux__ #ifdef __linux__
int ret = read(f, raw_array, number_of_elements*sizeof(unsigned)); int ret = read(f, raw_array, number_of_elements * sizeof(unsigned));
SimpleLogger().Write(logDEBUG) << SimpleLogger().Write(logDEBUG) << "read " << ret
"read " << ret << " bytes, error: " << strerror(errno); << " bytes, error: " << strerror(errno);
close(f); close(f);
f = open(test_path.string().c_str(), O_RDONLY|O_DIRECT|O_SYNC); f = open(test_path.string().c_str(), O_RDONLY | O_DIRECT | O_SYNC);
SimpleLogger().Write(logDEBUG) << SimpleLogger().Write(logDEBUG) << "opened, error: " << strerror(errno);
"opened, error: " << strerror(errno);
#endif #endif
time2 = get_timestamp(); time2 = get_timestamp();
SimpleLogger().Write(logDEBUG) << SimpleLogger().Write(logDEBUG) << "reading raw 1GB took " << (time2 - time1) * 1000
"reading raw 1GB took " << (time2-time1)*1000 << "ms"; << "ms";
SimpleLogger().Write() << "raw read performance: " << SimpleLogger().Write() << "raw read performance: " << std::setprecision(5) << std::fixed
std::setprecision(5) << std::fixed << << 1024 * 1024 / ((time2 - time1) * 1000) << "MB/sec";
1024*1024/((time2-time1)*1000) << "MB/sec";
std::vector<double> timing_results_raw_random; std::vector<double> timing_results_raw_random;
SimpleLogger().Write(logDEBUG) << "running 1000 random I/Os of 4KB"; SimpleLogger().Write(logDEBUG) << "running 1000 random I/Os of 4KB";
@ -217,60 +198,59 @@ int main (int argc, char * argv[]) {
#ifdef __linux__ #ifdef __linux__
lseek(f, 0, SEEK_SET); lseek(f, 0, SEEK_SET);
#endif #endif
//make 1000 random access, time each I/O seperately // make 1000 random access, time each I/O seperately
unsigned number_of_blocks = (number_of_elements*sizeof(unsigned)-1)/4096; unsigned number_of_blocks = (number_of_elements * sizeof(unsigned) - 1) / 4096;
for(unsigned i = 0; i < 1000; ++i) { for (unsigned i = 0; i < 1000; ++i)
unsigned block_to_read = std::rand()%number_of_blocks; {
off_t current_offset = block_to_read*4096; unsigned block_to_read = std::rand() % number_of_blocks;
off_t current_offset = block_to_read * 4096;
time1 = get_timestamp(); time1 = get_timestamp();
#ifdef __APPLE__ #ifdef __APPLE__
int ret1 = fseek(fd, current_offset, SEEK_SET); int ret1 = fseek(fd, current_offset, SEEK_SET);
int ret2 = read(fileno(fd), (char*)&single_block[0], 4096); int ret2 = read(fileno(fd), (char *)&single_block[0], 4096);
#endif #endif
#ifdef __FreeBSD__ #ifdef __FreeBSD__
int ret1 = 0; int ret1 = 0;
int ret2 = 0; int ret2 = 0;
#endif #endif
#ifdef __linux__ #ifdef __linux__
int ret1 = lseek(f, current_offset, SEEK_SET); int ret1 = lseek(f, current_offset, SEEK_SET);
int ret2 = read(f, (char*)single_block, 4096); int ret2 = read(f, (char *)single_block, 4096);
#endif #endif
time2 = get_timestamp(); time2 = get_timestamp();
if( ((off_t)-1) == ret1) { if (((off_t) - 1) == ret1)
SimpleLogger().Write(logWARNING) {
<< "offset: " << current_offset; SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
SimpleLogger().Write(logWARNING) SimpleLogger().Write(logWARNING) << "seek error " << strerror(errno);
<< "seek error " << strerror(errno);
throw OSRMException("seek error"); throw OSRMException("seek error");
} }
if(-1 == ret2) { if (-1 == ret2)
SimpleLogger().Write(logWARNING) {
<< "offset: " << current_offset; SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
SimpleLogger().Write(logWARNING) SimpleLogger().Write(logWARNING) << "read error " << strerror(errno);
<< "read error " << strerror(errno);
throw OSRMException("read error"); throw OSRMException("read error");
} }
timing_results_raw_random.push_back((time2-time1)*1000.); timing_results_raw_random.push_back((time2 - time1) * 1000.);
} }
// Do statistics // Do statistics
SimpleLogger().Write(logDEBUG) << "running raw random I/O statistics"; SimpleLogger().Write(logDEBUG) << "running raw random I/O statistics";
std::ofstream random_csv("random.csv", std::ios::trunc); std::ofstream random_csv("random.csv", std::ios::trunc);
for(unsigned i = 0; i < timing_results_raw_random.size(); ++i) { for (unsigned i = 0; i < timing_results_raw_random.size(); ++i)
{
random_csv << i << ", " << timing_results_raw_random[i] << std::endl; random_csv << i << ", " << timing_results_raw_random[i] << std::endl;
} }
random_csv.close(); random_csv.close();
RunStatistics(timing_results_raw_random, stats); RunStatistics(timing_results_raw_random, stats);
SimpleLogger().Write() << "raw random I/O: " << SimpleLogger().Write() << "raw random I/O: " << std::setprecision(5) << std::fixed
std::setprecision(5) << std::fixed << << "min: " << stats.min << "ms, "
"min: " << stats.min << "ms, " << << "mean: " << stats.mean << "ms, "
"mean: " << stats.mean << "ms, " << << "med: " << stats.med << "ms, "
"med: " << stats.med << "ms, " << << "max: " << stats.max << "ms, "
"max: " << stats.max << "ms, " << << "dev: " << stats.dev << "ms";
"dev: " << stats.dev << "ms";
std::vector<double> timing_results_raw_seq; std::vector<double> timing_results_raw_seq;
#ifdef __APPLE__ #ifdef __APPLE__
@ -280,81 +260,80 @@ int main (int argc, char * argv[]) {
lseek(f, 0, SEEK_SET); lseek(f, 0, SEEK_SET);
#endif #endif
//read every 100th block // read every 100th block
for( for (unsigned i = 0; i < 1000; ++i)
unsigned i = 0; {
i < 1000; off_t current_offset = i * 4096;
++i
) {
off_t current_offset = i*4096;
time1 = get_timestamp(); time1 = get_timestamp();
#ifdef __APPLE__ #ifdef __APPLE__
int ret1 = fseek(fd, current_offset, SEEK_SET); int ret1 = fseek(fd, current_offset, SEEK_SET);
int ret2 = read(fileno(fd), (char*)&single_block, 4096); int ret2 = read(fileno(fd), (char *)&single_block, 4096);
#endif #endif
#ifdef __FreeBSD__ #ifdef __FreeBSD__
int ret1 = 0; int ret1 = 0;
int ret2 = 0; int ret2 = 0;
#endif #endif
#ifdef __linux__ #ifdef __linux__
int ret1 = lseek(f, current_offset, SEEK_SET); int ret1 = lseek(f, current_offset, SEEK_SET);
int ret2 = read(f, (char*)single_block, 4096); int ret2 = read(f, (char *)single_block, 4096);
#endif #endif
time2 = get_timestamp(); time2 = get_timestamp();
if( ((off_t)-1) == ret1) { if (((off_t) - 1) == ret1)
SimpleLogger().Write(logWARNING) {
<< "offset: " << current_offset; SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
SimpleLogger().Write(logWARNING) SimpleLogger().Write(logWARNING) << "seek error " << strerror(errno);
<< "seek error " << strerror(errno);
throw OSRMException("seek error"); throw OSRMException("seek error");
} }
if(-1 == ret2) { if (-1 == ret2)
SimpleLogger().Write(logWARNING) {
<< "offset: " << current_offset; SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
SimpleLogger().Write(logWARNING) SimpleLogger().Write(logWARNING) << "read error " << strerror(errno);
<< "read error " << strerror(errno);
throw OSRMException("read error"); throw OSRMException("read error");
} }
timing_results_raw_seq.push_back((time2-time1)*1000.); timing_results_raw_seq.push_back((time2 - time1) * 1000.);
} }
#ifdef __APPLE__ #ifdef __APPLE__
fclose(fd); fclose(fd);
// free(single_element); // free(single_element);
free(raw_array); free(raw_array);
// free(single_block); // free(single_block);
#endif #endif
#ifdef __linux__ #ifdef __linux__
close(f); close(f);
#endif #endif
//Do statistics // Do statistics
SimpleLogger().Write(logDEBUG) << "running sequential I/O statistics"; SimpleLogger().Write(logDEBUG) << "running sequential I/O statistics";
//print simple statistics: min, max, median, variance // print simple statistics: min, max, median, variance
std::ofstream seq_csv("sequential.csv", std::ios::trunc); std::ofstream seq_csv("sequential.csv", std::ios::trunc);
for(unsigned i = 0; i < timing_results_raw_seq.size(); ++i) { for (unsigned i = 0; i < timing_results_raw_seq.size(); ++i)
{
seq_csv << i << ", " << timing_results_raw_seq[i] << std::endl; seq_csv << i << ", " << timing_results_raw_seq[i] << std::endl;
} }
seq_csv.close(); seq_csv.close();
RunStatistics(timing_results_raw_seq, stats); RunStatistics(timing_results_raw_seq, stats);
SimpleLogger().Write() << "raw sequential I/O: " << SimpleLogger().Write() << "raw sequential I/O: " << std::setprecision(5) << std::fixed
std::setprecision(5) << std::fixed << << "min: " << stats.min << "ms, "
"min: " << stats.min << "ms, " << << "mean: " << stats.mean << "ms, "
"mean: " << stats.mean << "ms, " << << "med: " << stats.med << "ms, "
"med: " << stats.med << "ms, " << << "max: " << stats.max << "ms, "
"max: " << stats.max << "ms, " << << "dev: " << stats.dev << "ms";
"dev: " << stats.dev << "ms";
if( boost::filesystem::exists(test_path) ) { if (boost::filesystem::exists(test_path))
{
boost::filesystem::remove(test_path); boost::filesystem::remove(test_path);
SimpleLogger().Write(logDEBUG) << "removing temporary files"; SimpleLogger().Write(logDEBUG) << "removing temporary files";
} }
} }
} catch ( const std::exception & e ) { }
catch (const std::exception &e)
{
SimpleLogger().Write(logWARNING) << "caught exception: " << e.what(); SimpleLogger().Write(logWARNING) << "caught exception: " << e.what();
SimpleLogger().Write(logWARNING) << "cleaning up, and exiting"; SimpleLogger().Write(logWARNING) << "cleaning up, and exiting";
if(boost::filesystem::exists(test_path)) { if (boost::filesystem::exists(test_path))
{
boost::filesystem::remove(test_path); boost::filesystem::remove(test_path);
SimpleLogger().Write(logWARNING) << "removing temporary files"; SimpleLogger().Write(logWARNING) << "removing temporary files";
} }