Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Check if files exist and contain data, fixes #693

Browse Source
This commit is contained in:
Dennis Luxen 2013-08-09 17:47:11 +02:00
parent 0765ebf735
commit a542292ce2
5 changed files with 97 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
DataStructures/NodeInformationHelpDesk.h
View File

@ -29,10 +29,11 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "../typedefs.h"
#include <boost/assert.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/noncopyable.hpp>
#include <iostream>
#include <fstream>
#include <string>
#include <vector>
@ -137,19 +138,29 @@ public:
private:
void LoadNodesAndEdges(
const std::string & nodes_file,
const std::string & edges_file
const std::string & nodes_filename,
const std::string & edges_filename
) {
std::ifstream nodes_input_stream(nodes_file.c_str(), std::ios::binary);
if(!nodes_input_stream) {
throw OSRMException("nodes file not found");
boost::filesystem::path nodes_file(nodes_filename);
if ( !boost::filesystem::exists( nodes_file ) ) {
throw OSRMException("nodes file does not exist");
}
std::ifstream edges_input_stream(edges_file.c_str(), std::ios::binary);
if(!edges_input_stream) {
throw OSRMException("edges file not found");
if ( 0 == boost::filesystem::file_size( nodes_file ) ) {
throw OSRMException("nodes file is empty");
}
SimpleLogger().Write(logDEBUG) << "Loading node data" << nodes_file.length() << " ->" << nodes_file << "<-";
boost::filesystem::path edges_file(edges_filename);
if ( !boost::filesystem::exists( edges_file ) ) {
throw OSRMException("edges file does not exist");
}
if ( 0 == boost::filesystem::file_size( edges_file ) ) {
throw OSRMException("edges file is empty");
}
boost::filesystem::ifstream nodes_input_stream(nodes_file, std::ios::binary);
boost::filesystem::ifstream edges_input_stream(edges_file, std::ios::binary);
SimpleLogger().Write(logDEBUG) << "Loading node data";
NodeInfo b;
while(!nodes_input_stream.eof()) {
nodes_input_stream.read((char *)&b, sizeof(NodeInfo));

39
DataStructures/StaticRTree.h
View File

@ -26,6 +26,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "PhantomNodes.h"
#include "DeallocatingVector.h"
#include "HilbertValue.h"
#include "../Util/OSRMException.h"
#include "../Util/SimpleLogger.h"
#include "../Util/TimingUtil.h"
#include "../typedefs.h"
@ -33,6 +34,8 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include <boost/assert.hpp>
#include <boost/bind.hpp>
#include <boost/foreach.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/algorithm/minmax.hpp>
#include <boost/algorithm/minmax_element.hpp>
#include <boost/range/algorithm_ext/erase.hpp>
@ -44,7 +47,6 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include <climits>
#include <algorithm>
#include <fstream>
#include <queue>
#include <string>
#include <vector>
@ -55,7 +57,7 @@ const static uint32_t RTREE_LEAF_NODE_SIZE = 1170;
// Implements a static, i.e. packed, R-tree
static boost::thread_specific_ptr<std::ifstream> thread_local_rtree_stream;
static boost::thread_specific_ptr<boost::filesystem::ifstream> thread_local_rtree_stream;
template<class DataT>
class StaticRTree : boost::noncopyable {
@ -309,7 +311,7 @@ public:
}
//open leaf file
std::ofstream leaf_node_file(leaf_node_filename.c_str(), std::ios::binary);
boost::filesystem::ofstream leaf_node_file(leaf_node_filename, std::ios::binary);
leaf_node_file.write((char*) &m_element_count, sizeof(uint64_t));
//sort the hilbert-value representatives
@ -390,7 +392,11 @@ public:
}
//open tree file
std::ofstream tree_node_file(tree_node_filename.c_str(), std::ios::binary);
boost::filesystem::ofstream tree_node_file(
tree_node_filename,
std::ios::binary
);
uint32_t size_of_tree = m_search_tree.size();
BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty");
tree_node_file.write((char *)&size_of_tree, sizeof(uint32_t));
@ -408,7 +414,16 @@ public:
const std::string & leaf_filename
) : m_leaf_node_filename(leaf_filename) {
//open tree node file and load into RAM.
std::ifstream tree_node_file(node_filename.c_str(), std::ios::binary);
boost::filesystem::path node_file(node_filename);
if ( !boost::filesystem::exists( node_file ) ) {
throw OSRMException("ram index file does not exist");
}
if ( 0 == boost::filesystem::file_size( node_file ) ) {
throw OSRMException("ram index file is empty");
}
boost::filesystem::ifstream tree_node_file( node_file, std::ios::binary );
uint32_t tree_size = 0;
tree_node_file.read((char*)&tree_size, sizeof(uint32_t));
//SimpleLogger().Write() << "reading " << tree_size << " tree nodes in " << (sizeof(TreeNode)*tree_size) << " bytes";
@ -417,7 +432,15 @@ public:
tree_node_file.close();
//open leaf node file and store thread specific pointer
std::ifstream leaf_node_file(leaf_filename.c_str(), std::ios::binary);
boost::filesystem::path leaf_file(leaf_filename);
if ( !boost::filesystem::exists( leaf_file ) ) {
throw OSRMException("mem index file does not exist");
}
if ( 0 == boost::filesystem::file_size( leaf_file ) ) {
throw OSRMException("mem index file is empty");
}
boost::filesystem::ifstream leaf_node_file( leaf_file, std::ios::binary );
leaf_node_file.read((char*)&m_element_count, sizeof(uint64_t));
leaf_node_file.close();
@ -729,8 +752,8 @@ private:
inline void LoadLeafFromDisk(const uint32_t leaf_id, LeafNode& result_node) {
if(!thread_local_rtree_stream.get() || !thread_local_rtree_stream->is_open()) {
thread_local_rtree_stream.reset(
new std::ifstream(
m_leaf_node_filename.c_str(),
new boost::filesystem::ifstream(
m_leaf_node_filename,
std::ios::in | std::ios::binary
)
);

37
Server/DataStructures/QueryObjectsStorage.cpp
View File

@ -50,20 +50,15 @@ QueryObjectsStorage::QueryObjectsStorage(
}
SimpleLogger().Write() << "loading graph data";
std::ifstream hsgrInStream(hsgrPath.c_str(), std::ios::binary);
if(!hsgrInStream) {
throw OSRMException("hsgr not found");
}
//Deserialize road network graph
std::vector< QueryGraph::_StrNode> nodeList;
std::vector< QueryGraph::_StrEdge> edgeList;
const int n = readHSGRFromStream(
hsgrInStream,
hsgrPath,
nodeList,
edgeList,
&checkSum
);
hsgrInStream.close();
SimpleLogger().Write() << "Data checksum is " << checkSum;
graph = new QueryGraph(nodeList, edgeList);
@ -100,23 +95,31 @@ QueryObjectsStorage::QueryObjectsStorage(
//deserialize street name list
SimpleLogger().Write() << "Loading names index";
std::ifstream namesInStream(namesPath.c_str(), std::ios::binary);
if(!namesInStream) {
throw OSRMException("names file not found");
boost::filesystem::path names_file(namesPath);
if ( !boost::filesystem::exists( names_file ) ) {
throw OSRMException("names file does not exist");
}
unsigned size(0);
namesInStream.read((char *)&size, sizeof(unsigned));
if ( 0 == boost::filesystem::file_size( names_file ) ) {
throw OSRMException("names file is empty");
}
boost::filesystem::ifstream name_stream(names_file, std::ios::binary);
unsigned size = 0;
name_stream.read((char *)&size, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != size, "name file empty");
char buf[1024];
for(unsigned i = 0; i < size; ++i) {
unsigned sizeOfString = 0;
namesInStream.read((char *)&sizeOfString, sizeof(unsigned));
buf[sizeOfString] = '\0'; // instead of memset
namesInStream.read(buf, sizeOfString);
for( unsigned i = 0; i < size; ++i ) {
unsigned size_of_string = 0;
name_stream.read((char *)&size_of_string, sizeof(unsigned));
buf[size_of_string] = '\0'; // instead of memset
name_stream.read(buf, size_of_string);
names.push_back(buf);
}
std::vector<std::string>(names).swap(names);
namesInStream.close();
BOOST_ASSERT_MSG(0 != names.size(), "could not load any names");
name_stream.close();
SimpleLogger().Write() << "All query data structures loaded";
}

10
Server/DataStructures/QueryObjectsStorage.h
View File

@ -22,8 +22,6 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifndef QUERYOBJECTSSTORAGE_H_
#define QUERYOBJECTSSTORAGE_H_
#include<vector>
#include<string>
#include "../../Util/GraphLoader.h"
#include "../../Util/OSRMException.h"
#include "../../Util/SimpleLogger.h"
@ -31,6 +29,14 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "../../DataStructures/QueryEdge.h"
#include "../../DataStructures/StaticGraph.h"
#include <boost/assert.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <vector>
#include <string>
struct QueryObjectsStorage {
typedef StaticGraph<QueryEdge::EdgeData> QueryGraph;
typedef QueryGraph::InputEdge InputEdge;

21
Util/GraphLoader.h
View File

@ -31,6 +31,8 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "../typedefs.h"
#include <boost/assert.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/unordered_map.hpp>
#include <cassert>
@ -398,40 +400,51 @@ NodeID readDDSGGraphFromStream(std::istream &in, std::vector<EdgeT>& edgeList, s
template<typename NodeT, typename EdgeT>
unsigned readHSGRFromStream(
std::istream &hsgr_input_stream,
const std::string & hsgr_filename,
std::vector<NodeT> & node_list,
std::vector<EdgeT> & edge_list,
unsigned * check_sum
) {
boost::filesystem::path hsgr_file(hsgr_filename);
if ( !boost::filesystem::exists( hsgr_file ) ) {
throw OSRMException("hsgr file does not exist");
}
if ( 0 == boost::filesystem::file_size( hsgr_file ) ) {
throw OSRMException("hsgr file is empty");
}
boost::filesystem::ifstream hsgr_input_stream(hsgr_file, std::ios::binary);
UUID uuid_loaded, uuid_orig;
hsgr_input_stream.read((char *)&uuid_loaded, sizeof(UUID));
if( !uuid_loaded.TestGraphUtil(uuid_orig) ) {
SimpleLogger().Write(logWARNING) <<
".hsgr was prepared with different build.\n"
".hsgr was prepared with different build. "
"Reprocess to get rid of this warning.";
}
unsigned number_of_nodes = 0;
hsgr_input_stream.read((char*) check_sum, sizeof(unsigned));
hsgr_input_stream.read((char*) & number_of_nodes, sizeof(unsigned));
BOOST_ASSERT_MSG( 0 != number_of_nodes, "number of nodes is zero");
node_list.resize(number_of_nodes + 1);
hsgr_input_stream.read(
(char*) &(node_list[0]),
number_of_nodes*sizeof(NodeT)
);
unsigned number_of_edges = 0;
hsgr_input_stream.read(
(char*) &number_of_edges,
sizeof(unsigned)
);
BOOST_ASSERT_MSG( 0 != number_of_edges, "number of edges is zero");
edge_list.resize(number_of_edges);
hsgr_input_stream.read(
(char*) &(edge_list[0]),
number_of_edges*sizeof(EdgeT)
);
hsgr_input_stream.close();
return number_of_nodes;
}