Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Check for valid data files. Implements #224

Browse Source
This commit is contained in:
DennisOSRM 2013-07-22 16:34:06 +02:00
parent 0367399c89
commit 4a39a4af1c
9 changed files with 123 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Extractor/ExtractionContainers.cpp
View File

@ -121,6 +121,7 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con
//serialize restrictions
std::ofstream restrictionsOutstream;
restrictionsOutstream.open(restrictionsFileName.c_str(), std::ios::binary);
restrictionsOutstream.write((char*)&uuid, sizeof(UUID));
restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned));
for(restrictionsIT = restrictionsVector.begin(); restrictionsIT != restrictionsVector.end(); ++restrictionsIT) {
if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
@ -131,6 +132,7 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con
std::ofstream fout;
fout.open(output_file_name.c_str(), std::ios::binary);
fout.write((char*)&uuid, sizeof(UUID));
fout.write((char*)&usedNodeCounter, sizeof(unsigned));
time = get_timestamp();
std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush;
@ -158,7 +160,7 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con
std::cout << "[extractor] setting number of nodes ... " << std::flush;
std::ios::pos_type positionInFile = fout.tellp();
fout.seekp(std::ios::beg);
fout.seekp(std::ios::beg+sizeof(UUID));
fout.write((char*)&usedNodeCounter, sizeof(unsigned));
fout.seekp(positionInFile);

3
Extractor/ExtractionContainers.h
View File

@ -23,6 +23,7 @@
#include "ExtractorStructs.h"
#include "../DataStructures/TimingUtil.h"
#include "../Util/UUID.h"
#include <boost/foreach.hpp>
#include <stxxl.h>
@ -63,7 +64,7 @@ public:
STXXLStringVector nameVector;
STXXLRestrictionsVector restrictionsVector;
STXXLWayIDStartEndVector wayStartEndVector;
const UUID uuid;
};
#endif /* EXTRACTIONCONTAINERS_H_ */

17
Server/DataStructures/QueryObjectsStorage.cpp
View File

@ -58,24 +58,23 @@ QueryObjectsStorage::QueryObjectsStorage(
getline(timestampInStream, timestamp);
timestampInStream.close();
}
if(!timestamp.length())
if(!timestamp.length()) {
timestamp = "n/a";
if(25 < timestamp.length())
}
if(25 < timestamp.length()) {
timestamp.resize(25);
}
INFO("Loading auxiliary information");
//Init nearest neighbor data structure
std::ifstream nodesInStream(nodesPath.c_str(), std::ios::binary);
if(!nodesInStream) { ERR(nodesPath << " not found"); }
std::ifstream edgesInStream(edgesPath.c_str(), std::ios::binary);
if(!edgesInStream) { ERR(edgesPath << " not found"); }
nodeHelpDesk = new NodeInformationHelpDesk(
ramIndexPath.c_str(),
fileIndexPath.c_str(),
ramIndexPath,
fileIndexPath,
nodesPath,
edgesPath,
n,
checkSum
);
nodeHelpDesk->initNNGrid(nodesInStream, edgesInStream);
//deserialize street name list
INFO("Loading names index");

67
Util/GraphLoader.h
View File

@ -50,28 +50,31 @@ struct _ExcessRemover {
};
template<typename EdgeT>
NodeID readBinaryOSRMGraphFromStream(std::istream &in, std::vector<EdgeT>& edgeList, std::vector<NodeID> &bollardNodes, std::vector<NodeID> &trafficLightNodes, std::vector<NodeInfo> * int2ExtNodeMap, std::vector<_Restriction> & inputRestrictions) {
NodeID readBinaryOSRMGraphFromStream(
std::istream &in,
std::vector<EdgeT>& edgeList,
std::vector<NodeID> &bollardNodes,
std::vector<NodeID> &trafficLightNodes,
std::vector<NodeInfo> * int2ExtNodeMap,
std::vector<_Restriction> & inputRestrictions
) {
const UUID uuid_orig;
UUID uuid_loaded;
in.read((char *) &uuid_loaded, sizeof(UUID));
if(!uuid_loaded.IsMagicNumberOK()) {
ERR("hsgr input file misses magic number. Check or reprocess the file");
}
if( uuid_loaded.TestHSGR(uuid_orig) ) {
if( !uuid_loaded.TestGraphUtil(uuid_orig) ) {
WARN(
".hsgr was prepared with different build.\n"
".osrm was prepared with different build.\n"
"Reprocess to get rid of this warning."
)
}
NodeID n, source, target;
EdgeID m;
short dir;// direction (0 = open, 1 = forward, 2+ = open)
ExternalNodeMap ext2IntNodeMap;
in.read((char*)&n, sizeof(NodeID));
DEBUG("Importing n = " << n << " nodes ");
INFO("Importing n = " << n << " nodes ");
_Node node;
for (NodeID i=0; i<n; ++i) {
in.read((char*)&node, sizeof(_Node));
@ -88,7 +91,7 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &in, std::vector<EdgeT>& edgeL
std::vector<NodeID>(trafficLightNodes).swap(trafficLightNodes);
in.read((char*)&m, sizeof(unsigned));
DEBUG(" and " << m << " edges ");
INFO(" and " << m << " edges ");
for(unsigned i = 0; i < inputRestrictions.size(); ++i) {
ExternalNodeMap::iterator intNodeID = ext2IntNodeMap.find(inputRestrictions[i].fromNode);
if( intNodeID == ext2IntNodeMap.end()) {
@ -380,19 +383,43 @@ NodeID readDDSGGraphFromStream(std::istream &in, std::vector<EdgeT>& edgeList, s
}
template<typename NodeT, typename EdgeT>
unsigned readHSGRFromStream(std::istream &in, std::vector<NodeT>& nodeList, std::vector<EdgeT> & edgeList, unsigned * checkSum) {
unsigned numberOfNodes = 0;
in.read((char*) checkSum, sizeof(unsigned));
in.read((char*) & numberOfNodes, sizeof(unsigned));
nodeList.resize(numberOfNodes + 1);
in.read((char*) &(nodeList[0]), numberOfNodes*sizeof(NodeT));
unsigned readHSGRFromStream(
std::istream &hsgr_input_stream,
std::vector<NodeT> & node_list,
std::vector<EdgeT> & edge_list,
unsigned * check_sum
) {
UUID uuid_loaded, uuid_orig;
hsgr_input_stream.read((char *)&uuid_loaded, sizeof(UUID));
if( !uuid_loaded.TestGraphUtil(uuid_orig) ) {
WARN(
".hsgr was prepared with different build.\n"
"Reprocess to get rid of this warning."
)
}
unsigned numberOfEdges = 0;
in.read((char*) &numberOfEdges, sizeof(unsigned));
edgeList.resize(numberOfEdges);
in.read((char*) &(edgeList[0]), numberOfEdges*sizeof(EdgeT));
unsigned number_of_nodes = 0;
hsgr_input_stream.read((char*) check_sum, sizeof(unsigned));
hsgr_input_stream.read((char*) & number_of_nodes, sizeof(unsigned));
node_list.resize(number_of_nodes + 1);
hsgr_input_stream.read(
(char*) &(node_list[0]),
number_of_nodes*sizeof(NodeT)
);
return numberOfNodes;
unsigned number_of_edges = 0;
hsgr_input_stream.read(
(char*) &number_of_edges,
sizeof(unsigned)
);
edge_list.resize(number_of_edges);
hsgr_input_stream.read(
(char*) &(edge_list[0]),
number_of_edges*sizeof(EdgeT)
);
return number_of_nodes;
}
#endif // GRAPHLOADER_H

33
Util/UUID.cpp.in
View File

@ -59,6 +59,37 @@ const bool UUID::IsMagicNumberOK() const {
return 1297240911 == magic_number;
}
const bool UUID::TestHSGR(const UUID & other) const {
const bool UUID::TestGraphUtil(const UUID & other) const {
if(!other.IsMagicNumberOK()) {
ERR("hsgr input file misses magic number. Check or reprocess the file");
}
return std::equal(md5_graph, md5_graph+32, other.md5_graph);
}
const bool UUID::TestPrepare(const UUID & other) const {
if(!other.IsMagicNumberOK()) {
ERR("extracted input file misses magic number. Check or reprocess the file");
}
return std::equal(md5_prepare, md5_prepare+32, other.md5_prepare);
}
const bool UUID::TestRTree(const UUID & other) const {
if(!other.IsMagicNumberOK()) {
ERR("r-tree input file misses magic number. Check or reprocess the file");
}
return std::equal(md5_tree, md5_tree+32, other.md5_tree);
}
const bool UUID::TestNodeInfo(const UUID & other) const {
if(!other.IsMagicNumberOK()) {
ERR("nodes file misses magic number. Check or reprocess the file");
}
return std::equal(md5_nodeinfo, md5_nodeinfo+32, other.md5_nodeinfo);
}
const bool UUID::TestQueryObjects(const UUID & other) const {
if(!other.IsMagicNumberOK()) {
ERR("missing magic number. Check or reprocess the file");
}
return std::equal(md5_objects, md5_objects+32, other.md5_objects);
}

8
Util/UUID.h
View File

@ -21,6 +21,8 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifndef UUID_H
#define UUID_H
#include "../typedefs.h"
#include <boost/noncopyable.hpp>
#include <boost/uuid/uuid.hpp> // uuid class
#include <boost/uuid/uuid_generators.hpp> // generators
@ -39,7 +41,11 @@ public:
~UUID();
const boost::uuids::uuid & GetUUID() const;
const bool IsMagicNumberOK() const;
const bool TestHSGR(const UUID & other) const;
const bool TestGraphUtil(const UUID & other) const;
const bool TestPrepare(const UUID & other) const;
const bool TestRTree(const UUID & other) const;
const bool TestNodeInfo(const UUID & other) const;
const bool TestQueryObjects(const UUID & other) const;
private:
const unsigned magic_number;
char md5_prepare[33];

25
createHierarchy.cpp
View File

@ -78,7 +78,16 @@ int main (int argc, char *argv[]) {
ERR("Could not access <osrm-restrictions> files");
}
_Restriction restriction;
UUID uuid_loaded, uuid_orig;
unsigned usableRestrictionsCounter(0);
restrictionsInstream.read((char*)&uuid_loaded, sizeof(UUID));
if( !uuid_loaded.TestPrepare(uuid_orig) ) {
WARN(
".restrictions was prepared with different build.\n"
"Reprocess to get rid of this warning."
)
}
restrictionsInstream.read((char*)&usableRestrictionsCounter, sizeof(unsigned));
inputRestrictions.resize(usableRestrictionsCounter);
restrictionsInstream.read((char *)&(inputRestrictions[0]), usableRestrictionsCounter*sizeof(_Restriction));
@ -211,8 +220,8 @@ int main (int argc, char *argv[]) {
unsigned numberOfNodes = 0;
unsigned numberOfEdges = contractedEdgeList.size();
INFO("Serializing compacted graph of " << numberOfEdges << " edges");
std::ofstream edgeOutFile(graphOut.c_str(), std::ios::binary);
std::ofstream hsgr_output_stream(graphOut.c_str(), std::ios::binary);
hsgr_output_stream.write((char*)&uuid_orig, sizeof(UUID) );
BOOST_FOREACH(const QueryEdge & edge, contractedEdgeList) {
if(edge.source > numberOfNodes) {
numberOfNodes = edge.source;
@ -237,11 +246,11 @@ int main (int argc, char *argv[]) {
}
++numberOfNodes;
//Serialize numberOfNodes, nodes
edgeOutFile.write((char*) &crc32OfNodeBasedEdgeList, sizeof(unsigned));
edgeOutFile.write((char*) &numberOfNodes, sizeof(unsigned));
edgeOutFile.write((char*) &_nodes[0], sizeof(StaticGraph<EdgeData>::_StrNode)*(numberOfNodes));
hsgr_output_stream.write((char*) &crc32OfNodeBasedEdgeList, sizeof(unsigned));
hsgr_output_stream.write((char*) &numberOfNodes, sizeof(unsigned));
hsgr_output_stream.write((char*) &_nodes[0], sizeof(StaticGraph<EdgeData>::_StrNode)*(numberOfNodes));
//Serialize number of Edges
edgeOutFile.write((char*) &position, sizeof(unsigned));
hsgr_output_stream.write((char*) &position, sizeof(unsigned));
--numberOfNodes;
edge = 0;
int usedEdgeCounter = 0;
@ -256,7 +265,7 @@ int main (int argc, char *argv[]) {
ERR("Failed at edges of node " << node << " of " << numberOfNodes);
}
//Serialize edges
edgeOutFile.write((char*) &currentEdge, sizeof(StaticGraph<EdgeData>::_StrEdge));
hsgr_output_stream.write((char*) &currentEdge, sizeof(StaticGraph<EdgeData>::_StrEdge));
++edge;
++usedEdgeCounter;
}
@ -265,7 +274,7 @@ int main (int argc, char *argv[]) {
INFO("Expansion : " << (nodeBasedNodeNumber/expansionHasFinishedTime) << " nodes/sec and "<< (edgeBasedNodeNumber/expansionHasFinishedTime) << " edges/sec");
INFO("Contraction: " << (edgeBasedNodeNumber/expansionHasFinishedTime) << " nodes/sec and "<< usedEdgeCounter/endTime << " edges/sec");
edgeOutFile.close();
hsgr_output_stream.close();
//cleanedEdgeList.clear();
_nodes.clear();
INFO("finished preprocessing");

2
extractor.cpp
View File

@ -28,6 +28,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "Util/MachineInfo.h"
#include "Util/OpenMPWrapper.h"
#include "Util/StringUtil.h"
#include "Util/UUID.h"
#include "typedefs.h"
#include <cstdlib>
@ -37,6 +38,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include <string>
ExtractorCallbacks * extractCallBacks;
UUID uuid;
int main (int argc, char *argv[]) {
try {

5
routed.cpp
View File

@ -26,6 +26,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "Util/BaseConfiguration.h"
#include "Util/InputFileUtil.h"
#include "Util/OpenMPWrapper.h"
#include "Util/UUID.h"
#ifdef __linux__
#include "Util/LinuxStackTrace.h"
@ -77,7 +78,9 @@ int main (int argc, char * argv[]) {
//}
try {
std::cout << "\n starting up engines, compile at " <<
//std::cout << "fingerprint: " << UUID::GetInstance().GetUUID() << std::endl;
std::cout << "starting up engines, compiled at " <<
__DATE__ << ", " __TIME__ << std::endl;
#ifndef _WIN32