Check for valid data files. Implements #224

Extractor/ExtractionContainers.cpp

@@ -121,6 +121,7 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con
         //serialize restrictions
         std::ofstream restrictionsOutstream;
         restrictionsOutstream.open(restrictionsFileName.c_str(), std::ios::binary);
+        restrictionsOutstream.write((char*)&uuid, sizeof(UUID));
         restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned));
         for(restrictionsIT = restrictionsVector.begin(); restrictionsIT != restrictionsVector.end(); ++restrictionsIT) {
             if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
@@ -131,6 +132,7 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con
 
         std::ofstream fout;
         fout.open(output_file_name.c_str(), std::ios::binary);
+        fout.write((char*)&uuid, sizeof(UUID));
         fout.write((char*)&usedNodeCounter, sizeof(unsigned));
         time = get_timestamp();
         std::cout << "[extractor] Confirming/Writing used nodes     ... " << std::flush;
@@ -158,7 +160,7 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con
 
         std::cout << "[extractor] setting number of nodes   ... " << std::flush;
         std::ios::pos_type positionInFile = fout.tellp();
-        fout.seekp(std::ios::beg);
+        fout.seekp(std::ios::beg+sizeof(UUID));
         fout.write((char*)&usedNodeCounter, sizeof(unsigned));
         fout.seekp(positionInFile);
 

Extractor/ExtractionContainers.h

@@ -23,6 +23,7 @@
 
 #include "ExtractorStructs.h"
 #include "../DataStructures/TimingUtil.h"
+#include "../Util/UUID.h"
 
 #include <boost/foreach.hpp>
 #include <stxxl.h>
@@ -63,7 +64,7 @@ public:
     STXXLStringVector           nameVector;
     STXXLRestrictionsVector     restrictionsVector;
     STXXLWayIDStartEndVector    wayStartEndVector;
-
+    const UUID uuid;
 };
 
 #endif /* EXTRACTIONCONTAINERS_H_ */

Server/DataStructures/QueryObjectsStorage.cpp

@@ -58,24 +58,23 @@ QueryObjectsStorage::QueryObjectsStorage(
 	    getline(timestampInStream, timestamp);
 	    timestampInStream.close();
 	}
-	if(!timestamp.length())
+	if(!timestamp.length()) {
 	    timestamp = "n/a";
-	if(25 < timestamp.length())
+	}
+	if(25 < timestamp.length()) {
 	    timestamp.resize(25);
+	}
 
     INFO("Loading auxiliary information");
     //Init nearest neighbor data structure
-	std::ifstream nodesInStream(nodesPath.c_str(), std::ios::binary);
-	if(!nodesInStream) { ERR(nodesPath <<  " not found"); }
-    std::ifstream edgesInStream(edgesPath.c_str(), std::ios::binary);
-    if(!edgesInStream) { ERR(edgesPath <<  " not found"); }
 	nodeHelpDesk = new NodeInformationHelpDesk(
-		ramIndexPath.c_str(),
+		ramIndexPath,
-		fileIndexPath.c_str(),
+		fileIndexPath,
+		nodesPath,
+		edgesPath,
 		n,
 		checkSum
 	);
-	nodeHelpDesk->initNNGrid(nodesInStream, edgesInStream);
 
 	//deserialize street name list
 	INFO("Loading names index");

Util/GraphLoader.h

@@ -50,28 +50,31 @@ struct _ExcessRemover {
 };
 
 template<typename EdgeT>
-NodeID readBinaryOSRMGraphFromStream(std::istream &in, std::vector<EdgeT>& edgeList, std::vector<NodeID> &bollardNodes, std::vector<NodeID> &trafficLightNodes, std::vector<NodeInfo> * int2ExtNodeMap, std::vector<_Restriction> & inputRestrictions) {
+NodeID readBinaryOSRMGraphFromStream(
+    std::istream &in,
+    std::vector<EdgeT>& edgeList,
+    std::vector<NodeID> &bollardNodes,
+    std::vector<NodeID> &trafficLightNodes,
+    std::vector<NodeInfo> * int2ExtNodeMap,
+    std::vector<_Restriction> & inputRestrictions
+) {
     const UUID uuid_orig;
     UUID uuid_loaded;
     in.read((char *) &uuid_loaded, sizeof(UUID));
-    if(!uuid_loaded.IsMagicNumberOK()) {
-        ERR("hsgr input file misses magic number. Check or reprocess the file");
-    }
 
-    if( uuid_loaded.TestHSGR(uuid_orig) ) {
+    if( !uuid_loaded.TestGraphUtil(uuid_orig) ) {
         WARN(
-            ".hsgr was prepared with different build.\n"
+            ".osrm was prepared with different build.\n"
             "Reprocess to get rid of this warning."
             )
     }
 
-
     NodeID n, source, target;
     EdgeID m;
     short dir;// direction (0 = open, 1 = forward, 2+ = open)
     ExternalNodeMap ext2IntNodeMap;
     in.read((char*)&n, sizeof(NodeID));
-    DEBUG("Importing n = " << n << " nodes ");
+    INFO("Importing n = " << n << " nodes ");
     _Node node;
     for (NodeID i=0; i<n; ++i) {
         in.read((char*)&node, sizeof(_Node));
@@ -88,7 +91,7 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &in, std::vector<EdgeT>& edgeL
     std::vector<NodeID>(trafficLightNodes).swap(trafficLightNodes);
 
     in.read((char*)&m, sizeof(unsigned));
-    DEBUG(" and " << m << " edges ");
+    INFO(" and " << m << " edges ");
     for(unsigned i = 0; i < inputRestrictions.size(); ++i) {
         ExternalNodeMap::iterator intNodeID = ext2IntNodeMap.find(inputRestrictions[i].fromNode);
         if( intNodeID == ext2IntNodeMap.end()) {
@@ -380,19 +383,43 @@ NodeID readDDSGGraphFromStream(std::istream &in, std::vector<EdgeT>& edgeList, s
 }
 
 template<typename NodeT, typename EdgeT>
-unsigned readHSGRFromStream(std::istream &in, std::vector<NodeT>& nodeList, std::vector<EdgeT> & edgeList, unsigned * checkSum) {
+unsigned readHSGRFromStream(
-    unsigned numberOfNodes = 0;
+    std::istream &hsgr_input_stream,
-    in.read((char*) checkSum, sizeof(unsigned));
+    std::vector<NodeT> & node_list,
-    in.read((char*) & numberOfNodes, sizeof(unsigned));
+    std::vector<EdgeT> & edge_list,
-    nodeList.resize(numberOfNodes + 1);
+    unsigned * check_sum
-    in.read((char*) &(nodeList[0]), numberOfNodes*sizeof(NodeT));
+) {
+    UUID uuid_loaded, uuid_orig;
+    hsgr_input_stream.read((char *)&uuid_loaded, sizeof(UUID));
+    if( !uuid_loaded.TestGraphUtil(uuid_orig) ) {
+        WARN(
+            ".hsgr was prepared with different build.\n"
+            "Reprocess to get rid of this warning."
+            )
+    }
 
-    unsigned numberOfEdges = 0;
+    unsigned number_of_nodes = 0;
-    in.read((char*) &numberOfEdges, sizeof(unsigned));
+    hsgr_input_stream.read((char*) check_sum, sizeof(unsigned));
-    edgeList.resize(numberOfEdges);
+    hsgr_input_stream.read((char*) & number_of_nodes, sizeof(unsigned));
-    in.read((char*) &(edgeList[0]), numberOfEdges*sizeof(EdgeT));
+    node_list.resize(number_of_nodes + 1);
+    hsgr_input_stream.read(
+        (char*) &(node_list[0]),
+        number_of_nodes*sizeof(NodeT)
+    );
 
-    return numberOfNodes;
+    unsigned number_of_edges = 0;
+    hsgr_input_stream.read(
+        (char*) &number_of_edges,
+        sizeof(unsigned)
+    );
+
+    edge_list.resize(number_of_edges);
+    hsgr_input_stream.read(
+        (char*) &(edge_list[0]),
+        number_of_edges*sizeof(EdgeT)
+    );
+
+    return number_of_nodes;
 }
 
 #endif // GRAPHLOADER_H

Util/UUID.cpp.in

@@ -59,6 +59,37 @@ const bool UUID::IsMagicNumberOK() const {
 	return 1297240911 == magic_number;
 }
 
-const bool UUID::TestHSGR(const UUID & other) const {
+const bool UUID::TestGraphUtil(const UUID & other) const {
+    if(!other.IsMagicNumberOK()) {
+        ERR("hsgr input file misses magic number. Check or reprocess the file");
+    }
 	return std::equal(md5_graph, md5_graph+32, other.md5_graph);
-}
+}
+
+const bool UUID::TestPrepare(const UUID & other) const {
+    if(!other.IsMagicNumberOK()) {
+        ERR("extracted input file misses magic number. Check or reprocess the file");
+    }
+	return std::equal(md5_prepare, md5_prepare+32, other.md5_prepare);
+}
+
+const bool UUID::TestRTree(const UUID & other) const {
+    if(!other.IsMagicNumberOK()) {
+        ERR("r-tree input file misses magic number. Check or reprocess the file");
+    }
+	return std::equal(md5_tree, md5_tree+32, other.md5_tree);
+}
+
+const bool UUID::TestNodeInfo(const UUID & other) const {
+    if(!other.IsMagicNumberOK()) {
+        ERR("nodes file misses magic number. Check or reprocess the file");
+    }
+	return std::equal(md5_nodeinfo, md5_nodeinfo+32, other.md5_nodeinfo);
+}
+
+const bool UUID::TestQueryObjects(const UUID & other) const {
+    if(!other.IsMagicNumberOK()) {
+        ERR("missing magic number. Check or reprocess the file");
+    }
+	return std::equal(md5_objects, md5_objects+32, other.md5_objects);
+}

Util/UUID.h

@@ -21,6 +21,8 @@ or see http://www.gnu.org/licenses/agpl.txt.
 #ifndef UUID_H
 #define UUID_H
 
+#include "../typedefs.h"
+
 #include <boost/noncopyable.hpp>
 #include <boost/uuid/uuid.hpp>            // uuid class
 #include <boost/uuid/uuid_generators.hpp> // generators
@@ -39,7 +41,11 @@ public:
 	~UUID();
 	const boost::uuids::uuid & GetUUID() const;
 	const bool IsMagicNumberOK() const;
-	const bool TestHSGR(const UUID & other) const;
+	const bool TestGraphUtil(const UUID & other) const;
+	const bool TestPrepare(const UUID & other) const;
+	const bool TestRTree(const UUID & other) const;
+	const bool TestNodeInfo(const UUID & other) const;
+	const bool TestQueryObjects(const UUID & other) const;
 private:
     const unsigned magic_number;
     char md5_prepare[33];

createHierarchy.cpp

@@ -78,7 +78,16 @@ int main (int argc, char *argv[]) {
             ERR("Could not access <osrm-restrictions> files");
         }
         _Restriction restriction;
+        UUID uuid_loaded, uuid_orig;
         unsigned usableRestrictionsCounter(0);
+        restrictionsInstream.read((char*)&uuid_loaded, sizeof(UUID));
+        if( !uuid_loaded.TestPrepare(uuid_orig) ) {
+        WARN(
+            ".restrictions was prepared with different build.\n"
+            "Reprocess to get rid of this warning."
+            )
+        }
+
         restrictionsInstream.read((char*)&usableRestrictionsCounter, sizeof(unsigned));
         inputRestrictions.resize(usableRestrictionsCounter);
         restrictionsInstream.read((char *)&(inputRestrictions[0]), usableRestrictionsCounter*sizeof(_Restriction));
@@ -211,8 +220,8 @@ int main (int argc, char *argv[]) {
         unsigned numberOfNodes = 0;
         unsigned numberOfEdges = contractedEdgeList.size();
         INFO("Serializing compacted graph of " << numberOfEdges << " edges");
-        std::ofstream edgeOutFile(graphOut.c_str(), std::ios::binary);
+        std::ofstream hsgr_output_stream(graphOut.c_str(), std::ios::binary);
-
+        hsgr_output_stream.write((char*)&uuid_orig, sizeof(UUID) );
         BOOST_FOREACH(const QueryEdge & edge, contractedEdgeList) {
             if(edge.source > numberOfNodes) {
                 numberOfNodes = edge.source;
@@ -237,11 +246,11 @@ int main (int argc, char *argv[]) {
         }
         ++numberOfNodes;
         //Serialize numberOfNodes, nodes
-        edgeOutFile.write((char*) &crc32OfNodeBasedEdgeList, sizeof(unsigned));
+        hsgr_output_stream.write((char*) &crc32OfNodeBasedEdgeList, sizeof(unsigned));
-        edgeOutFile.write((char*) &numberOfNodes, sizeof(unsigned));
+        hsgr_output_stream.write((char*) &numberOfNodes, sizeof(unsigned));
-        edgeOutFile.write((char*) &_nodes[0], sizeof(StaticGraph<EdgeData>::_StrNode)*(numberOfNodes));
+        hsgr_output_stream.write((char*) &_nodes[0], sizeof(StaticGraph<EdgeData>::_StrNode)*(numberOfNodes));
         //Serialize number of Edges
-        edgeOutFile.write((char*) &position, sizeof(unsigned));
+        hsgr_output_stream.write((char*) &position, sizeof(unsigned));
         --numberOfNodes;
         edge = 0;
         int usedEdgeCounter = 0;
@@ -256,7 +265,7 @@ int main (int argc, char *argv[]) {
                     ERR("Failed at edges of node " << node << " of " << numberOfNodes);
                 }
                 //Serialize edges
-                edgeOutFile.write((char*) &currentEdge, sizeof(StaticGraph<EdgeData>::_StrEdge));
+                hsgr_output_stream.write((char*) &currentEdge, sizeof(StaticGraph<EdgeData>::_StrEdge));
                 ++edge;
                 ++usedEdgeCounter;
             }
@@ -265,7 +274,7 @@ int main (int argc, char *argv[]) {
         INFO("Expansion  : " << (nodeBasedNodeNumber/expansionHasFinishedTime) << " nodes/sec and "<< (edgeBasedNodeNumber/expansionHasFinishedTime) << " edges/sec");
         INFO("Contraction: " << (edgeBasedNodeNumber/expansionHasFinishedTime) << " nodes/sec and "<< usedEdgeCounter/endTime << " edges/sec");
 
-        edgeOutFile.close();
+        hsgr_output_stream.close();
         //cleanedEdgeList.clear();
         _nodes.clear();
         INFO("finished preprocessing");

extractor.cpp

@@ -28,6 +28,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
 #include "Util/MachineInfo.h"
 #include "Util/OpenMPWrapper.h"
 #include "Util/StringUtil.h"
+#include "Util/UUID.h"
 #include "typedefs.h"
 
 #include <cstdlib>
@@ -37,6 +38,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
 #include <string>
 
 ExtractorCallbacks * extractCallBacks;
+UUID uuid;
 
 int main (int argc, char *argv[]) {
     try {

routed.cpp

@@ -26,6 +26,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
 #include "Util/BaseConfiguration.h"
 #include "Util/InputFileUtil.h"
 #include "Util/OpenMPWrapper.h"
+#include "Util/UUID.h"
 
 #ifdef __linux__
 #include "Util/LinuxStackTrace.h"
@@ -77,7 +78,9 @@ int main (int argc, char * argv[]) {
     //}
 
     try {
-        std::cout << "\n starting up engines, compile at " <<
+        //std::cout << "fingerprint: " << UUID::GetInstance().GetUUID() << std::endl;
+
+        std::cout << "starting up engines, compiled at " <<
                                         __DATE__ << ", " __TIME__ << std::endl;
 
 #ifndef _WIN32