From 4a39a4af1c1f647651dbfb4d74493f556385bf5e Mon Sep 17 00:00:00 2001 From: DennisOSRM Date: Mon, 22 Jul 2013 16:34:06 +0200 Subject: [PATCH] Check for valid data files. Implements #224 --- Extractor/ExtractionContainers.cpp | 4 +- Extractor/ExtractionContainers.h | 3 +- Server/DataStructures/QueryObjectsStorage.cpp | 17 +++-- Util/GraphLoader.h | 67 +++++++++++++------ Util/UUID.cpp.in | 35 +++++++++- Util/UUID.h | 8 ++- createHierarchy.cpp | 25 ++++--- extractor.cpp | 2 + routed.cpp | 5 +- 9 files changed, 123 insertions(+), 43 deletions(-) diff --git a/Extractor/ExtractionContainers.cpp b/Extractor/ExtractionContainers.cpp index 14e8352b5..292e994cf 100644 --- a/Extractor/ExtractionContainers.cpp +++ b/Extractor/ExtractionContainers.cpp @@ -121,6 +121,7 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con //serialize restrictions std::ofstream restrictionsOutstream; restrictionsOutstream.open(restrictionsFileName.c_str(), std::ios::binary); + restrictionsOutstream.write((char*)&uuid, sizeof(UUID)); restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned)); for(restrictionsIT = restrictionsVector.begin(); restrictionsIT != restrictionsVector.end(); ++restrictionsIT) { if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) { @@ -131,6 +132,7 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con std::ofstream fout; fout.open(output_file_name.c_str(), std::ios::binary); + fout.write((char*)&uuid, sizeof(UUID)); fout.write((char*)&usedNodeCounter, sizeof(unsigned)); time = get_timestamp(); std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush; @@ -158,7 +160,7 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con std::cout << "[extractor] setting number of nodes ... " << std::flush; std::ios::pos_type positionInFile = fout.tellp(); - fout.seekp(std::ios::beg); + fout.seekp(std::ios::beg+sizeof(UUID)); fout.write((char*)&usedNodeCounter, sizeof(unsigned)); fout.seekp(positionInFile); diff --git a/Extractor/ExtractionContainers.h b/Extractor/ExtractionContainers.h index f3815c12f..78c1b2a62 100644 --- a/Extractor/ExtractionContainers.h +++ b/Extractor/ExtractionContainers.h @@ -23,6 +23,7 @@ #include "ExtractorStructs.h" #include "../DataStructures/TimingUtil.h" +#include "../Util/UUID.h" #include #include @@ -63,7 +64,7 @@ public: STXXLStringVector nameVector; STXXLRestrictionsVector restrictionsVector; STXXLWayIDStartEndVector wayStartEndVector; - + const UUID uuid; }; #endif /* EXTRACTIONCONTAINERS_H_ */ diff --git a/Server/DataStructures/QueryObjectsStorage.cpp b/Server/DataStructures/QueryObjectsStorage.cpp index 16de52eb8..69142434c 100644 --- a/Server/DataStructures/QueryObjectsStorage.cpp +++ b/Server/DataStructures/QueryObjectsStorage.cpp @@ -58,24 +58,23 @@ QueryObjectsStorage::QueryObjectsStorage( getline(timestampInStream, timestamp); timestampInStream.close(); } - if(!timestamp.length()) + if(!timestamp.length()) { timestamp = "n/a"; - if(25 < timestamp.length()) + } + if(25 < timestamp.length()) { timestamp.resize(25); + } INFO("Loading auxiliary information"); //Init nearest neighbor data structure - std::ifstream nodesInStream(nodesPath.c_str(), std::ios::binary); - if(!nodesInStream) { ERR(nodesPath << " not found"); } - std::ifstream edgesInStream(edgesPath.c_str(), std::ios::binary); - if(!edgesInStream) { ERR(edgesPath << " not found"); } nodeHelpDesk = new NodeInformationHelpDesk( - ramIndexPath.c_str(), - fileIndexPath.c_str(), + ramIndexPath, + fileIndexPath, + nodesPath, + edgesPath, n, checkSum ); - nodeHelpDesk->initNNGrid(nodesInStream, edgesInStream); //deserialize street name list INFO("Loading names index"); diff --git a/Util/GraphLoader.h b/Util/GraphLoader.h index 6fdcd3228..cd3e0cf7d 100644 --- a/Util/GraphLoader.h +++ b/Util/GraphLoader.h @@ -50,28 +50,31 @@ struct _ExcessRemover { }; template -NodeID readBinaryOSRMGraphFromStream(std::istream &in, std::vector& edgeList, std::vector &bollardNodes, std::vector &trafficLightNodes, std::vector * int2ExtNodeMap, std::vector<_Restriction> & inputRestrictions) { +NodeID readBinaryOSRMGraphFromStream( + std::istream &in, + std::vector& edgeList, + std::vector &bollardNodes, + std::vector &trafficLightNodes, + std::vector * int2ExtNodeMap, + std::vector<_Restriction> & inputRestrictions +) { const UUID uuid_orig; UUID uuid_loaded; in.read((char *) &uuid_loaded, sizeof(UUID)); - if(!uuid_loaded.IsMagicNumberOK()) { - ERR("hsgr input file misses magic number. Check or reprocess the file"); - } - if( uuid_loaded.TestHSGR(uuid_orig) ) { + if( !uuid_loaded.TestGraphUtil(uuid_orig) ) { WARN( - ".hsgr was prepared with different build.\n" + ".osrm was prepared with different build.\n" "Reprocess to get rid of this warning." ) } - NodeID n, source, target; EdgeID m; short dir;// direction (0 = open, 1 = forward, 2+ = open) ExternalNodeMap ext2IntNodeMap; in.read((char*)&n, sizeof(NodeID)); - DEBUG("Importing n = " << n << " nodes "); + INFO("Importing n = " << n << " nodes "); _Node node; for (NodeID i=0; i& edgeL std::vector(trafficLightNodes).swap(trafficLightNodes); in.read((char*)&m, sizeof(unsigned)); - DEBUG(" and " << m << " edges "); + INFO(" and " << m << " edges "); for(unsigned i = 0; i < inputRestrictions.size(); ++i) { ExternalNodeMap::iterator intNodeID = ext2IntNodeMap.find(inputRestrictions[i].fromNode); if( intNodeID == ext2IntNodeMap.end()) { @@ -380,19 +383,43 @@ NodeID readDDSGGraphFromStream(std::istream &in, std::vector& edgeList, s } template -unsigned readHSGRFromStream(std::istream &in, std::vector& nodeList, std::vector & edgeList, unsigned * checkSum) { - unsigned numberOfNodes = 0; - in.read((char*) checkSum, sizeof(unsigned)); - in.read((char*) & numberOfNodes, sizeof(unsigned)); - nodeList.resize(numberOfNodes + 1); - in.read((char*) &(nodeList[0]), numberOfNodes*sizeof(NodeT)); +unsigned readHSGRFromStream( + std::istream &hsgr_input_stream, + std::vector & node_list, + std::vector & edge_list, + unsigned * check_sum +) { + UUID uuid_loaded, uuid_orig; + hsgr_input_stream.read((char *)&uuid_loaded, sizeof(UUID)); + if( !uuid_loaded.TestGraphUtil(uuid_orig) ) { + WARN( + ".hsgr was prepared with different build.\n" + "Reprocess to get rid of this warning." + ) + } - unsigned numberOfEdges = 0; - in.read((char*) &numberOfEdges, sizeof(unsigned)); - edgeList.resize(numberOfEdges); - in.read((char*) &(edgeList[0]), numberOfEdges*sizeof(EdgeT)); + unsigned number_of_nodes = 0; + hsgr_input_stream.read((char*) check_sum, sizeof(unsigned)); + hsgr_input_stream.read((char*) & number_of_nodes, sizeof(unsigned)); + node_list.resize(number_of_nodes + 1); + hsgr_input_stream.read( + (char*) &(node_list[0]), + number_of_nodes*sizeof(NodeT) + ); - return numberOfNodes; + unsigned number_of_edges = 0; + hsgr_input_stream.read( + (char*) &number_of_edges, + sizeof(unsigned) + ); + + edge_list.resize(number_of_edges); + hsgr_input_stream.read( + (char*) &(edge_list[0]), + number_of_edges*sizeof(EdgeT) + ); + + return number_of_nodes; } #endif // GRAPHLOADER_H diff --git a/Util/UUID.cpp.in b/Util/UUID.cpp.in index c5b522af7..36726c4ae 100644 --- a/Util/UUID.cpp.in +++ b/Util/UUID.cpp.in @@ -59,6 +59,37 @@ const bool UUID::IsMagicNumberOK() const { return 1297240911 == magic_number; } -const bool UUID::TestHSGR(const UUID & other) const { +const bool UUID::TestGraphUtil(const UUID & other) const { + if(!other.IsMagicNumberOK()) { + ERR("hsgr input file misses magic number. Check or reprocess the file"); + } return std::equal(md5_graph, md5_graph+32, other.md5_graph); -} \ No newline at end of file +} + +const bool UUID::TestPrepare(const UUID & other) const { + if(!other.IsMagicNumberOK()) { + ERR("extracted input file misses magic number. Check or reprocess the file"); + } + return std::equal(md5_prepare, md5_prepare+32, other.md5_prepare); +} + +const bool UUID::TestRTree(const UUID & other) const { + if(!other.IsMagicNumberOK()) { + ERR("r-tree input file misses magic number. Check or reprocess the file"); + } + return std::equal(md5_tree, md5_tree+32, other.md5_tree); +} + +const bool UUID::TestNodeInfo(const UUID & other) const { + if(!other.IsMagicNumberOK()) { + ERR("nodes file misses magic number. Check or reprocess the file"); + } + return std::equal(md5_nodeinfo, md5_nodeinfo+32, other.md5_nodeinfo); +} + +const bool UUID::TestQueryObjects(const UUID & other) const { + if(!other.IsMagicNumberOK()) { + ERR("missing magic number. Check or reprocess the file"); + } + return std::equal(md5_objects, md5_objects+32, other.md5_objects); +} diff --git a/Util/UUID.h b/Util/UUID.h index 354540e07..188ab26a1 100644 --- a/Util/UUID.h +++ b/Util/UUID.h @@ -21,6 +21,8 @@ or see http://www.gnu.org/licenses/agpl.txt. #ifndef UUID_H #define UUID_H +#include "../typedefs.h" + #include #include // uuid class #include // generators @@ -39,7 +41,11 @@ public: ~UUID(); const boost::uuids::uuid & GetUUID() const; const bool IsMagicNumberOK() const; - const bool TestHSGR(const UUID & other) const; + const bool TestGraphUtil(const UUID & other) const; + const bool TestPrepare(const UUID & other) const; + const bool TestRTree(const UUID & other) const; + const bool TestNodeInfo(const UUID & other) const; + const bool TestQueryObjects(const UUID & other) const; private: const unsigned magic_number; char md5_prepare[33]; diff --git a/createHierarchy.cpp b/createHierarchy.cpp index 3f76fcd19..85cfea942 100644 --- a/createHierarchy.cpp +++ b/createHierarchy.cpp @@ -78,7 +78,16 @@ int main (int argc, char *argv[]) { ERR("Could not access files"); } _Restriction restriction; + UUID uuid_loaded, uuid_orig; unsigned usableRestrictionsCounter(0); + restrictionsInstream.read((char*)&uuid_loaded, sizeof(UUID)); + if( !uuid_loaded.TestPrepare(uuid_orig) ) { + WARN( + ".restrictions was prepared with different build.\n" + "Reprocess to get rid of this warning." + ) + } + restrictionsInstream.read((char*)&usableRestrictionsCounter, sizeof(unsigned)); inputRestrictions.resize(usableRestrictionsCounter); restrictionsInstream.read((char *)&(inputRestrictions[0]), usableRestrictionsCounter*sizeof(_Restriction)); @@ -211,8 +220,8 @@ int main (int argc, char *argv[]) { unsigned numberOfNodes = 0; unsigned numberOfEdges = contractedEdgeList.size(); INFO("Serializing compacted graph of " << numberOfEdges << " edges"); - std::ofstream edgeOutFile(graphOut.c_str(), std::ios::binary); - + std::ofstream hsgr_output_stream(graphOut.c_str(), std::ios::binary); + hsgr_output_stream.write((char*)&uuid_orig, sizeof(UUID) ); BOOST_FOREACH(const QueryEdge & edge, contractedEdgeList) { if(edge.source > numberOfNodes) { numberOfNodes = edge.source; @@ -237,11 +246,11 @@ int main (int argc, char *argv[]) { } ++numberOfNodes; //Serialize numberOfNodes, nodes - edgeOutFile.write((char*) &crc32OfNodeBasedEdgeList, sizeof(unsigned)); - edgeOutFile.write((char*) &numberOfNodes, sizeof(unsigned)); - edgeOutFile.write((char*) &_nodes[0], sizeof(StaticGraph::_StrNode)*(numberOfNodes)); + hsgr_output_stream.write((char*) &crc32OfNodeBasedEdgeList, sizeof(unsigned)); + hsgr_output_stream.write((char*) &numberOfNodes, sizeof(unsigned)); + hsgr_output_stream.write((char*) &_nodes[0], sizeof(StaticGraph::_StrNode)*(numberOfNodes)); //Serialize number of Edges - edgeOutFile.write((char*) &position, sizeof(unsigned)); + hsgr_output_stream.write((char*) &position, sizeof(unsigned)); --numberOfNodes; edge = 0; int usedEdgeCounter = 0; @@ -256,7 +265,7 @@ int main (int argc, char *argv[]) { ERR("Failed at edges of node " << node << " of " << numberOfNodes); } //Serialize edges - edgeOutFile.write((char*) ¤tEdge, sizeof(StaticGraph::_StrEdge)); + hsgr_output_stream.write((char*) ¤tEdge, sizeof(StaticGraph::_StrEdge)); ++edge; ++usedEdgeCounter; } @@ -265,7 +274,7 @@ int main (int argc, char *argv[]) { INFO("Expansion : " << (nodeBasedNodeNumber/expansionHasFinishedTime) << " nodes/sec and "<< (edgeBasedNodeNumber/expansionHasFinishedTime) << " edges/sec"); INFO("Contraction: " << (edgeBasedNodeNumber/expansionHasFinishedTime) << " nodes/sec and "<< usedEdgeCounter/endTime << " edges/sec"); - edgeOutFile.close(); + hsgr_output_stream.close(); //cleanedEdgeList.clear(); _nodes.clear(); INFO("finished preprocessing"); diff --git a/extractor.cpp b/extractor.cpp index a324f5970..5e580d8e7 100644 --- a/extractor.cpp +++ b/extractor.cpp @@ -28,6 +28,7 @@ or see http://www.gnu.org/licenses/agpl.txt. #include "Util/MachineInfo.h" #include "Util/OpenMPWrapper.h" #include "Util/StringUtil.h" +#include "Util/UUID.h" #include "typedefs.h" #include @@ -37,6 +38,7 @@ or see http://www.gnu.org/licenses/agpl.txt. #include ExtractorCallbacks * extractCallBacks; +UUID uuid; int main (int argc, char *argv[]) { try { diff --git a/routed.cpp b/routed.cpp index 596f610d7..1f4e43550 100644 --- a/routed.cpp +++ b/routed.cpp @@ -26,6 +26,7 @@ or see http://www.gnu.org/licenses/agpl.txt. #include "Util/BaseConfiguration.h" #include "Util/InputFileUtil.h" #include "Util/OpenMPWrapper.h" +#include "Util/UUID.h" #ifdef __linux__ #include "Util/LinuxStackTrace.h" @@ -77,7 +78,9 @@ int main (int argc, char * argv[]) { //} try { - std::cout << "\n starting up engines, compile at " << + //std::cout << "fingerprint: " << UUID::GetInstance().GetUUID() << std::endl; + + std::cout << "starting up engines, compiled at " << __DATE__ << ", " __TIME__ << std::endl; #ifndef _WIN32