street name list is now a char array /w indices array

DataStructures/SearchEngine.cpp

@@ -20,12 +20,8 @@ or see http://www.gnu.org/licenses/agpl.txt.
 
 #include "SearchEngine.h"
 
-SearchEngine::SearchEngine(
+SearchEngine::SearchEngine( QueryObjectsStorage * query_objects ) :
-    QueryGraph * g,
+    _queryData(query_objects),
-    NodeInformationHelpDesk * nh,
-    std::vector<std::string> & n
-) :
-        _queryData(g, nh, n),
     shortestPath(_queryData),
     alternativePaths(_queryData)
 {}
@@ -73,12 +69,9 @@ NodeID SearchEngine::GetNameIDForOriginDestinationNodeID(
 }
 
 std::string SearchEngine::GetEscapedNameForNameID(const unsigned nameID) const {
-    bool is_name_invalid = (nameID >= _queryData.names.size() || nameID == 0);
+    std::string result;
-    if (is_name_invalid) {
+    _queryData.query_objects->GetName(nameID, result);
-        return std::string("");
+    return HTMLEntitize(result);
-    }
-
-    return HTMLEntitize(_queryData.names.at(nameID));
 }
 
 SearchEngineHeapPtr SearchEngineData::forwardHeap;

DataStructures/SearchEngine.h

@@ -28,6 +28,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
 #include "SearchEngineData.h"
 #include "../RoutingAlgorithms/AlternativePathRouting.h"
 #include "../RoutingAlgorithms/ShortestPathRouting.h"
+#include "../Server/DataStructures/QueryObjectsStorage.h"
 
 #include "../Util/StringUtil.h"
 #include "../typedefs.h"
@@ -44,11 +45,7 @@ public:
     ShortestPathRouting<SearchEngineData> shortestPath;
     AlternativeRouting<SearchEngineData> alternativePaths;
 
-    SearchEngine(
+    SearchEngine( QueryObjectsStorage * query_objects );
-        QueryGraph * g,
-        NodeInformationHelpDesk * nh,
-        std::vector<std::string> & n
-    );
 	~SearchEngine();
 
 	void GetCoordinatesForNodeID(NodeID id, FixedPointCoordinate& result) const;

DataStructures/SearchEngineData.h

@@ -20,8 +20,8 @@ or see http://www.gnu.org/licenses/agpl.txt.
 
 #include "BinaryHeap.h"
 #include "QueryEdge.h"
-#include "NodeInformationHelpDesk.h"
 #include "StaticGraph.h"
+#include "../Server/DataStructures/QueryObjectsStorage.h"
 
 #include "../typedefs.h"
 
@@ -41,10 +41,17 @@ typedef boost::thread_specific_ptr<QueryHeapType> SearchEngineHeapPtr;
 struct SearchEngineData {
     typedef QueryGraph Graph;
     typedef QueryHeapType QueryHeap;
-    SearchEngineData(QueryGraph * g, NodeInformationHelpDesk * nh, std::vector<std::string> & n) :graph(g), nodeHelpDesk(nh), names(n) {}
+    SearchEngineData(QueryObjectsStorage * query_objects)
+     :
+        query_objects(query_objects),
+        graph(query_objects->graph),
+        nodeHelpDesk(query_objects->nodeHelpDesk)
+    {}
+
+    const QueryObjectsStorage       * query_objects;
     const QueryGraph                * graph;
-    NodeInformationHelpDesk * nodeHelpDesk;
+    const NodeInformationHelpDesk   * nodeHelpDesk;
-    std::vector<std::string> & names;
+
     static SearchEngineHeapPtr forwardHeap;
     static SearchEngineHeapPtr backwardHeap;
     static SearchEngineHeapPtr forwardHeap2;

Extractor/ExtractionContainers.cpp

@@ -282,33 +282,39 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con
         fout.close();
         std::cout << "ok" << std::endl;
         time = get_timestamp();
-        std::cout << "[extractor] writing street name index ... " << std::flush;
-        std::string nameOutFileName = (output_file_name + ".names");
-        std::ofstream nameOutFile(nameOutFileName.c_str(), std::ios::binary);
-        unsigned sizeOfNameIndex = nameVector.size();
-        nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned));
 
-        BOOST_FOREACH(const std::string & str, nameVector) {
+        std::cout << "[extractor] writing street name index ... " << std::flush;
-            unsigned lengthOfRawString = strlen(str.c_str());
+        std::string name_file_streamName = (output_file_name + ".names");
-            nameOutFile.write((char *)&(lengthOfRawString), sizeof(unsigned));
+        boost::filesystem::ofstream name_file_stream(
-            nameOutFile.write(str.c_str(), lengthOfRawString);
+            name_file_streamName,
+            std::ios::binary
+        );
+
+        const unsigned number_of_ways = name_list.size()+1;
+        name_file_stream.write((char *)&(number_of_ways), sizeof(unsigned));
+        unsigned name_lengths_prefix_sum = 0;
+        BOOST_FOREACH(const std::string & str, name_list) {
+            name_file_stream.write(
+                (char *)&(name_lengths_prefix_sum),
+                sizeof(unsigned)
+            );
+            name_lengths_prefix_sum += strlen(str.c_str());
+        }
+        name_file_stream.write(
+            (char *)&(name_lengths_prefix_sum),
+            sizeof(unsigned)
+        );
+        //duplicate on purpose!
+        name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
+        BOOST_FOREACH(const std::string & str, name_list) {
+            const unsigned lengthOfRawString = strlen(str.c_str());
+            name_file_stream.write(str.c_str(), lengthOfRawString);
         }
 
-        nameOutFile.close();
+        name_file_stream.close();
         std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
-
+        SimpleLogger().Write() <<
-        //        time = get_timestamp();
+            "Processed " << usedNodeCounter << " nodes and " << usedEdgeCounter << " edges";
-        //        cout << "[extractor] writing address list      ... " << flush;
-        //
-        //        adressFileName.append(".address");
-        //        ofstream addressOutFile(adressFileName.c_str());
-        //        for(STXXLAddressVector::iterator it = adressVector.begin(); it != adressVector.end(); it++) {
-        //            addressOutFile << it->node.id << "|" << it->node.lat << "|" << it->node.lon << "|" << it->city << "|" << it->street << "|" << it->housenumber << "|" << it->state << "|" << it->country << "\n";
-        //        }
-        //        addressOutFile.close();
-        //        cout << "ok, after " << get_timestamp() - time << "s" << endl;
-
-        SimpleLogger().Write() << "Processed " << usedNodeCounter << " nodes and " << usedEdgeCounter << " edges";
 
 
     } catch ( const std::exception& e ) {

Extractor/ExtractionContainers.h

@@ -27,6 +27,8 @@
 #include "../Util/UUID.h"
 
 #include <boost/foreach.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
 #include <stxxl.h>
 
 class ExtractionContainers {
@@ -38,30 +40,35 @@ public:
     typedef stxxl::vector<_RawRestrictionContainer> STXXLRestrictionsVector;
     typedef stxxl::vector<_WayIDStartAndEndEdge>    STXXLWayIDStartEndVector;
 
+
+    STXXLNodeIDVector                               usedNodeIDs;
+    STXXLNodeVector                                 allNodes;
+    STXXLEdgeVector                                 allEdges;
+    STXXLStringVector                               name_list;
+    STXXLRestrictionsVector                         restrictionsVector;
+    STXXLWayIDStartEndVector                        wayStartEndVector;
+    const UUID uuid;
+
     ExtractionContainers() {
         //Check if another instance of stxxl is already running or if there is a general problem
         stxxl::vector<unsigned> testForRunningInstance;
-        nameVector.push_back("");
+        name_list.push_back("");
     }
 
     virtual ~ExtractionContainers() {
         usedNodeIDs.clear();
         allNodes.clear();
         allEdges.clear();
-        nameVector.clear();
+        name_list.clear();
         restrictionsVector.clear();
         wayStartEndVector.clear();
     }
 
-    void PrepareData( const std::string & output_file_name, const std::string restrictionsFileName, const unsigned amountOfRAM);
+    void PrepareData(
-
+        const std::string & output_file_name,
-    STXXLNodeIDVector           usedNodeIDs;
+        const std::string restrictionsFileName,
-    STXXLNodeVector             allNodes;
+        const unsigned amountOfRAM
-    STXXLEdgeVector             allEdges;
+    );
-    STXXLStringVector           nameVector;
-    STXXLRestrictionsVector     restrictionsVector;
-    STXXLWayIDStartEndVector    wayStartEndVector;
-    const UUID uuid;
 };
 
 #endif /* EXTRACTIONCONTAINERS_H_ */

Extractor/ExtractorCallbacks.cpp

@@ -65,8 +65,8 @@ void ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
         //Get the unique identifier for the street name
         const StringMap::const_iterator string_map_iterator = stringMap->find(parsed_way.name);
         if(stringMap->end() == string_map_iterator) {
-            parsed_way.nameID = externalMemory->nameVector.size();
+            parsed_way.nameID = externalMemory->name_list.size();
-            externalMemory->nameVector.push_back(parsed_way.name);
+            externalMemory->name_list.push_back(parsed_way.name);
             stringMap->insert(std::make_pair(parsed_way.name, parsed_way.nameID));
         } else {
             parsed_way.nameID = string_map_iterator->second;

Plugins/NearestPlugin.h

@@ -34,11 +34,9 @@ class NearestPlugin : public BasePlugin {
 public:
     NearestPlugin(QueryObjectsStorage * objects )
      :
-        names(objects->names),
+        m_query_objects(objects),
         descriptor_string("nearest")
     {
-        nodeHelpDesk = objects->nodeHelpDesk;
-
         descriptorTable.insert(std::make_pair(""    , 0)); //default descriptor
         descriptorTable.insert(std::make_pair("json", 1));
     }
@@ -53,12 +51,16 @@ public:
             reply = http::Reply::stockReply(http::Reply::badRequest);
             return;
         }
-
+        NodeInformationHelpDesk * nodeHelpDesk = m_query_objects->nodeHelpDesk;
         //query to helpdesk
         PhantomNode result;
-        nodeHelpDesk->FindPhantomNodeForCoordinate(routeParameters.coordinates[0], result, routeParameters.zoomLevel);
+        nodeHelpDesk->FindPhantomNodeForCoordinate(
+            routeParameters.coordinates[0],
+            result,
+            routeParameters.zoomLevel
+        );
 
-        std::string tmp;
+        std::string temp_string;
         //json
 
         if("" != routeParameters.jsonpParameter) {
@@ -70,23 +72,26 @@ public:
         reply.content += ("{");
         reply.content += ("\"version\":0.3,");
         reply.content += ("\"status\":");
-        if(UINT_MAX != result.edgeBasedNode)
+        if(UINT_MAX != result.edgeBasedNode) {
             reply.content += "0,";
-        else
+        } else {
             reply.content += "207,";
+        }
         reply.content += ("\"mapped_coordinate\":");
         reply.content += "[";
         if(UINT_MAX != result.edgeBasedNode) {
-            convertInternalLatLonToString(result.location.lat, tmp);
+            convertInternalLatLonToString(result.location.lat, temp_string);
-            reply.content += tmp;
+            reply.content += temp_string;
-            convertInternalLatLonToString(result.location.lon, tmp);
+            convertInternalLatLonToString(result.location.lon, temp_string);
             reply.content += ",";
-            reply.content += tmp;
+            reply.content += temp_string;
         }
         reply.content += "],";
         reply.content += "\"name\":\"";
-        if(UINT_MAX != result.edgeBasedNode)
+        if(UINT_MAX != result.edgeBasedNode) {
-            reply.content += names[result.nodeBasedEdgeNameID];
+            m_query_objects->GetName(result.nodeBasedEdgeNameID, temp_string);
+            reply.content += temp_string;
+        }
         reply.content += "\"";
         reply.content += ",\"transactionId\":\"OSRM Routing Engine JSON Nearest (v0.3)\"";
         reply.content += ("}");
@@ -104,14 +109,13 @@ public:
             reply.headers[2].value = "attachment; filename=\"location.json\"";
         }
         reply.headers[0].name = "Content-Length";
-        intToString(reply.content.size(), tmp);
+        intToString(reply.content.size(), temp_string);
-        reply.headers[0].value = tmp;
+        reply.headers[0].value = temp_string;
     }
 
 private:
-    NodeInformationHelpDesk * nodeHelpDesk;
+    QueryObjectsStorage * m_query_objects;
     HashTable<std::string, unsigned> descriptorTable;
-    std::vector<std::string> & names;
     std::string descriptor_string;
 };
 

Plugins/ViaRoutePlugin.h

@@ -43,7 +43,6 @@ or see http://www.gnu.org/licenses/agpl.txt.
 class ViaRoutePlugin : public BasePlugin {
 private:
     NodeInformationHelpDesk * nodeHelpDesk;
-    std::vector<std::string> & names;
     StaticGraph<QueryEdge::EdgeData> * graph;
     HashTable<std::string, unsigned> descriptorTable;
     SearchEngine * searchEnginePtr;
@@ -51,13 +50,13 @@ public:
 
     ViaRoutePlugin(QueryObjectsStorage * objects)
      :
-        names(objects->names),
+        // objects(objects),
         descriptor_string("viaroute")
     {
         nodeHelpDesk = objects->nodeHelpDesk;
         graph = objects->graph;
 
-        searchEnginePtr = new SearchEngine(graph, nodeHelpDesk, names);
+        searchEnginePtr = new SearchEngine(objects);
 
         descriptorTable.insert(std::make_pair(""    , 0));
         descriptorTable.insert(std::make_pair("json", 0));

Server/DataStructures/QueryObjectsStorage.cpp

@@ -22,54 +22,54 @@ or see http://www.gnu.org/licenses/agpl.txt.
 #include "QueryObjectsStorage.h"
 
 QueryObjectsStorage::QueryObjectsStorage(
-	const std::string & hsgrPath,
+	const std::string & hsgr_path,
-	const std::string & ramIndexPath,
+	const std::string & ram_index_path,
-	const std::string & fileIndexPath,
+	const std::string & file_index_path,
-	const std::string & nodesPath,
+	const std::string & nodes_path,
-	const std::string & edgesPath,
+	const std::string & edges_path,
-	const std::string & namesPath,
+	const std::string & names_path,
-	const std::string & timestampPath
+	const std::string & timestamp_path
 ) {
-	if( hsgrPath.empty() ) {
+	if( hsgr_path.empty() ) {
 		throw OSRMException("no hsgr file given in ini file");
 	}
-	if( ramIndexPath.empty() ) {
+	if( ram_index_path.empty() ) {
 		throw OSRMException("no ram index file given in ini file");
 	}
-	if( fileIndexPath.empty() ) {
+	if( file_index_path.empty() ) {
 		throw OSRMException("no mem index file given in ini file");
 	}
-	if( nodesPath.empty() ) {
+	if( nodes_path.empty() ) {
 		throw OSRMException("no nodes file given in ini file");
 	}
-	if( edgesPath.empty() ) {
+	if( edges_path.empty() ) {
 		throw OSRMException("no edges file given in ini file");
 	}
-	if( namesPath.empty() ) {
+	if( names_path.empty() ) {
 		throw OSRMException("no names file given in ini file");
 	}
 
 	SimpleLogger().Write() << "loading graph data";
 	//Deserialize road network graph
-	std::vector< QueryGraph::_StrNode> nodeList;
+	std::vector< QueryGraph::_StrNode> node_list;
-	std::vector< QueryGraph::_StrEdge> edgeList;
+	std::vector< QueryGraph::_StrEdge> edge_list;
-	const int n = readHSGRFromStream(
+	const int number_of_nodes = readHSGRFromStream(
-		hsgrPath,
+		hsgr_path,
-		nodeList,
+		node_list,
-		edgeList,
+		edge_list,
-		&checkSum
+		&check_sum
 	);
 
-	SimpleLogger().Write() << "Data checksum is " << checkSum;
+	SimpleLogger().Write() << "Data checksum is " << check_sum;
-	graph = new QueryGraph(nodeList, edgeList);
+	graph = new QueryGraph(node_list, edge_list);
-	assert(0 == nodeList.size());
+	assert(0 == node_list.size());
-	assert(0 == edgeList.size());
+	assert(0 == edge_list.size());
 
-	if(timestampPath.length()) {
+	if(timestamp_path.length()) {
 	    SimpleLogger().Write() << "Loading Timestamp";
-	    std::ifstream timestampInStream(timestampPath.c_str());
+	    std::ifstream timestampInStream(timestamp_path.c_str());
 	    if(!timestampInStream) {
-	    	SimpleLogger().Write(logWARNING) <<  timestampPath <<  " not found";
+	    	SimpleLogger().Write(logWARNING) <<  timestamp_path <<  " not found";
 	    }
 
 	    getline(timestampInStream, timestamp);
@@ -85,17 +85,17 @@ QueryObjectsStorage::QueryObjectsStorage(
     SimpleLogger().Write() << "Loading auxiliary information";
     //Init nearest neighbor data structure
 	nodeHelpDesk = new NodeInformationHelpDesk(
-		ramIndexPath,
+		ram_index_path,
-		fileIndexPath,
+		file_index_path,
-		nodesPath,
+		nodes_path,
-		edgesPath,
+		edges_path,
-		n,
+		number_of_nodes,
-		checkSum
+		check_sum
 	);
 
 	//deserialize street name list
 	SimpleLogger().Write() << "Loading names index";
-	boost::filesystem::path names_file(namesPath);
+	boost::filesystem::path names_file(names_path);
 
     if ( !boost::filesystem::exists( names_file ) ) {
         throw OSRMException("names file does not exist");
@@ -107,24 +107,55 @@ QueryObjectsStorage::QueryObjectsStorage(
 	boost::filesystem::ifstream name_stream(names_file, std::ios::binary);
 	unsigned size = 0;
 	name_stream.read((char *)&size, sizeof(unsigned));
-	BOOST_ASSERT_MSG(0 != size, "name file empty");
+	BOOST_ASSERT_MSG(0 != size, "name file broken");
+
+	m_name_begin_indices.resize(size);
+	name_stream.read((char*)&m_name_begin_indices[0], size*sizeof(unsigned));
+	name_stream.read((char *)&size, sizeof(unsigned));
+	BOOST_ASSERT_MSG(0 != size, "name file broken");
+
+	m_names_char_list.resize(size+1); //+1 is sentinel/dummy element
+	name_stream.read((char *)&m_names_char_list[0], size*sizeof(char));
+	BOOST_ASSERT_MSG(0 != m_names_char_list.size(), "could not load any names");
 
-	char buf[1024];
-	for( unsigned i = 0; i < size; ++i ) {
-		unsigned size_of_string = 0;
-		name_stream.read((char *)&size_of_string, sizeof(unsigned));
-		buf[size_of_string] = '\0'; // instead of memset
-		name_stream.read(buf, size_of_string);
-		names.push_back(buf);
-	}
-	std::vector<std::string>(names).swap(names);
-	BOOST_ASSERT_MSG(0 != names.size(), "could not load any names");
 	name_stream.close();
 	SimpleLogger().Write() << "All query data structures loaded";
 }
 
+void QueryObjectsStorage::GetName(
+	const unsigned name_id,
+	std::string & result
+) const {
+	if(UINT_MAX == name_id) {
+		result = "";
+		return;
+	}
+	BOOST_ASSERT_MSG(
+		name_id < m_name_begin_indices.size(),
+		"name id too high"
+	);
+	unsigned begin_index = m_name_begin_indices[name_id];
+	unsigned end_index = m_name_begin_indices[name_id+1];
+	BOOST_ASSERT_MSG(
+		begin_index < m_names_char_list.size(),
+		"begin index of name too high"
+	);
+	BOOST_ASSERT_MSG(
+		end_index < m_names_char_list.size(),
+		"end index of name too high"
+	);
+
+	BOOST_ASSERT_MSG(begin_index <= end_index, "string ends before begin");
+	result.clear();
+	result.resize(end_index - begin_index);
+	std::copy(
+		m_names_char_list.begin() + begin_index,
+		m_names_char_list.begin() + end_index,
+		result.begin()
+	);
+}
+
 QueryObjectsStorage::~QueryObjectsStorage() {
-	//        delete names;
 	delete graph;
 	delete nodeHelpDesk;
 }

Server/DataStructures/QueryObjectsStorage.h

@@ -42,10 +42,13 @@ struct QueryObjectsStorage {
     typedef QueryGraph::InputEdge               InputEdge;
 
     NodeInformationHelpDesk                   * nodeHelpDesk;
-    std::vector<std::string> names;
+    std::vector<char>                           m_names_char_list;
+    std::vector<unsigned>                       m_name_begin_indices;
     QueryGraph                                * graph;
     std::string                                 timestamp;
-    unsigned checkSum;
+    unsigned                                    check_sum;
+
+    void GetName(const unsigned name_id, std::string & result) const;
 
     QueryObjectsStorage(
         const std::string & hsgrPath,