Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

street name list is now a char array /w indices array

Browse Source
This commit is contained in:
Dennis Luxen 2013-08-20 17:05:36 +02:00
parent d0db09cb92
commit fb9822b507
10 changed files with 185 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
DataStructures/SearchEngine.cpp
View File

@ -20,14 +20,10 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "SearchEngine.h"
SearchEngine::SearchEngine(
QueryGraph * g,
NodeInformationHelpDesk * nh,
std::vector<std::string> & n
) :
_queryData(g, nh, n),
shortestPath(_queryData),
alternativePaths(_queryData)
SearchEngine::SearchEngine( QueryObjectsStorage * query_objects ) :
_queryData(query_objects),
shortestPath(_queryData),
alternativePaths(_queryData)
{}
SearchEngine::~SearchEngine() {}
@ -73,12 +69,9 @@ NodeID SearchEngine::GetNameIDForOriginDestinationNodeID(
}
std::string SearchEngine::GetEscapedNameForNameID(const unsigned nameID) const {
bool is_name_invalid = (nameID >= _queryData.names.size() || nameID == 0);
if (is_name_invalid) {
return std::string("");
}
return HTMLEntitize(_queryData.names.at(nameID));
std::string result;
_queryData.query_objects->GetName(nameID, result);
return HTMLEntitize(result);
}
SearchEngineHeapPtr SearchEngineData::forwardHeap;

7
DataStructures/SearchEngine.h
View File

@ -28,6 +28,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "SearchEngineData.h"
#include "../RoutingAlgorithms/AlternativePathRouting.h"
#include "../RoutingAlgorithms/ShortestPathRouting.h"
#include "../Server/DataStructures/QueryObjectsStorage.h"
#include "../Util/StringUtil.h"
#include "../typedefs.h"
@ -44,11 +45,7 @@ public:
ShortestPathRouting<SearchEngineData> shortestPath;
AlternativeRouting<SearchEngineData> alternativePaths;
SearchEngine(
QueryGraph * g,
NodeInformationHelpDesk * nh,
std::vector<std::string> & n
);
SearchEngine( QueryObjectsStorage * query_objects );
~SearchEngine();
void GetCoordinatesForNodeID(NodeID id, FixedPointCoordinate& result) const;

17
DataStructures/SearchEngineData.h
View File

@ -20,8 +20,8 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "BinaryHeap.h"
#include "QueryEdge.h"
#include "NodeInformationHelpDesk.h"
#include "StaticGraph.h"
#include "../Server/DataStructures/QueryObjectsStorage.h"
#include "../typedefs.h"
@ -41,10 +41,17 @@ typedef boost::thread_specific_ptr<QueryHeapType> SearchEngineHeapPtr;
struct SearchEngineData {
typedef QueryGraph Graph;
typedef QueryHeapType QueryHeap;
SearchEngineData(QueryGraph * g, NodeInformationHelpDesk * nh, std::vector<std::string> & n) :graph(g), nodeHelpDesk(nh), names(n) {}
const QueryGraph * graph;
NodeInformationHelpDesk * nodeHelpDesk;
std::vector<std::string> & names;
SearchEngineData(QueryObjectsStorage * query_objects)
:
query_objects(query_objects),
graph(query_objects->graph),
nodeHelpDesk(query_objects->nodeHelpDesk)
{}
const QueryObjectsStorage * query_objects;
const QueryGraph * graph;
const NodeInformationHelpDesk * nodeHelpDesk;
static SearchEngineHeapPtr forwardHeap;
static SearchEngineHeapPtr backwardHeap;
static SearchEngineHeapPtr forwardHeap2;

54
Extractor/ExtractionContainers.cpp
View File

@ -282,37 +282,43 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con
fout.close();
std::cout << "ok" << std::endl;
time = get_timestamp();
std::cout << "[extractor] writing street name index ... " << std::flush;
std::string nameOutFileName = (output_file_name + ".names");
std::ofstream nameOutFile(nameOutFileName.c_str(), std::ios::binary);
unsigned sizeOfNameIndex = nameVector.size();
nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned));
BOOST_FOREACH(const std::string & str, nameVector) {
unsigned lengthOfRawString = strlen(str.c_str());
nameOutFile.write((char *)&(lengthOfRawString), sizeof(unsigned));
nameOutFile.write(str.c_str(), lengthOfRawString);
std::cout << "[extractor] writing street name index ... " << std::flush;
std::string name_file_streamName = (output_file_name + ".names");
boost::filesystem::ofstream name_file_stream(
name_file_streamName,
std::ios::binary
);
const unsigned number_of_ways = name_list.size()+1;
name_file_stream.write((char *)&(number_of_ways), sizeof(unsigned));
unsigned name_lengths_prefix_sum = 0;
BOOST_FOREACH(const std::string & str, name_list) {
name_file_stream.write(
(char *)&(name_lengths_prefix_sum),
sizeof(unsigned)
);
name_lengths_prefix_sum += strlen(str.c_str());
}
name_file_stream.write(
(char *)&(name_lengths_prefix_sum),
sizeof(unsigned)
);
//duplicate on purpose!
name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
BOOST_FOREACH(const std::string & str, name_list) {
const unsigned lengthOfRawString = strlen(str.c_str());
name_file_stream.write(str.c_str(), lengthOfRawString);
}
nameOutFile.close();
name_file_stream.close();
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
// time = get_timestamp();
// cout << "[extractor] writing address list ... " << flush;
//
// adressFileName.append(".address");
// ofstream addressOutFile(adressFileName.c_str());
// for(STXXLAddressVector::iterator it = adressVector.begin(); it != adressVector.end(); it++) {
// addressOutFile << it->node.id << "|" << it->node.lat << "|" << it->node.lon << "|" << it->city << "|" << it->street << "|" << it->housenumber << "|" << it->state << "|" << it->country << "\n";
// }
// addressOutFile.close();
// cout << "ok, after " << get_timestamp() - time << "s" << endl;
SimpleLogger().Write() << "Processed " << usedNodeCounter << " nodes and " << usedEdgeCounter << " edges";
SimpleLogger().Write() <<
"Processed " << usedNodeCounter << " nodes and " << usedEdgeCounter << " edges";
} catch ( const std::exception& e ) {
std::cerr << "Caught Execption:" << e.what() << std::endl;
std::cerr << "Caught Execption:" << e.what() << std::endl;
}
}

39
Extractor/ExtractionContainers.h
View File

@ -27,41 +27,48 @@
#include "../Util/UUID.h"
#include <boost/foreach.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <stxxl.h>
class ExtractionContainers {
public:
typedef stxxl::vector<NodeID> STXXLNodeIDVector;
typedef stxxl::vector<_Node> STXXLNodeVector;
typedef stxxl::vector<InternalExtractorEdge> STXXLEdgeVector;
typedef stxxl::vector<std::string> STXXLStringVector;
typedef stxxl::vector<NodeID> STXXLNodeIDVector;
typedef stxxl::vector<_Node> STXXLNodeVector;
typedef stxxl::vector<InternalExtractorEdge> STXXLEdgeVector;
typedef stxxl::vector<std::string> STXXLStringVector;
typedef stxxl::vector<_RawRestrictionContainer> STXXLRestrictionsVector;
typedef stxxl::vector<_WayIDStartAndEndEdge> STXXLWayIDStartEndVector;
typedef stxxl::vector<_WayIDStartAndEndEdge> STXXLWayIDStartEndVector;
STXXLNodeIDVector usedNodeIDs;
STXXLNodeVector allNodes;
STXXLEdgeVector allEdges;
STXXLStringVector name_list;
STXXLRestrictionsVector restrictionsVector;
STXXLWayIDStartEndVector wayStartEndVector;
const UUID uuid;
ExtractionContainers() {
//Check if another instance of stxxl is already running or if there is a general problem
stxxl::vector<unsigned> testForRunningInstance;
nameVector.push_back("");
name_list.push_back("");
}
virtual ~ExtractionContainers() {
usedNodeIDs.clear();
allNodes.clear();
allEdges.clear();
nameVector.clear();
name_list.clear();
restrictionsVector.clear();
wayStartEndVector.clear();
}
void PrepareData( const std::string & output_file_name, const std::string restrictionsFileName, const unsigned amountOfRAM);
STXXLNodeIDVector usedNodeIDs;
STXXLNodeVector allNodes;
STXXLEdgeVector allEdges;
STXXLStringVector nameVector;
STXXLRestrictionsVector restrictionsVector;
STXXLWayIDStartEndVector wayStartEndVector;
const UUID uuid;
void PrepareData(
const std::string & output_file_name,
const std::string restrictionsFileName,
const unsigned amountOfRAM
);
};
#endif /* EXTRACTIONCONTAINERS_H_ */

4
Extractor/ExtractorCallbacks.cpp
View File

@ -65,8 +65,8 @@ void ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
//Get the unique identifier for the street name
const StringMap::const_iterator string_map_iterator = stringMap->find(parsed_way.name);
if(stringMap->end() == string_map_iterator) {
parsed_way.nameID = externalMemory->nameVector.size();
externalMemory->nameVector.push_back(parsed_way.name);
parsed_way.nameID = externalMemory->name_list.size();
externalMemory->name_list.push_back(parsed_way.name);
stringMap->insert(std::make_pair(parsed_way.name, parsed_way.nameID));
} else {
parsed_way.nameID = string_map_iterator->second;

40
Plugins/NearestPlugin.h
View File

@ -34,11 +34,9 @@ class NearestPlugin : public BasePlugin {
public:
NearestPlugin(QueryObjectsStorage * objects )
:
names(objects->names),
m_query_objects(objects),
descriptor_string("nearest")
{
nodeHelpDesk = objects->nodeHelpDesk;
descriptorTable.insert(std::make_pair("" , 0)); //default descriptor
descriptorTable.insert(std::make_pair("json", 1));
}
@ -53,12 +51,16 @@ public:
reply = http::Reply::stockReply(http::Reply::badRequest);
return;
}
NodeInformationHelpDesk * nodeHelpDesk = m_query_objects->nodeHelpDesk;
//query to helpdesk
PhantomNode result;
nodeHelpDesk->FindPhantomNodeForCoordinate(routeParameters.coordinates[0], result, routeParameters.zoomLevel);
nodeHelpDesk->FindPhantomNodeForCoordinate(
routeParameters.coordinates[0],
result,
routeParameters.zoomLevel
);
std::string tmp;
std::string temp_string;
//json
if("" != routeParameters.jsonpParameter) {
@ -70,23 +72,26 @@ public:
reply.content += ("{");
reply.content += ("\"version\":0.3,");
reply.content += ("\"status\":");
if(UINT_MAX != result.edgeBasedNode)
if(UINT_MAX != result.edgeBasedNode) {
reply.content += "0,";
else
} else {
reply.content += "207,";
}
reply.content += ("\"mapped_coordinate\":");
reply.content += "[";
if(UINT_MAX != result.edgeBasedNode) {
convertInternalLatLonToString(result.location.lat, tmp);
reply.content += tmp;
convertInternalLatLonToString(result.location.lon, tmp);
convertInternalLatLonToString(result.location.lat, temp_string);
reply.content += temp_string;
convertInternalLatLonToString(result.location.lon, temp_string);
reply.content += ",";
reply.content += tmp;
reply.content += temp_string;
}
reply.content += "],";
reply.content += "\"name\":\"";
if(UINT_MAX != result.edgeBasedNode)
reply.content += names[result.nodeBasedEdgeNameID];
if(UINT_MAX != result.edgeBasedNode) {
m_query_objects->GetName(result.nodeBasedEdgeNameID, temp_string);
reply.content += temp_string;
}
reply.content += "\"";
reply.content += ",\"transactionId\":\"OSRM Routing Engine JSON Nearest (v0.3)\"";
reply.content += ("}");
@ -104,14 +109,13 @@ public:
reply.headers[2].value = "attachment; filename=\"location.json\"";
}
reply.headers[0].name = "Content-Length";
intToString(reply.content.size(), tmp);
reply.headers[0].value = tmp;
intToString(reply.content.size(), temp_string);
reply.headers[0].value = temp_string;
}
private:
NodeInformationHelpDesk * nodeHelpDesk;
QueryObjectsStorage * m_query_objects;
HashTable<std::string, unsigned> descriptorTable;
std::vector<std::string> & names;
std::string descriptor_string;
};

5
Plugins/ViaRoutePlugin.h
View File

@ -43,7 +43,6 @@ or see http://www.gnu.org/licenses/agpl.txt.
class ViaRoutePlugin : public BasePlugin {
private:
NodeInformationHelpDesk * nodeHelpDesk;
std::vector<std::string> & names;
StaticGraph<QueryEdge::EdgeData> * graph;
HashTable<std::string, unsigned> descriptorTable;
SearchEngine * searchEnginePtr;
@ -51,13 +50,13 @@ public:
ViaRoutePlugin(QueryObjectsStorage * objects)
:
names(objects->names),
// objects(objects),
descriptor_string("viaroute")
{
nodeHelpDesk = objects->nodeHelpDesk;
graph = objects->graph;
searchEnginePtr = new SearchEngine(graph, nodeHelpDesk, names);
searchEnginePtr = new SearchEngine(objects);
descriptorTable.insert(std::make_pair("" , 0));
descriptorTable.insert(std::make_pair("json", 0));

123
Server/DataStructures/QueryObjectsStorage.cpp
View File

@ -22,54 +22,54 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "QueryObjectsStorage.h"
QueryObjectsStorage::QueryObjectsStorage(
const std::string & hsgrPath,
const std::string & ramIndexPath,
const std::string & fileIndexPath,
const std::string & nodesPath,
const std::string & edgesPath,
const std::string & namesPath,
const std::string & timestampPath
const std::string & hsgr_path,
const std::string & ram_index_path,
const std::string & file_index_path,
const std::string & nodes_path,
const std::string & edges_path,
const std::string & names_path,
const std::string & timestamp_path
) {
if( hsgrPath.empty() ) {
if( hsgr_path.empty() ) {
throw OSRMException("no hsgr file given in ini file");
}
if( ramIndexPath.empty() ) {
if( ram_index_path.empty() ) {
throw OSRMException("no ram index file given in ini file");
}
if( fileIndexPath.empty() ) {
if( file_index_path.empty() ) {
throw OSRMException("no mem index file given in ini file");
}
if( nodesPath.empty() ) {
if( nodes_path.empty() ) {
throw OSRMException("no nodes file given in ini file");
}
if( edgesPath.empty() ) {
if( edges_path.empty() ) {
throw OSRMException("no edges file given in ini file");
}
if( namesPath.empty() ) {
if( names_path.empty() ) {
throw OSRMException("no names file given in ini file");
}
SimpleLogger().Write() << "loading graph data";
//Deserialize road network graph
std::vector< QueryGraph::_StrNode> nodeList;
std::vector< QueryGraph::_StrEdge> edgeList;
const int n = readHSGRFromStream(
hsgrPath,
nodeList,
edgeList,
&checkSum
std::vector< QueryGraph::_StrNode> node_list;
std::vector< QueryGraph::_StrEdge> edge_list;
const int number_of_nodes = readHSGRFromStream(
hsgr_path,
node_list,
edge_list,
&check_sum
);
SimpleLogger().Write() << "Data checksum is " << checkSum;
graph = new QueryGraph(nodeList, edgeList);
assert(0 == nodeList.size());
assert(0 == edgeList.size());
SimpleLogger().Write() << "Data checksum is " << check_sum;
graph = new QueryGraph(node_list, edge_list);
assert(0 == node_list.size());
assert(0 == edge_list.size());
if(timestampPath.length()) {
if(timestamp_path.length()) {
SimpleLogger().Write() << "Loading Timestamp";
std::ifstream timestampInStream(timestampPath.c_str());
std::ifstream timestampInStream(timestamp_path.c_str());
if(!timestampInStream) {
SimpleLogger().Write(logWARNING) << timestampPath << " not found";
SimpleLogger().Write(logWARNING) << timestamp_path << " not found";
}
getline(timestampInStream, timestamp);
@ -85,17 +85,17 @@ QueryObjectsStorage::QueryObjectsStorage(
SimpleLogger().Write() << "Loading auxiliary information";
//Init nearest neighbor data structure
nodeHelpDesk = new NodeInformationHelpDesk(
ramIndexPath,
fileIndexPath,
nodesPath,
edgesPath,
n,
checkSum
ram_index_path,
file_index_path,
nodes_path,
edges_path,
number_of_nodes,
check_sum
);
//deserialize street name list
SimpleLogger().Write() << "Loading names index";
boost::filesystem::path names_file(namesPath);
boost::filesystem::path names_file(names_path);
if ( !boost::filesystem::exists( names_file ) ) {
throw OSRMException("names file does not exist");
@ -107,24 +107,55 @@ QueryObjectsStorage::QueryObjectsStorage(
boost::filesystem::ifstream name_stream(names_file, std::ios::binary);
unsigned size = 0;
name_stream.read((char *)&size, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != size, "name file empty");
BOOST_ASSERT_MSG(0 != size, "name file broken");
m_name_begin_indices.resize(size);
name_stream.read((char*)&m_name_begin_indices[0], size*sizeof(unsigned));
name_stream.read((char *)&size, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != size, "name file broken");
m_names_char_list.resize(size+1); //+1 is sentinel/dummy element
name_stream.read((char *)&m_names_char_list[0], size*sizeof(char));
BOOST_ASSERT_MSG(0 != m_names_char_list.size(), "could not load any names");
char buf[1024];
for( unsigned i = 0; i < size; ++i ) {
unsigned size_of_string = 0;
name_stream.read((char *)&size_of_string, sizeof(unsigned));
buf[size_of_string] = '\0'; // instead of memset
name_stream.read(buf, size_of_string);
names.push_back(buf);
}
std::vector<std::string>(names).swap(names);
BOOST_ASSERT_MSG(0 != names.size(), "could not load any names");
name_stream.close();
SimpleLogger().Write() << "All query data structures loaded";
}
void QueryObjectsStorage::GetName(
const unsigned name_id,
std::string & result
) const {
if(UINT_MAX == name_id) {
result = "";
return;
}
BOOST_ASSERT_MSG(
name_id < m_name_begin_indices.size(),
"name id too high"
);
unsigned begin_index = m_name_begin_indices[name_id];
unsigned end_index = m_name_begin_indices[name_id+1];
BOOST_ASSERT_MSG(
begin_index < m_names_char_list.size(),
"begin index of name too high"
);
BOOST_ASSERT_MSG(
end_index < m_names_char_list.size(),
"end index of name too high"
);
BOOST_ASSERT_MSG(begin_index <= end_index, "string ends before begin");
result.clear();
result.resize(end_index - begin_index);
std::copy(
m_names_char_list.begin() + begin_index,
m_names_char_list.begin() + end_index,
result.begin()
);
}
QueryObjectsStorage::~QueryObjectsStorage() {
// delete names;
delete graph;
delete nodeHelpDesk;
}

13
Server/DataStructures/QueryObjectsStorage.h
View File

@ -41,11 +41,14 @@ struct QueryObjectsStorage {
typedef StaticGraph<QueryEdge::EdgeData> QueryGraph;
typedef QueryGraph::InputEdge InputEdge;
NodeInformationHelpDesk * nodeHelpDesk;
std::vector<std::string> names;
QueryGraph * graph;
std::string timestamp;
unsigned checkSum;
NodeInformationHelpDesk * nodeHelpDesk;
std::vector<char> m_names_char_list;
std::vector<unsigned> m_name_begin_indices;
QueryGraph * graph;
std::string timestamp;
unsigned check_sum;
void GetName(const unsigned name_id, std::string & result) const;
QueryObjectsStorage(
const std::string & hsgrPath,