Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Merge branch 'master' of https://github.com/DennisOSRM/Project-OSRM

Browse Source
This commit is contained in:
DennisOSRM 2011-11-09 16:12:12 +01:00
commit f8135c56b7
14 changed files with 220 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Contractor/Contractor.h
View File

@ -35,7 +35,9 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include <set>
#include <stack>
#include <limits>
#ifdef _OPEMP
#include <omp.h>
#endif
class Contractor {

1
DataStructures/BaseParser.h
View File

@ -28,7 +28,6 @@ public:
virtual bool Init() = 0;
virtual bool RegisterCallbacks(bool (*nodeCallbackPointer)(NodeT), bool (*restrictionCallbackPointer)(RestrictionT), bool (*wayCallbackPointer)(WayT), bool (*addressCallbackPointer)(NodeT, HashTable<std::string, std::string>)) = 0;
virtual bool Parse() = 0;
private:
};
#endif /* BASEPARSER_H_ */

103
DataStructures/ConcurrentQueue.h Normal file
View File

@ -0,0 +1,103 @@
/*
open source routing machine
Copyright (C) Dennis Luxen, others 2010
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
or see http://www.gnu.org/licenses/agpl.txt.
*/
#ifndef CONCURRENTQUEUE_H_INCLUDED
#define CONCURRENTQUEUE_H_INCLUDED
#include "typedefs.h"
/*
Concurrent Queue written by Anthony Williams:
http://www.justsoftwaresolutions.co.uk/threading/implementing-a-thread-safe-queue-using-condition-variables.html
*/
template<typename Data>
class concurrent_queue
{
private:
std::queue<Data> internal_queue;
mutable boost::mutex queue_mutex;
mutable boost::mutex queue_full_mutex;
boost::condition_variable queue_cv;
boost::condition_variable queue_full_cv;
const size_t max_queue_size;
bool size_exceeded() const {
boost::mutex::scoped_lock lock(queue_mutex);
return internal_queue.size() >= max_queue_size;
}
public:
concurrent_queue(const size_t max_size)
: max_queue_size(max_size) {
}
void push(Data const& data)
{
if (size_exceeded()) {
boost::mutex::scoped_lock qf_lock(queue_full_mutex);
queue_full_cv.wait(qf_lock);
}
boost::mutex::scoped_lock lock(queue_mutex);
internal_queue.push(data);
lock.unlock();
queue_cv.notify_one();
}
bool empty() const
{
boost::mutex::scoped_lock lock(queue_mutex);
return internal_queue.empty();
}
bool try_pop(Data& popped_value)
{
boost::mutex::scoped_lock lock(queue_mutex);
if(internal_queue.empty())
{
return false;
}
popped_value=internal_queue.front();
internal_queue.pop();
queue_full_cv.notify_one();
return true;
}
void wait_and_pop(Data& popped_value)
{
boost::mutex::scoped_lock lock(queue_mutex);
while(internal_queue.empty())
{
queue_cv.wait(lock);
}
popped_value=internal_queue.front();
internal_queue.pop();
queue_full_cv.notify_one();
}
int size() const {
boost::mutex::scoped_lock lock(queue_mutex);
return static_cast<int>(internal_queue.size());
}
};
#endif //#ifndef CONCURRENTQUEUE_H_INCLUDED

6
DataStructures/ExtractorCallBacks.h
View File

@ -167,13 +167,13 @@ public:
w.nameID = strit->second;
}
GUARANTEE(w.id != UINT_MAX, "found way with unknown type");
GUARANTEE(-1 != w.speed, "found way with unknown speed");
if ( w.direction == _Way::opposite ){
std::reverse( w.path.begin(), w.path.end() );
}
GUARANTEE(w.id != UINT_MAX, "found way with unknown type");
GUARANTEE(-1 != w.speed, "found way with unknown speed");
for(vector< NodeID >::size_type n = 0; n < w.path.size()-1; ++n) {
externalMemory->allEdges.push_back(_Edge(w.path[n], w.path[n+1], w.type, w.direction, w.speed, w.nameID));
externalMemory->usedNodeIDs.push_back(w.path[n]);

3
DataStructures/ImportEdge.h
View File

@ -23,8 +23,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include <cassert>
class Edge
{
class Edge {
public:
bool operator< (const Edge& e) const {

3
DataStructures/InputReaderFactory.h
View File

@ -32,8 +32,7 @@ struct BZ2Context {
char unused[BZ_MAX_UNUSED];
};
int readFromBz2Stream( void* pointer, char* buffer, int len )
{
int readFromBz2Stream( void* pointer, char* buffer, int len ) {
void *unusedTmpVoid=NULL;
char *unusedTmp=NULL;
BZ2Context* context = (BZ2Context*) pointer;

2
DataStructures/NNGrid.h
View File

@ -28,7 +28,9 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include <limits>
#include <vector>
#include <stxxl.h>
#ifdef _OPENMP
#include <omp.h>
#endif
#ifdef _WIN32
#include <math.h>

143
DataStructures/PBFParser.h
View File

@ -30,7 +30,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include "../typedefs.h"
#include "HashTable.h"
#include "ExtractorStructs.h"
#include "ConcurrentQueue.h"
class PBFParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> {
@ -61,9 +61,9 @@ class PBFParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> {
};
public:
PBFParser(const char * fileName) {
PBFParser(const char * fileName)
: threadDataQueue( new concurrent_queue<_ThreadData*>(25) ) { /* Max 25 items in queue */
GOOGLE_PROTOBUF_VERIFY_VERSION;
omp_set_num_threads(1);
input.open(fileName, std::ios::in | std::ios::binary);
if (!input) {
@ -86,10 +86,12 @@ public:
if(input.is_open())
input.close();
unsigned maxThreads = omp_get_max_threads();
for ( unsigned threadNum = 0; threadNum < maxThreads; ++threadNum ) {
delete threadDataVector[threadNum];
// Clean up any leftover ThreadData objects in the queue
_ThreadData* td;
while (threadDataQueue->try_pop(td)) {
delete td;
}
delete threadDataQueue;
google::protobuf::ShutdownProtobufLibrary();
@ -100,12 +102,6 @@ public:
}
bool Init() {
/** Init Vector with ThreadData Objects */
unsigned maxThreads = omp_get_max_threads();
for ( unsigned threadNum = 0; threadNum < maxThreads; ++threadNum ) {
threadDataVector.push_back( new _ThreadData( ) );
}
_ThreadData initData;
/** read Header */
if(!readPBFBlobHeader(input, &initData)) {
@ -137,36 +133,58 @@ public:
return true;
}
bool Parse() {
#pragma omp parallel
{
_ThreadData * threadData = threadDataVector[omp_get_thread_num()];
//parse through all Blocks
bool keepRunning = true;
// while(readNextBlock(input)) {
do{
#pragma omp critical
{
keepRunning = readNextBlock(input, threadData);
}
if(keepRunning) {
loadBlock(threadData);
for(int i = 0; i < threadData->PBFprimitiveBlock.primitivegroup_size(); i++) {
threadData->currentGroupID = i;
loadGroup(threadData);
void ReadData() {
bool keepRunning = true;
do {
_ThreadData *threadData = new _ThreadData();
keepRunning = readNextBlock(input, threadData);
if(threadData->entityTypeIndicator == TypeNode)
parseNode(threadData);
if(threadData->entityTypeIndicator == TypeWay)
parseWay(threadData);
if(threadData->entityTypeIndicator == TypeRelation)
parseRelation(threadData);
if(threadData->entityTypeIndicator == TypeDenseNode)
parseDenseNode(threadData);
}
}
}while(keepRunning);
if (keepRunning)
threadDataQueue->push(threadData);
else
threadDataQueue->push(NULL); // No more data to read, parse stops when NULL encountered
} while(keepRunning);
}
void ParseData() {
while (1) {
_ThreadData *threadData;
threadDataQueue->wait_and_pop(threadData);
if (threadData == NULL) {
cout << "Parse Data Thread Finished" << endl;
threadDataQueue->push(NULL); // Signal end of data for other threads
break;
}
loadBlock(threadData);
for(int i = 0; i < threadData->PBFprimitiveBlock.primitivegroup_size(); i++) {
threadData->currentGroupID = i;
loadGroup(threadData);
if(threadData->entityTypeIndicator == TypeNode)
parseNode(threadData);
if(threadData->entityTypeIndicator == TypeWay)
parseWay(threadData);
if(threadData->entityTypeIndicator == TypeRelation)
parseRelation(threadData);
if(threadData->entityTypeIndicator == TypeDenseNode)
parseDenseNode(threadData);
}
delete threadData;
threadData = NULL;
}
}
bool Parse() {
// Start the read and parse threads
boost::thread readThread(boost::bind(&PBFParser::ReadData, this));
boost::thread parseThread(boost::bind(&PBFParser::ParseData, this));
// Wait for the threads to finish
readThread.join();
parseThread.join();
return true;
}
@ -200,27 +218,18 @@ private:
keyVals.Add(key, value);
denseTagIndex += 2;
}
#pragma omp critical
{
if(!(*addressCallback)(n, keyVals))
std::cerr << "[PBFParser] adress not parsed" << std::endl;
}
#pragma omp critical
{
if(!(*nodeCallback)(n))
std::cerr << "[PBFParser] dense node not parsed" << std::endl;
}
if(!(*addressCallback)(n, keyVals))
std::cerr << "[PBFParser] adress not parsed" << std::endl;
if(!(*nodeCallback)(n))
std::cerr << "[PBFParser] dense node not parsed" << std::endl;
}
}
void parseNode(_ThreadData * threadData) {
_Node n;
#pragma omp critical
{
if(!(*nodeCallback)(n))
std::cerr << "[PBFParser] simple node not parsed" << std::endl;
}
if(!(*nodeCallback)(n))
std::cerr << "[PBFParser] simple node not parsed" << std::endl;
}
void parseRelation(_ThreadData * threadData) {
@ -291,11 +300,8 @@ private:
// cout << "node " << currentRestriction.viaNode;
// cout << " to " << currentRestriction.to << endl;
// }
#pragma omp critical
{
if(!(*restrictionCallback)(currentRestrictionContainer))
std::cerr << "[PBFParser] relation not parsed" << std::endl;
}
if(!(*restrictionCallback)(currentRestrictionContainer))
std::cerr << "[PBFParser] relation not parsed" << std::endl;
}
}
}
@ -317,18 +323,15 @@ private:
const std::string val = threadData->PBFprimitiveBlock.stringtable().s(inputWay.vals(i));
w.keyVals.Add(key, val);
}
#pragma omp critical
{
if(!(*wayCallback)(w)) {
std::cerr << "[PBFParser] way not parsed" << std::endl;
}
if(!(*wayCallback)(w)) {
std::cerr << "[PBFParser] way not parsed" << std::endl;
}
}
}
}
void loadGroup(_ThreadData * threadData) {
#pragma omp atomic
groupCount++;
const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID );
@ -350,7 +353,6 @@ private:
}
void loadBlock(_ThreadData * threadData) {
#pragma omp critical
blockCount++;
threadData->currentGroupID = 0;
threadData->currentEntityID = 0;
@ -522,9 +524,8 @@ private:
/* the input stream to parse */
std::fstream input;
/* ThreadData Array */
std::vector < _ThreadData* > threadDataVector;
/* ThreadData Queue */
concurrent_queue < _ThreadData* >* threadDataQueue;
};
#endif /* PBFPARSER_H_ */

2
README.TXT
View File

@ -5,7 +5,7 @@ Compiling the source code is easy. If you are running a decent linux
installing dependencies and running make should suffice. Make sure the following
dependencies are installed:
- Boost 1.37+
- Boost 1.41+
- sparsehash 1.4+
- g++ 4.2+ (with warnings and unsupported), 4.4+
- libxml2 2.7+

5
createHierarchy.cpp
View File

@ -38,6 +38,11 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifdef _OPENMP
#include <omp.h>
#else
int omp_get_num_procs() { return 1; }
int omp_get_max_threads() { return 1; }
int omp_get_thread_num() { return 0; }
int omp_set_num_threads(int i) {}
#endif
#include "typedefs.h"

40
extractor.cpp
View File

@ -229,11 +229,11 @@ int main (int argc, char *argv[]) {
while(wayStartAndEndEdgeIT != externalMemory.wayStartEndVector.end() && restrictionsIT != externalMemory.restrictionsVector.end()) {
if(wayStartAndEndEdgeIT->wayID < restrictionsIT->fromWay){
wayStartAndEndEdgeIT++;
++wayStartAndEndEdgeIT;
continue;
}
if(wayStartAndEndEdgeIT->wayID > restrictionsIT->fromWay) {
restrictionsIT++;
++restrictionsIT;
continue;
}
assert(wayStartAndEndEdgeIT->wayID == restrictionsIT->fromWay);
@ -248,7 +248,7 @@ int main (int argc, char *argv[]) {
} else if(wayStartAndEndEdgeIT->lastTarget == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastStart;
}
restrictionsIT++;
++restrictionsIT;
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
@ -266,11 +266,11 @@ int main (int argc, char *argv[]) {
while(wayStartAndEndEdgeIT != externalMemory.wayStartEndVector.end() &&
restrictionsIT != externalMemory.restrictionsVector.end()) {
if(wayStartAndEndEdgeIT->wayID < restrictionsIT->toWay){
wayStartAndEndEdgeIT++;
++wayStartAndEndEdgeIT;
continue;
}
if(wayStartAndEndEdgeIT->wayID > restrictionsIT->toWay) {
restrictionsIT++;
++restrictionsIT;
continue;
}
NodeID viaNode = restrictionsIT->restriction.viaNode;
@ -285,9 +285,9 @@ int main (int argc, char *argv[]) {
}
if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
usableRestrictionsCounter++;
++usableRestrictionsCounter;
}
restrictionsIT++;
++restrictionsIT;
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
@ -295,7 +295,7 @@ int main (int argc, char *argv[]) {
ofstream restrictionsOutstream;
restrictionsOutstream.open(restrictionsFileName.c_str(), ios::binary);
restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned));
for(restrictionsIT = externalMemory.restrictionsVector.begin(); restrictionsIT != externalMemory.restrictionsVector.end(); restrictionsIT++) {
for(restrictionsIT = externalMemory.restrictionsVector.begin(); restrictionsIT != externalMemory.restrictionsVector.end(); ++restrictionsIT) {
if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
restrictionsOutstream.write((char *)&(restrictionsIT->restriction), sizeof(_Restriction));
}
@ -311,20 +311,20 @@ int main (int argc, char *argv[]) {
STXXLNodeIDVector::iterator usedNodeIDsIT = externalMemory.usedNodeIDs.begin();
while(usedNodeIDsIT != externalMemory.usedNodeIDs.end() && nodesIT != externalMemory.allNodes.end()) {
if(*usedNodeIDsIT < nodesIT->id){
usedNodeIDsIT++;
++usedNodeIDsIT;
continue;
}
if(*usedNodeIDsIT > nodesIT->id) {
nodesIT++;
++nodesIT;
continue;
}
if(*usedNodeIDsIT == nodesIT->id) {
fout.write((char*)&(nodesIT->id), sizeof(unsigned));
fout.write((char*)&(nodesIT->lon), sizeof(int));
fout.write((char*)&(nodesIT->lat), sizeof(int));
usedNodeCounter++;
usedNodeIDsIT++;
nodesIT++;
++usedNodeCounter;
++usedNodeIDsIT;
++nodesIT;
}
}
@ -353,7 +353,7 @@ int main (int argc, char *argv[]) {
STXXLEdgeVector::iterator edgeIT = externalMemory.allEdges.begin();
while(edgeIT != externalMemory.allEdges.end() && nodesIT != externalMemory.allNodes.end()) {
if(edgeIT->start < nodesIT->id){
edgeIT++;
++edgeIT;
continue;
}
if(edgeIT->start > nodesIT->id) {
@ -363,7 +363,7 @@ int main (int argc, char *argv[]) {
if(edgeIT->start == nodesIT->id) {
edgeIT->startCoord.lat = nodesIT->lat;
edgeIT->startCoord.lon = nodesIT->lon;
edgeIT++;
++edgeIT;
}
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
@ -381,11 +381,11 @@ int main (int argc, char *argv[]) {
edgeIT = externalMemory.allEdges.begin();
while(edgeIT != externalMemory.allEdges.end() && nodesIT != externalMemory.allNodes.end()) {
if(edgeIT->target < nodesIT->id){
edgeIT++;
++edgeIT;
continue;
}
if(edgeIT->target > nodesIT->id) {
nodesIT++;
++nodesIT;
continue;
}
if(edgeIT->target == nodesIT->id) {
@ -430,8 +430,8 @@ int main (int argc, char *argv[]) {
fout.write((char*)&edgeType, sizeof(short));
fout.write((char*)&edgeIT->nameID, sizeof(unsigned));
}
usedEdgeCounter++;
edgeIT++;
++usedEdgeCounter;
++edgeIT;
}
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
@ -494,7 +494,7 @@ bool adressFunction(_Node n, HashTable<string, string> keyVals){
bool restrictionFunction(_RawRestrictionContainer r) {
extractCallBacks->restrictionFunction(r);
globalRestrictionCounter++;
++globalRestrictionCounter;
return true;
}
bool wayFunction(_Way w) {

4
routed.cpp
View File

@ -23,6 +23,10 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifdef _OPENMP
#include <omp.h>
#else
int omp_get_num_procs() { return 1; }
int omp_get_max_threads() { return 1; }
int omp_get_thread_num() { return 0; }
#endif
#include <boost/thread.hpp>

2
vcprojects/createHierarchy.vcproj
View File

@ -99,7 +99,7 @@
<Configuration
Name="Release|Win32"
OutputDirectory="$(SolutionDir)..\bin"
IntermediateDirectory="(SolutionDir)..\bin"
IntermediateDirectory="$(SolutionDir)..\bin"
ConfigurationType="1"
InheritedPropertySheets=".\osrm.vsprops"
CharacterSet="1"

6
vcprojects/extractor.vcproj
View File

@ -177,7 +177,7 @@
/>
<Tool
Name="VCPostBuildEventTool"
CommandLine="xcopy &quot;$(SolutionDir)..\.stxxl&quot; &quot;$(OutDir)\&quot; /f /y&#x0D;&#x0A;xcopy &quot;$(SolutionDir)..\extractor.ini&quot; &quot;$(OutDir)\&quot; /f /y&#x0D;&#x0A;xcopy &quot;$(Bzip2Path)\bin\libbz2.dll&quot; &quot;$(OutDir)\&quot; /f /y&#x0D;&#x0A;"
CommandLine="xcopy &quot;$(SolutionDir)..\.stxxl&quot; &quot;$(OutDir)\&quot; /f /y&#x0D;&#x0A;xcopy &quot;$(SolutionDir)..\extractor.ini&quot; &quot;$(OutDir)\&quot; /f /y&#x0D;&#x0A;xcopy &quot;$(SolutionDir)..\speedprofile.ini&quot; &quot;$(OutDir)\&quot; /f /y&#x0D;&#x0A;xcopy &quot;$(Bzip2Path)\bin\libbz2.dll&quot; &quot;$(OutDir)\&quot; /f /y&#x0D;&#x0A;"
/>
</Configuration>
</Configurations>
@ -250,6 +250,10 @@
RelativePath="..\DataStructures\BinaryHeap.h"
>
</File>
<File
RelativePath="..\DataStructures\ConcurrentQueue.h"
>
</File>
<File
RelativePath="..\DataStructures\DynamicGraph.h"
>