Move files in src/ include/

src/benchmarks/static_rtree.cpp

@@ -0,0 +1,152 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "../data_structures/query_node.hpp"
+#include "../data_structures/static_rtree.hpp"
+#include "../data_structures/edge_based_node.hpp"
+#include "../algorithms/geospatial_query.hpp"
+#include "../util/timing_util.hpp"
+
+#include <osrm/coordinate.hpp>
+
+#include <random>
+#include <iostream>
+
+// Choosen by a fair W20 dice roll (this value is completely arbitrary)
+constexpr unsigned RANDOM_SEED = 13;
+constexpr int32_t WORLD_MIN_LAT = -90 * COORDINATE_PRECISION;
+constexpr int32_t WORLD_MAX_LAT = 90 * COORDINATE_PRECISION;
+constexpr int32_t WORLD_MIN_LON = -180 * COORDINATE_PRECISION;
+constexpr int32_t WORLD_MAX_LON = 180 * COORDINATE_PRECISION;
+
+using RTreeLeaf = EdgeBasedNode;
+using FixedPointCoordinateListPtr = std::shared_ptr<std::vector<FixedPointCoordinate>>;
+using BenchStaticRTree = StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, false>::vector, false>;
+using BenchQuery = GeospatialQuery<BenchStaticRTree>;
+
+FixedPointCoordinateListPtr LoadCoordinates(const boost::filesystem::path &nodes_file)
+{
+    boost::filesystem::ifstream nodes_input_stream(nodes_file, std::ios::binary);
+
+    QueryNode current_node;
+    unsigned coordinate_count = 0;
+    nodes_input_stream.read((char *)&coordinate_count, sizeof(unsigned));
+    auto coords = std::make_shared<std::vector<FixedPointCoordinate>>(coordinate_count);
+    for (unsigned i = 0; i < coordinate_count; ++i)
+    {
+        nodes_input_stream.read((char *)&current_node, sizeof(QueryNode));
+        coords->at(i) = FixedPointCoordinate(current_node.lat, current_node.lon);
+        BOOST_ASSERT((std::abs(coords->at(i).lat) >> 30) == 0);
+        BOOST_ASSERT((std::abs(coords->at(i).lon) >> 30) == 0);
+    }
+    nodes_input_stream.close();
+    return coords;
+}
+
+template <typename QueryT>
+void BenchmarkQuery(const std::vector<FixedPointCoordinate> &queries,
+                    const std::string& name,
+                    QueryT query)
+{
+    std::cout << "Running " << name << " with " << queries.size() << " coordinates: " << std::flush;
+
+    TIMER_START(query);
+    for (const auto &q : queries)
+    {
+        auto result = query(q);
+    }
+    TIMER_STOP(query);
+
+    std::cout << "Took " << TIMER_SEC(query) << " seconds "
+              << "(" << TIMER_MSEC(query) << "ms"
+              << ")  ->  " << TIMER_MSEC(query) / queries.size() << " ms/query "
+              << "(" << TIMER_MSEC(query) << "ms"
+              << ")" << std::endl;
+}
+
+void Benchmark(BenchStaticRTree &rtree, BenchQuery &geo_query, unsigned num_queries)
+{
+    std::mt19937 mt_rand(RANDOM_SEED);
+    std::uniform_int_distribution<> lat_udist(WORLD_MIN_LAT, WORLD_MAX_LAT);
+    std::uniform_int_distribution<> lon_udist(WORLD_MIN_LON, WORLD_MAX_LON);
+    std::vector<FixedPointCoordinate> queries;
+    for (unsigned i = 0; i < num_queries; i++)
+    {
+        queries.emplace_back(lat_udist(mt_rand), lon_udist(mt_rand));
+    }
+
+    BenchmarkQuery(queries, "raw RTree queries (1 result)", [&rtree](const FixedPointCoordinate &q)
+                   {
+                       return rtree.Nearest(q, 1);
+                   });
+    BenchmarkQuery(queries, "raw RTree queries (10 results)",
+                   [&rtree](const FixedPointCoordinate &q)
+                   {
+                       return rtree.Nearest(q, 10);
+                   });
+
+    BenchmarkQuery(queries, "big component alternative queries",
+                   [&geo_query](const FixedPointCoordinate &q)
+                   {
+                       return geo_query.NearestPhantomNodeWithAlternativeFromBigComponent(q);
+                   });
+    BenchmarkQuery(queries, "max distance 1000", [&geo_query](const FixedPointCoordinate &q)
+                   {
+                       return geo_query.NearestPhantomNodesInRange(q, 1000);
+                   });
+    BenchmarkQuery(queries, "PhantomNode query (1 result)", [&geo_query](const FixedPointCoordinate &q)
+                   {
+                       return geo_query.NearestPhantomNodes(q, 1);
+                   });
+    BenchmarkQuery(queries, "PhantomNode query (10 result)", [&geo_query](const FixedPointCoordinate &q)
+                   {
+                       return geo_query.NearestPhantomNodes(q, 10);
+                   });
+}
+
+int main(int argc, char **argv)
+{
+    if (argc < 4)
+    {
+        std::cout << "./rtree-bench file.ramIndex file.fileIndx file.nodes"
+                  << "\n";
+        return 1;
+    }
+
+    const char *ramPath = argv[1];
+    const char *filePath = argv[2];
+    const char *nodesPath = argv[3];
+
+    auto coords = LoadCoordinates(nodesPath);
+
+    BenchStaticRTree rtree(ramPath, filePath, coords);
+    BenchQuery query(rtree, coords);
+
+    Benchmark(rtree, query, 10000);
+
+    return 0;
+}

src/contractor/contractor_options.cpp

@@ -0,0 +1,142 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "contractor_options.hpp"
+
+#include "util/version.hpp"
+#include "../util/simple_logger.hpp"
+
+#include <boost/filesystem.hpp>
+#include <boost/program_options.hpp>
+
+#include <tbb/task_scheduler_init.h>
+
+return_code
+ContractorOptions::ParseArguments(int argc, char *argv[], ContractorConfig &contractor_config)
+{
+    // declare a group of options that will be allowed only on command line
+    boost::program_options::options_description generic_options("Options");
+    generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message")(
+        "config,c", boost::program_options::value<boost::filesystem::path>(&contractor_config.config_file_path)
+                        ->default_value("contractor.ini"),
+        "Path to a configuration file.");
+
+    // declare a group of options that will be allowed both on command line and in config file
+    boost::program_options::options_description config_options("Configuration");
+    config_options.add_options()(
+        "profile,p",
+        boost::program_options::value<boost::filesystem::path>(&contractor_config.profile_path)
+            ->default_value("profile.lua"),
+        "Path to LUA routing profile")(
+        "threads,t",
+        boost::program_options::value<unsigned int>(&contractor_config.requested_num_threads)
+            ->default_value(tbb::task_scheduler_init::default_num_threads()),
+        "Number of threads to use")(
+		"core,k", boost::program_options::value<double>(&contractor_config.core_factor)
+						 ->default_value(1.0),"Percentage of the graph (in vertices) to contract [0..1]")(
+		"segment-speed-file", boost::program_options::value<std::string>(&contractor_config.segment_speed_lookup_path),
+						 "Lookup file containing nodeA,nodeB,speed data to adjust edge weights")(
+        "level-cache,o",
+        boost::program_options::value<bool>(&contractor_config.use_cached_priority)->default_value(false),
+        "Use .level file to retain the contaction level for each node from the last run.");
+
+#ifdef DEBUG_GEOMETRY
+    config_options.add_options()(
+		"debug-geometry", boost::program_options::value<std::string>(&contractor_config.debug_geometry_path)
+						 ,"Write out edge-weight debugging geometry data in GeoJSON format to this file");
+#endif
+
+    // hidden options, will be allowed both on command line and in config file, but will not be
+    // shown to the user
+    boost::program_options::options_description hidden_options("Hidden options");
+    hidden_options.add_options()(
+        "input,i", boost::program_options::value<boost::filesystem::path>(&contractor_config.osrm_input_path),
+        "Input file in .osm, .osm.bz2 or .osm.pbf format");
+
+    // positional option
+    boost::program_options::positional_options_description positional_options;
+    positional_options.add("input", 1);
+
+    // combine above options for parsing
+    boost::program_options::options_description cmdline_options;
+    cmdline_options.add(generic_options).add(config_options).add(hidden_options);
+
+    boost::program_options::options_description config_file_options;
+    config_file_options.add(config_options).add(hidden_options);
+
+    boost::program_options::options_description visible_options(
+        "Usage: " + boost::filesystem::basename(argv[0]) + " <input.osrm> [options]");
+    visible_options.add(generic_options).add(config_options);
+
+    // parse command line options
+    boost::program_options::variables_map option_variables;
+    boost::program_options::store(boost::program_options::command_line_parser(argc, argv)
+                                      .options(cmdline_options)
+                                      .positional(positional_options)
+                                      .run(),
+                                  option_variables);
+
+    const auto &temp_config_path = option_variables["config"].as<boost::filesystem::path>();
+    if (boost::filesystem::is_regular_file(temp_config_path))
+    {
+        boost::program_options::store(boost::program_options::parse_config_file<char>(
+                                          temp_config_path.string().c_str(), cmdline_options, true),
+                                      option_variables);
+    }
+
+    if (option_variables.count("version"))
+    {
+        SimpleLogger().Write() << OSRM_VERSION;
+        return return_code::exit;
+    }
+
+    if (option_variables.count("help"))
+    {
+        SimpleLogger().Write() << "\n" << visible_options;
+        return return_code::exit;
+    }
+
+    boost::program_options::notify(option_variables);
+
+    if (!option_variables.count("input"))
+    {
+        SimpleLogger().Write() << "\n" << visible_options;
+        return return_code::fail;
+    }
+
+    return return_code::ok;
+}
+
+void ContractorOptions::GenerateOutputFilesNames(ContractorConfig &contractor_config)
+{
+    contractor_config.level_output_path = contractor_config.osrm_input_path.string() + ".level";
+    contractor_config.core_output_path = contractor_config.osrm_input_path.string() + ".core";
+    contractor_config.graph_output_path = contractor_config.osrm_input_path.string() + ".hsgr";
+    contractor_config.edge_based_graph_path = contractor_config.osrm_input_path.string() + ".ebg";
+    contractor_config.edge_segment_lookup_path = contractor_config.osrm_input_path.string() + ".edge_segment_lookup";
+    contractor_config.edge_penalty_path = contractor_config.osrm_input_path.string() + ".edge_penalties";
+}

src/contractor/processing_chain.cpp

@@ -0,0 +1,441 @@
+/*
+
+Copyright (c) 2015, Project OSRM, Dennis Luxen, others
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "processing_chain.hpp"
+#include "contractor.hpp"
+
+#include "contractor.hpp"
+
+#include "../data_structures/deallocating_vector.hpp"
+
+#include "../algorithms/crc32_processor.hpp"
+#include "../util/graph_loader.hpp"
+#include "../util/integer_range.hpp"
+#include "../util/lua_util.hpp"
+#include "../util/osrm_exception.hpp"
+#include "../util/simple_logger.hpp"
+#include "../util/string_util.hpp"
+#include "../util/timing_util.hpp"
+#include "../typedefs.h"
+
+#include <fast-cpp-csv-parser/csv.h>
+
+#include <boost/filesystem/fstream.hpp>
+#include <boost/program_options.hpp>
+
+#include <tbb/parallel_sort.h>
+
+#include <chrono>
+#include <memory>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "../util/debug_geometry.hpp"
+
+Prepare::~Prepare() {}
+
+int Prepare::Run()
+{
+#ifdef WIN32
+#pragma message("Memory consumption on Windows can be higher due to different bit packing")
+#else
+    static_assert(sizeof(NodeBasedEdge) == 20,
+                  "changing NodeBasedEdge type has influence on memory consumption!");
+    static_assert(sizeof(EdgeBasedEdge) == 16,
+                  "changing EdgeBasedEdge type has influence on memory consumption!");
+#endif
+
+    if (config.core_factor > 1.0 || config.core_factor < 0) 
+    {
+       throw osrm::exception("Core factor must be between 0.0 to 1.0 (inclusive)");
+    }
+
+    TIMER_START(preparing);
+
+    // Create a new lua state
+
+    SimpleLogger().Write() << "Loading edge-expanded graph representation";
+
+    DeallocatingVector<EdgeBasedEdge> edge_based_edge_list;
+
+    size_t max_edge_id = LoadEdgeExpandedGraph(
+        config.edge_based_graph_path, edge_based_edge_list, config.edge_segment_lookup_path,
+        config.edge_penalty_path, config.segment_speed_lookup_path);
+
+    // Contracting the edge-expanded graph
+
+    TIMER_START(contraction);
+    std::vector<bool> is_core_node;
+    std::vector<float> node_levels;
+    if (config.use_cached_priority)
+    {
+        ReadNodeLevels(node_levels);
+    }
+    DeallocatingVector<QueryEdge> contracted_edge_list;
+    ContractGraph(max_edge_id, edge_based_edge_list, contracted_edge_list, is_core_node,
+                  node_levels);
+    TIMER_STOP(contraction);
+
+    SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec";
+
+    std::size_t number_of_used_edges = WriteContractedGraph(max_edge_id, contracted_edge_list);
+    WriteCoreNodeMarker(std::move(is_core_node));
+    if (!config.use_cached_priority)
+    {
+        WriteNodeLevels(std::move(node_levels));
+    }
+
+    TIMER_STOP(preparing);
+
+    SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds";
+    SimpleLogger().Write() << "Contraction: " << ((max_edge_id + 1) / TIMER_SEC(contraction))
+                           << " nodes/sec and " << number_of_used_edges / TIMER_SEC(contraction)
+                           << " edges/sec";
+
+    SimpleLogger().Write() << "finished preprocessing";
+
+    return 0;
+}
+
+namespace std
+{
+
+template <> struct hash<std::pair<OSMNodeID, OSMNodeID>>
+{
+    std::size_t operator()(const std::pair<OSMNodeID, OSMNodeID> &k) const
+    {
+        return OSMNodeID_to_uint64_t(k.first) ^ (OSMNodeID_to_uint64_t(k.second) << 12);
+    }
+};
+}
+
+std::size_t Prepare::LoadEdgeExpandedGraph(std::string const &edge_based_graph_filename,
+                                           DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list,
+                                           const std::string &edge_segment_lookup_filename,
+                                           const std::string &edge_penalty_filename,
+                                           const std::string &segment_speed_filename)
+{
+    SimpleLogger().Write() << "Opening " << edge_based_graph_filename;
+    boost::filesystem::ifstream input_stream(edge_based_graph_filename, std::ios::binary);
+
+    const bool update_edge_weights = segment_speed_filename != "";
+
+    boost::filesystem::ifstream edge_segment_input_stream;
+    boost::filesystem::ifstream edge_fixed_penalties_input_stream;
+
+    if (update_edge_weights)
+    {
+        edge_segment_input_stream.open(edge_segment_lookup_filename, std::ios::binary);
+        edge_fixed_penalties_input_stream.open(edge_penalty_filename, std::ios::binary);
+        if (!edge_segment_input_stream || !edge_fixed_penalties_input_stream)
+        {
+            throw osrm::exception("Could not load .edge_segment_lookup or .edge_penalties, did you "
+                                  "run osrm-extract with '--generate-edge-lookup'?");
+        }
+    }
+
+    const FingerPrint fingerprint_valid = FingerPrint::GetValid();
+    FingerPrint fingerprint_loaded;
+    input_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint));
+    fingerprint_loaded.TestPrepare(fingerprint_valid);
+
+    size_t number_of_edges = 0;
+    size_t max_edge_id = SPECIAL_EDGEID;
+    input_stream.read((char *)&number_of_edges, sizeof(size_t));
+    input_stream.read((char *)&max_edge_id, sizeof(size_t));
+
+    edge_based_edge_list.resize(number_of_edges);
+    SimpleLogger().Write() << "Reading " << number_of_edges << " edges from the edge based graph";
+
+    std::unordered_map<std::pair<OSMNodeID, OSMNodeID>, unsigned> segment_speed_lookup;
+
+    if (update_edge_weights)
+    {
+        SimpleLogger().Write() << "Segment speed data supplied, will update edge weights from "
+                               << segment_speed_filename;
+        io::CSVReader<3> csv_in(segment_speed_filename);
+        csv_in.set_header("from_node", "to_node", "speed");
+        uint64_t from_node_id;
+        uint64_t to_node_id;
+        unsigned speed;
+        while (csv_in.read_row(from_node_id, to_node_id, speed))
+        {
+            segment_speed_lookup[std::make_pair(OSMNodeID(from_node_id), OSMNodeID(to_node_id))] = speed;
+        }
+    }
+
+    DEBUG_GEOMETRY_START(config);
+
+    // TODO: can we read this in bulk?  DeallocatingVector isn't necessarily
+    // all stored contiguously
+    for (; number_of_edges > 0; --number_of_edges)
+    {
+        EdgeBasedEdge inbuffer;
+        input_stream.read((char *) &inbuffer, sizeof(EdgeBasedEdge));
+
+        if (update_edge_weights)
+        {
+            // Processing-time edge updates
+            unsigned fixed_penalty;
+            edge_fixed_penalties_input_stream.read(reinterpret_cast<char *>(&fixed_penalty),
+                                                   sizeof(fixed_penalty));
+
+            int new_weight = 0;
+
+            unsigned num_osm_nodes = 0;
+            edge_segment_input_stream.read(reinterpret_cast<char *>(&num_osm_nodes),
+                                           sizeof(num_osm_nodes));
+            OSMNodeID previous_osm_node_id;
+            edge_segment_input_stream.read(reinterpret_cast<char *>(&previous_osm_node_id),
+                                           sizeof(previous_osm_node_id));
+            OSMNodeID this_osm_node_id;
+            double segment_length;
+            int segment_weight;
+            --num_osm_nodes;
+            for (; num_osm_nodes != 0; --num_osm_nodes)
+            {
+                edge_segment_input_stream.read(reinterpret_cast<char *>(&this_osm_node_id),
+                                               sizeof(this_osm_node_id));
+                edge_segment_input_stream.read(reinterpret_cast<char *>(&segment_length),
+                                               sizeof(segment_length));
+                edge_segment_input_stream.read(reinterpret_cast<char *>(&segment_weight),
+                                               sizeof(segment_weight));
+
+                auto speed_iter = segment_speed_lookup.find(
+                    std::make_pair(previous_osm_node_id, this_osm_node_id));
+                if (speed_iter != segment_speed_lookup.end())
+                {
+                    // This sets the segment weight using the same formula as the
+                    // EdgeBasedGraphFactory for consistency.  The *why* of this formula
+                    // is lost in the annals of time.
+                    int new_segment_weight =
+                        std::max(1, static_cast<int>(std::floor(
+                                        (segment_length * 10.) / (speed_iter->second / 3.6) + .5)));
+                    new_weight += new_segment_weight;
+
+                    DEBUG_GEOMETRY_EDGE( 
+                            new_segment_weight, 
+                            segment_length,
+                            previous_osm_node_id,
+                            this_osm_node_id);
+                }
+                else
+                {
+                    // If no lookup found, use the original weight value for this segment
+                    new_weight += segment_weight;
+
+                    DEBUG_GEOMETRY_EDGE(
+                            segment_weight,
+                            segment_length,
+                            previous_osm_node_id,
+                            this_osm_node_id);
+                }
+
+                previous_osm_node_id = this_osm_node_id;
+            }
+
+            inbuffer.weight = fixed_penalty + new_weight;
+        }
+
+        edge_based_edge_list.emplace_back(std::move(inbuffer));
+    }
+
+    DEBUG_GEOMETRY_STOP();
+    SimpleLogger().Write() << "Done reading edges";
+    return max_edge_id;
+}
+
+void Prepare::ReadNodeLevels(std::vector<float> &node_levels) const
+{
+    boost::filesystem::ifstream order_input_stream(config.level_output_path, std::ios::binary);
+
+    unsigned level_size;
+    order_input_stream.read((char *)&level_size, sizeof(unsigned));
+    node_levels.resize(level_size);
+    order_input_stream.read((char *)node_levels.data(), sizeof(float) * node_levels.size());
+}
+
+void Prepare::WriteNodeLevels(std::vector<float> &&in_node_levels) const
+{
+    std::vector<float> node_levels(std::move(in_node_levels));
+
+    boost::filesystem::ofstream order_output_stream(config.level_output_path, std::ios::binary);
+
+    unsigned level_size = node_levels.size();
+    order_output_stream.write((char *)&level_size, sizeof(unsigned));
+    order_output_stream.write((char *)node_levels.data(), sizeof(float) * node_levels.size());
+}
+
+void Prepare::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const
+{
+    std::vector<bool> is_core_node(std::move(in_is_core_node));
+    std::vector<char> unpacked_bool_flags(std::move(is_core_node.size()));
+    for (auto i = 0u; i < is_core_node.size(); ++i)
+    {
+        unpacked_bool_flags[i] = is_core_node[i] ? 1 : 0;
+    }
+
+    boost::filesystem::ofstream core_marker_output_stream(config.core_output_path,
+                                                          std::ios::binary);
+    unsigned size = unpacked_bool_flags.size();
+    core_marker_output_stream.write((char *)&size, sizeof(unsigned));
+    core_marker_output_stream.write((char *)unpacked_bool_flags.data(),
+                                    sizeof(char) * unpacked_bool_flags.size());
+}
+
+std::size_t Prepare::WriteContractedGraph(unsigned max_node_id,
+                                          const DeallocatingVector<QueryEdge> &contracted_edge_list)
+{
+    // Sorting contracted edges in a way that the static query graph can read some in in-place.
+    tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end());
+    const unsigned contracted_edge_count = contracted_edge_list.size();
+    SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count
+                           << " edges";
+
+    const FingerPrint fingerprint = FingerPrint::GetValid();
+    boost::filesystem::ofstream hsgr_output_stream(config.graph_output_path, std::ios::binary);
+    hsgr_output_stream.write((char *)&fingerprint, sizeof(FingerPrint));
+    const unsigned max_used_node_id = [&contracted_edge_list]
+    {
+        unsigned tmp_max = 0;
+        for (const QueryEdge &edge : contracted_edge_list)
+        {
+            BOOST_ASSERT(SPECIAL_NODEID != edge.source);
+            BOOST_ASSERT(SPECIAL_NODEID != edge.target);
+            tmp_max = std::max(tmp_max, edge.source);
+            tmp_max = std::max(tmp_max, edge.target);
+        }
+        return tmp_max;
+    }();
+
+    SimpleLogger().Write(logDEBUG) << "input graph has " << (max_node_id + 1) << " nodes";
+    SimpleLogger().Write(logDEBUG) << "contracted graph has " << (max_used_node_id + 1) << " nodes";
+
+    std::vector<StaticGraph<EdgeData>::NodeArrayEntry> node_array;
+    // make sure we have at least one sentinel
+    node_array.resize(max_node_id + 2);
+
+    SimpleLogger().Write() << "Building node array";
+    StaticGraph<EdgeData>::EdgeIterator edge = 0;
+    StaticGraph<EdgeData>::EdgeIterator position = 0;
+    StaticGraph<EdgeData>::EdgeIterator last_edge;
+
+    // initializing 'first_edge'-field of nodes:
+    for (const auto node : osrm::irange(0u, max_used_node_id + 1))
+    {
+        last_edge = edge;
+        while ((edge < contracted_edge_count) && (contracted_edge_list[edge].source == node))
+        {
+            ++edge;
+        }
+        node_array[node].first_edge = position; //=edge
+        position += edge - last_edge;           // remove
+    }
+
+    for (const auto sentinel_counter :
+         osrm::irange<unsigned>(max_used_node_id + 1, node_array.size()))
+    {
+        // sentinel element, guarded against underflow
+        node_array[sentinel_counter].first_edge = contracted_edge_count;
+    }
+
+    SimpleLogger().Write() << "Serializing node array";
+
+    RangebasedCRC32 crc32_calculator;
+    const unsigned edges_crc32 = crc32_calculator(contracted_edge_list);
+    SimpleLogger().Write() << "Writing CRC32: " << edges_crc32;
+
+    const unsigned node_array_size = node_array.size();
+    // serialize crc32, aka checksum
+    hsgr_output_stream.write((char *)&edges_crc32, sizeof(unsigned));
+    // serialize number of nodes
+    hsgr_output_stream.write((char *)&node_array_size, sizeof(unsigned));
+    // serialize number of edges
+    hsgr_output_stream.write((char *)&contracted_edge_count, sizeof(unsigned));
+    // serialize all nodes
+    if (node_array_size > 0)
+    {
+        hsgr_output_stream.write((char *)&node_array[0],
+                                 sizeof(StaticGraph<EdgeData>::NodeArrayEntry) * node_array_size);
+    }
+
+    // serialize all edges
+    SimpleLogger().Write() << "Building edge array";
+    int number_of_used_edges = 0;
+
+    StaticGraph<EdgeData>::EdgeArrayEntry current_edge;
+    for (const auto edge : osrm::irange<std::size_t>(0, contracted_edge_list.size()))
+    {
+        // no eigen loops
+        BOOST_ASSERT(contracted_edge_list[edge].source != contracted_edge_list[edge].target);
+        current_edge.target = contracted_edge_list[edge].target;
+        current_edge.data = contracted_edge_list[edge].data;
+
+        // every target needs to be valid
+        BOOST_ASSERT(current_edge.target <= max_used_node_id);
+#ifndef NDEBUG
+        if (current_edge.data.distance <= 0)
+        {
+            SimpleLogger().Write(logWARNING) << "Edge: " << edge
+                                             << ",source: " << contracted_edge_list[edge].source
+                                             << ", target: " << contracted_edge_list[edge].target
+                                             << ", dist: " << current_edge.data.distance;
+
+            SimpleLogger().Write(logWARNING) << "Failed at adjacency list of node "
+                                             << contracted_edge_list[edge].source << "/"
+                                             << node_array.size() - 1;
+            return 1;
+        }
+#endif
+        hsgr_output_stream.write((char *)&current_edge,
+                                 sizeof(StaticGraph<EdgeData>::EdgeArrayEntry));
+
+        ++number_of_used_edges;
+    }
+
+    return number_of_used_edges;
+}
+
+/**
+ \brief Build contracted graph.
+ */
+void Prepare::ContractGraph(const unsigned max_edge_id,
+                            DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list,
+                            DeallocatingVector<QueryEdge> &contracted_edge_list,
+                            std::vector<bool> &is_core_node,
+                            std::vector<float> &inout_node_levels) const
+{
+    std::vector<float> node_levels;
+    node_levels.swap(inout_node_levels);
+
+    Contractor contractor(max_edge_id + 1, edge_based_edge_list, std::move(node_levels));
+    contractor.Run(config.core_factor);
+    contractor.GetEdges(contracted_edge_list);
+    contractor.GetCoreMarker(is_core_node);
+    contractor.GetNodeLevels(inout_node_levels);
+}

src/engine/descriptors/description_factory.cpp

@@ -0,0 +1,249 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "description_factory.hpp"
+
+#include "../algorithms/polyline_formatter.hpp"
+#include "../algorithms/coordinate_calculation.hpp"
+#include "../data_structures/internal_route_result.hpp"
+#include "../data_structures/turn_instructions.hpp"
+#include "../util/container.hpp"
+#include "../util/integer_range.hpp"
+#include "../typedefs.h"
+
+DescriptionFactory::DescriptionFactory() : entire_length(0) { via_indices.push_back(0); }
+
+std::vector<unsigned> const &DescriptionFactory::GetViaIndices() const { return via_indices; }
+
+void DescriptionFactory::SetStartSegment(const PhantomNode &source, const bool traversed_in_reverse)
+{
+    start_phantom = source;
+    const EdgeWeight segment_duration =
+        (traversed_in_reverse ? source.reverse_weight : source.forward_weight);
+    const TravelMode travel_mode =
+        (traversed_in_reverse ? source.backward_travel_mode : source.forward_travel_mode);
+    AppendSegment(source.location, PathData(0, source.name_id, TurnInstruction::HeadOn,
+                                            segment_duration, travel_mode));
+    BOOST_ASSERT(path_description.back().duration == segment_duration);
+}
+
+void DescriptionFactory::SetEndSegment(const PhantomNode &target,
+                                       const bool traversed_in_reverse,
+                                       const bool is_via_location)
+{
+    target_phantom = target;
+    const EdgeWeight segment_duration =
+        (traversed_in_reverse ? target.reverse_weight : target.forward_weight);
+    const TravelMode travel_mode =
+        (traversed_in_reverse ? target.backward_travel_mode : target.forward_travel_mode);
+    path_description.emplace_back(target.location, target.name_id, segment_duration, 0.f,
+                                  is_via_location ? TurnInstruction::ReachViaLocation
+                                                  : TurnInstruction::NoTurn,
+                                  true, true, travel_mode);
+    BOOST_ASSERT(path_description.back().duration == segment_duration);
+}
+
+void DescriptionFactory::AppendSegment(const FixedPointCoordinate &coordinate,
+                                       const PathData &path_point)
+{
+    // if the start location is on top of a node, the first movement might be zero-length,
+    // in which case we dont' add a new description, but instead update the existing one
+    if ((1 == path_description.size()) && (path_description.front().location == coordinate))
+    {
+        if (path_point.segment_duration > 0)
+        {
+            path_description.front().name_id = path_point.name_id;
+            path_description.front().travel_mode = path_point.travel_mode;
+        }
+        return;
+    }
+
+    // make sure mode changes are announced, even when there otherwise is no turn
+    const TurnInstruction turn = [&]() -> TurnInstruction
+    {
+        if (TurnInstruction::NoTurn == path_point.turn_instruction &&
+            path_description.front().travel_mode != path_point.travel_mode &&
+            path_point.segment_duration > 0)
+        {
+            return TurnInstruction::GoStraight;
+        }
+        return path_point.turn_instruction;
+    }();
+
+    path_description.emplace_back(coordinate, path_point.name_id, path_point.segment_duration, 0.f,
+                                  turn, path_point.travel_mode);
+}
+
+osrm::json::Value DescriptionFactory::AppendGeometryString(const bool return_encoded)
+{
+    if (return_encoded)
+    {
+        return PolylineFormatter().printEncodedString(path_description);
+    }
+    return PolylineFormatter().printUnencodedString(path_description);
+}
+
+void DescriptionFactory::BuildRouteSummary(const double distance, const unsigned time)
+{
+    summary.source_name_id = start_phantom.name_id;
+    summary.target_name_id = target_phantom.name_id;
+    summary.BuildDurationAndLengthStrings(distance, time);
+}
+
+void DescriptionFactory::Run(const unsigned zoom_level)
+{
+    if (path_description.empty())
+    {
+        return;
+    }
+
+    /** starts at index 1 */
+    path_description[0].length = 0.f;
+    for (const auto i : osrm::irange<std::size_t>(1, path_description.size()))
+    {
+        // move down names by one, q&d hack
+        path_description[i - 1].name_id = path_description[i].name_id;
+        path_description[i].length = coordinate_calculation::great_circle_distance(
+            path_description[i - 1].location, path_description[i].location);
+    }
+
+    /*Simplify turn instructions
+    Input :
+    10. Turn left on B 36 for 20 km
+    11. Continue on B 35; B 36 for 2 km
+    12. Continue on B 36 for 13 km
+
+    becomes:
+    10. Turn left on B 36 for 35 km
+    */
+    // TODO: rework to check only end and start of string.
+    //      stl string is way to expensive
+
+    //    unsigned lastTurn = 0;
+    //    for(unsigned i = 1; i < path_description.size(); ++i) {
+    //        string1 = sEngine.GetEscapedNameForNameID(path_description[i].name_id);
+    //        if(TurnInstruction::GoStraight == path_description[i].turn_instruction) {
+    //            if(std::string::npos != string0.find(string1+";")
+    //                  || std::string::npos != string0.find(";"+string1)
+    //                  || std::string::npos != string0.find(string1+" ;")
+    //                    || std::string::npos != string0.find("; "+string1)
+    //                    ){
+    //                SimpleLogger().Write() << "->next correct: " << string0 << " contains " <<
+    //                string1;
+    //                for(; lastTurn != i; ++lastTurn)
+    //                    path_description[lastTurn].name_id = path_description[i].name_id;
+    //                path_description[i].turn_instruction = TurnInstruction::NoTurn;
+    //            } else if(std::string::npos != string1.find(string0+";")
+    //                  || std::string::npos != string1.find(";"+string0)
+    //                    || std::string::npos != string1.find(string0+" ;")
+    //                    || std::string::npos != string1.find("; "+string0)
+    //                    ){
+    //                SimpleLogger().Write() << "->prev correct: " << string1 << " contains " <<
+    //                string0;
+    //                path_description[i].name_id = path_description[i-1].name_id;
+    //                path_description[i].turn_instruction = TurnInstruction::NoTurn;
+    //            }
+    //        }
+    //        if (TurnInstruction::NoTurn != path_description[i].turn_instruction) {
+    //            lastTurn = i;
+    //        }
+    //        string0 = string1;
+    //    }
+    //
+
+    float segment_length = 0.;
+    EdgeWeight segment_duration = 0;
+    std::size_t segment_start_index = 0;
+
+    for (const auto i : osrm::irange<std::size_t>(1, path_description.size()))
+    {
+        entire_length += path_description[i].length;
+        segment_length += path_description[i].length;
+        segment_duration += path_description[i].duration;
+        path_description[segment_start_index].length = segment_length;
+        path_description[segment_start_index].duration = segment_duration;
+
+        if (TurnInstruction::NoTurn != path_description[i].turn_instruction)
+        {
+            BOOST_ASSERT(path_description[i].necessary);
+            segment_length = 0;
+            segment_duration = 0;
+            segment_start_index = i;
+        }
+    }
+
+    // Post-processing to remove empty or nearly empty path segments
+    if (path_description.size() > 2 &&
+        std::numeric_limits<float>::epsilon() > path_description.back().length &&
+        !(path_description.end() - 2)->is_via_location)
+    {
+        path_description.pop_back();
+        path_description.back().necessary = true;
+        path_description.back().turn_instruction = TurnInstruction::NoTurn;
+        target_phantom.name_id = (path_description.end() - 2)->name_id;
+    }
+
+    if (path_description.size() > 2 &&
+        std::numeric_limits<float>::epsilon() > path_description.front().length &&
+        !(path_description.begin() + 1)->is_via_location)
+    {
+        path_description.erase(path_description.begin());
+        path_description.front().turn_instruction = TurnInstruction::HeadOn;
+        path_description.front().necessary = true;
+        start_phantom.name_id = path_description.front().name_id;
+    }
+
+    // Generalize poly line
+    polyline_generalizer.Run(path_description.begin(), path_description.end(), zoom_level);
+
+    // fix what needs to be fixed else
+    unsigned necessary_segments = 0; // a running index that counts the necessary pieces
+    osrm::for_each_pair(
+        path_description, [&](SegmentInformation &first, const SegmentInformation &second)
+        {
+            if (!first.necessary)
+            {
+                return;
+            }
+
+            if (first.is_via_location)
+            { // mark the end of a leg (of several segments)
+                via_indices.push_back(necessary_segments);
+            }
+
+            const double post_turn_bearing = coordinate_calculation::bearing(first.location, second.location);
+            const double pre_turn_bearing = coordinate_calculation::bearing(second.location, first.location);
+            first.post_turn_bearing = static_cast<short>(post_turn_bearing * 10);
+            first.pre_turn_bearing = static_cast<short>(pre_turn_bearing * 10);
+
+            ++necessary_segments;
+        });
+
+    via_indices.push_back(necessary_segments);
+    BOOST_ASSERT(via_indices.size() >= 2);
+    return;
+}

src/engine/douglas_peucker.cpp

@@ -0,0 +1,164 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "douglas_peucker.hpp"
+
+#include "../data_structures/segment_information.hpp"
+
+#include <boost/assert.hpp>
+#include <osrm/coordinate.hpp>
+
+#include <cmath>
+#include <algorithm>
+#include <iterator>
+
+namespace
+{
+struct CoordinatePairCalculator
+{
+    CoordinatePairCalculator() = delete;
+    CoordinatePairCalculator(const FixedPointCoordinate &coordinate_a,
+                             const FixedPointCoordinate &coordinate_b)
+    {
+        // initialize distance calculator with two fixed coordinates a, b
+        const float RAD = 0.017453292519943295769236907684886f;
+        first_lat = (coordinate_a.lat / COORDINATE_PRECISION) * RAD;
+        first_lon = (coordinate_a.lon / COORDINATE_PRECISION) * RAD;
+        second_lat = (coordinate_b.lat / COORDINATE_PRECISION) * RAD;
+        second_lon = (coordinate_b.lon / COORDINATE_PRECISION) * RAD;
+    }
+
+    int operator()(FixedPointCoordinate &other) const
+    {
+        // set third coordinate c
+        const float RAD = 0.017453292519943295769236907684886f;
+        const float earth_radius = 6372797.560856f;
+        const float float_lat1 = (other.lat / COORDINATE_PRECISION) * RAD;
+        const float float_lon1 = (other.lon / COORDINATE_PRECISION) * RAD;
+
+        // compute distance (a,c)
+        const float x_value_1 = (first_lon - float_lon1) * cos((float_lat1 + first_lat) / 2.f);
+        const float y_value_1 = first_lat - float_lat1;
+        const float dist1 = std::hypot(x_value_1, y_value_1) * earth_radius;
+
+        // compute distance (b,c)
+        const float x_value_2 = (second_lon - float_lon1) * cos((float_lat1 + second_lat) / 2.f);
+        const float y_value_2 = second_lat - float_lat1;
+        const float dist2 = std::hypot(x_value_2, y_value_2) * earth_radius;
+
+        // return the minimum
+        return static_cast<int>(std::min(dist1, dist2));
+    }
+
+    float first_lat;
+    float first_lon;
+    float second_lat;
+    float second_lon;
+};
+}
+
+void DouglasPeucker::Run(std::vector<SegmentInformation> &input_geometry, const unsigned zoom_level)
+{
+    Run(std::begin(input_geometry), std::end(input_geometry), zoom_level);
+}
+
+void DouglasPeucker::Run(RandomAccessIt begin, RandomAccessIt end, const unsigned zoom_level)
+{
+    const auto size = std::distance(begin, end);
+    if (size < 2)
+    {
+        return;
+    }
+
+    begin->necessary = true;
+    std::prev(end)->necessary = true;
+
+    {
+        BOOST_ASSERT_MSG(zoom_level < DOUGLAS_PEUCKER_THRESHOLDS.size(), "unsupported zoom level");
+        auto left_border = begin;
+        auto right_border = std::next(begin);
+        // Sweep over array and identify those ranges that need to be checked
+        do
+        {
+            // traverse list until new border element found
+            if (right_border->necessary)
+            {
+                // sanity checks
+                BOOST_ASSERT(left_border->necessary);
+                BOOST_ASSERT(right_border->necessary);
+                recursion_stack.emplace(left_border, right_border);
+                left_border = right_border;
+            }
+            ++right_border;
+        } while (right_border != end);
+    }
+
+    // mark locations as 'necessary' by divide-and-conquer
+    while (!recursion_stack.empty())
+    {
+        // pop next element
+        const GeometryRange pair = recursion_stack.top();
+        recursion_stack.pop();
+        // sanity checks
+        BOOST_ASSERT_MSG(pair.first->necessary, "left border must be necessary");
+        BOOST_ASSERT_MSG(pair.second->necessary, "right border must be necessary");
+        BOOST_ASSERT_MSG(std::distance(pair.second, end) > 0, "right border outside of geometry");
+        BOOST_ASSERT_MSG(std::distance(pair.first, pair.second) >= 0,
+                         "left border on the wrong side");
+
+        int max_int_distance = 0;
+        auto farthest_entry_it = pair.second;
+        const CoordinatePairCalculator dist_calc(pair.first->location, pair.second->location);
+
+        // sweep over range to find the maximum
+        for (auto it = std::next(pair.first); it != pair.second; ++it)
+        {
+            const int distance = dist_calc(it->location);
+            // found new feasible maximum?
+            if (distance > max_int_distance && distance > DOUGLAS_PEUCKER_THRESHOLDS[zoom_level])
+            {
+                farthest_entry_it = it;
+                max_int_distance = distance;
+            }
+        }
+
+        // check if maximum violates a zoom level dependent threshold
+        if (max_int_distance > DOUGLAS_PEUCKER_THRESHOLDS[zoom_level])
+        {
+            //  mark idx as necessary
+            farthest_entry_it->necessary = true;
+            if (1 < std::distance(pair.first, farthest_entry_it))
+            {
+                recursion_stack.emplace(pair.first, farthest_entry_it);
+            }
+            if (1 < std::distance(farthest_entry_it, pair.second))
+            {
+                recursion_stack.emplace(farthest_entry_it, pair.second);
+            }
+        }
+    }
+}

src/engine/osrm_impl.cpp

@@ -0,0 +1,174 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+namespace boost
+{
+namespace interprocess
+{
+class named_mutex;
+}
+}
+
+#include "osrm_impl.hpp"
+
+#include "../plugins/distance_table.hpp"
+#include "../plugins/hello_world.hpp"
+#include "../plugins/nearest.hpp"
+#include "../plugins/timestamp.hpp"
+#include "../plugins/trip.hpp"
+#include "../plugins/viaroute.hpp"
+#include "../plugins/match.hpp"
+#include "../server/data_structures/datafacade_base.hpp"
+#include "../server/data_structures/internal_datafacade.hpp"
+#include "../server/data_structures/shared_barriers.hpp"
+#include "../server/data_structures/shared_datafacade.hpp"
+#include "../util/make_unique.hpp"
+#include "../util/routed_options.hpp"
+#include "../util/simple_logger.hpp"
+
+#include <boost/assert.hpp>
+#include <boost/interprocess/sync/named_condition.hpp>
+#include <boost/interprocess/sync/scoped_lock.hpp>
+
+#include <osrm/route_parameters.hpp>
+#include <osrm/libosrm_config.hpp>
+#include <osrm/osrm.hpp>
+
+#include <algorithm>
+#include <fstream>
+#include <utility>
+#include <vector>
+
+OSRM::OSRM_impl::OSRM_impl(LibOSRMConfig& lib_config)
+{
+    if (lib_config.use_shared_memory)
+    {
+        barrier = osrm::make_unique<SharedBarriers>();
+        query_data_facade = new SharedDataFacade<QueryEdge::EdgeData>();
+    }
+    else
+    {
+        // populate base path
+        populate_base_path(lib_config.server_paths);
+        query_data_facade = new InternalDataFacade<QueryEdge::EdgeData>(lib_config.server_paths);
+    }
+
+    // The following plugins handle all requests.
+    RegisterPlugin(new DistanceTablePlugin<BaseDataFacade<QueryEdge::EdgeData>>(
+        query_data_facade, lib_config.max_locations_distance_table));
+    RegisterPlugin(new HelloWorldPlugin());
+    RegisterPlugin(new NearestPlugin<BaseDataFacade<QueryEdge::EdgeData>>(query_data_facade));
+    RegisterPlugin(new MapMatchingPlugin<BaseDataFacade<QueryEdge::EdgeData>>(
+        query_data_facade, lib_config.max_locations_map_matching));
+    RegisterPlugin(new TimestampPlugin<BaseDataFacade<QueryEdge::EdgeData>>(query_data_facade));
+    RegisterPlugin(new ViaRoutePlugin<BaseDataFacade<QueryEdge::EdgeData>>(query_data_facade,
+                lib_config.max_locations_viaroute));
+    RegisterPlugin(new RoundTripPlugin<BaseDataFacade<QueryEdge::EdgeData>>(query_data_facade,
+                lib_config.max_locations_trip));
+}
+
+void OSRM::OSRM_impl::RegisterPlugin(BasePlugin *raw_plugin_ptr)
+{
+    std::unique_ptr<BasePlugin> plugin_ptr(raw_plugin_ptr);
+    SimpleLogger().Write() << "loaded plugin: " << plugin_ptr->GetDescriptor();
+    plugin_map[plugin_ptr->GetDescriptor()] = std::move(plugin_ptr);
+}
+
+int OSRM::OSRM_impl::RunQuery(const RouteParameters &route_parameters, osrm::json::Object &json_result)
+{
+    const auto &plugin_iterator = plugin_map.find(route_parameters.service);
+
+    if (plugin_map.end() == plugin_iterator)
+    {
+        json_result.values["status_message"] = "Service not found";
+        return 400;
+    }
+
+    increase_concurrent_query_count();
+    auto return_code = plugin_iterator->second->HandleRequest(route_parameters, json_result);
+    decrease_concurrent_query_count();
+    return static_cast<int>(return_code);
+}
+
+// decrease number of concurrent queries
+void OSRM::OSRM_impl::decrease_concurrent_query_count()
+{
+    if (!barrier)
+    {
+        return;
+    }
+    // lock query
+    boost::interprocess::scoped_lock<boost::interprocess::named_mutex> query_lock(
+        barrier->query_mutex);
+
+    // decrement query count
+    --(barrier->number_of_queries);
+    BOOST_ASSERT_MSG(0 <= barrier->number_of_queries, "invalid number of queries");
+
+    // notify all processes that were waiting for this condition
+    if (0 == barrier->number_of_queries)
+    {
+        barrier->no_running_queries_condition.notify_all();
+    }
+}
+
+// increase number of concurrent queries
+void OSRM::OSRM_impl::increase_concurrent_query_count()
+{
+    if (!barrier)
+    {
+        return;
+    }
+
+    // lock update pending
+    boost::interprocess::scoped_lock<boost::interprocess::named_mutex> pending_lock(
+        barrier->pending_update_mutex);
+
+    // lock query
+    boost::interprocess::scoped_lock<boost::interprocess::named_mutex> query_lock(
+        barrier->query_mutex);
+
+    // unlock update pending
+    pending_lock.unlock();
+
+    // increment query count
+    ++(barrier->number_of_queries);
+
+    (static_cast<SharedDataFacade<QueryEdge::EdgeData> *>(query_data_facade))
+        ->CheckAndReloadFacade();
+}
+
+// proxy code for compilation firewall
+OSRM::OSRM(LibOSRMConfig &lib_config) : OSRM_pimpl_(osrm::make_unique<OSRM_impl>(lib_config)) {}
+
+// needed because unique_ptr needs the size of OSRM_impl for delete
+OSRM::~OSRM() {}
+
+int OSRM::RunQuery(const RouteParameters &route_parameters, osrm::json::Object &json_result)
+{
+    return OSRM_pimpl_->RunQuery(route_parameters, json_result);
+}

src/engine/phantom_node.cpp

@@ -0,0 +1,105 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "phantom_node.hpp"
+
+#include "../typedefs.h"
+#include "travel_mode.hpp"
+
+#include <osrm/coordinate.hpp>
+
+#include <limits>
+
+PhantomNode::PhantomNode(NodeID forward_node_id,
+                         NodeID reverse_node_id,
+                         unsigned name_id,
+                         int forward_weight,
+                         int reverse_weight,
+                         int forward_offset,
+                         int reverse_offset,
+                         unsigned packed_geometry_id,
+                         bool is_tiny_component,
+                         unsigned component_id,
+                         FixedPointCoordinate &location,
+                         unsigned short fwd_segment_position,
+                         TravelMode forward_travel_mode,
+                         TravelMode backward_travel_mode)
+    : forward_node_id(forward_node_id), reverse_node_id(reverse_node_id), name_id(name_id),
+      forward_weight(forward_weight), reverse_weight(reverse_weight),
+      forward_offset(forward_offset), reverse_offset(reverse_offset),
+      packed_geometry_id(packed_geometry_id), component{component_id, is_tiny_component}, location(location),
+      fwd_segment_position(fwd_segment_position), forward_travel_mode(forward_travel_mode),
+      backward_travel_mode(backward_travel_mode)
+{
+}
+
+PhantomNode::PhantomNode()
+    : forward_node_id(SPECIAL_NODEID), reverse_node_id(SPECIAL_NODEID),
+      name_id(std::numeric_limits<unsigned>::max()), forward_weight(INVALID_EDGE_WEIGHT),
+      reverse_weight(INVALID_EDGE_WEIGHT), forward_offset(0), reverse_offset(0),
+      packed_geometry_id(SPECIAL_EDGEID), component{INVALID_COMPONENTID, false},
+      fwd_segment_position(0), forward_travel_mode(TRAVEL_MODE_INACCESSIBLE),
+      backward_travel_mode(TRAVEL_MODE_INACCESSIBLE)
+{
+}
+
+int PhantomNode::GetForwardWeightPlusOffset() const
+{
+    if (SPECIAL_NODEID == forward_node_id)
+    {
+        return 0;
+    }
+    return forward_offset + forward_weight;
+}
+
+int PhantomNode::GetReverseWeightPlusOffset() const
+{
+    if (SPECIAL_NODEID == reverse_node_id)
+    {
+        return 0;
+    }
+    return reverse_offset + reverse_weight;
+}
+
+bool PhantomNode::is_bidirected() const
+{
+    return (forward_node_id != SPECIAL_NODEID) && (reverse_node_id != SPECIAL_NODEID);
+}
+
+bool PhantomNode::is_compressed() const { return (forward_offset != 0) || (reverse_offset != 0); }
+
+bool PhantomNode::is_valid(const unsigned number_of_nodes) const
+{
+    return location.is_valid() &&
+           ((forward_node_id < number_of_nodes) || (reverse_node_id < number_of_nodes)) &&
+           ((forward_weight != INVALID_EDGE_WEIGHT) || (reverse_weight != INVALID_EDGE_WEIGHT)) &&
+           (component.id != INVALID_COMPONENTID) && (name_id != INVALID_NAMEID);
+}
+
+bool PhantomNode::is_valid() const { return location.is_valid() && (name_id != INVALID_NAMEID); }
+
+bool PhantomNode::operator==(const PhantomNode &other) const { return location == other.location; }

src/engine/polyline_compressor.cpp

@@ -0,0 +1,128 @@
+/*
+
+Copyright (c) 2014, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "polyline_compressor.hpp"
+#include "../data_structures/segment_information.hpp"
+
+#include <osrm/coordinate.hpp>
+
+std::string PolylineCompressor::encode_vector(std::vector<int> &numbers) const
+{
+    std::string output;
+    const auto end = numbers.size();
+    for (std::size_t i = 0; i < end; ++i)
+    {
+        numbers[i] <<= 1;
+        if (numbers[i] < 0)
+        {
+            numbers[i] = ~(numbers[i]);
+        }
+    }
+    for (const int number : numbers)
+    {
+        output += encode_number(number);
+    }
+    return output;
+}
+
+std::string PolylineCompressor::encode_number(int number_to_encode) const
+{
+    std::string output;
+    while (number_to_encode >= 0x20)
+    {
+        const int next_value = (0x20 | (number_to_encode & 0x1f)) + 63;
+        output += static_cast<char>(next_value);
+        number_to_encode >>= 5;
+    }
+
+    number_to_encode += 63;
+    output += static_cast<char>(number_to_encode);
+    return output;
+}
+
+std::string
+PolylineCompressor::get_encoded_string(const std::vector<SegmentInformation> &polyline) const
+{
+    if (polyline.empty())
+    {
+        return {};
+    }
+
+    std::vector<int> delta_numbers;
+    delta_numbers.reserve((polyline.size() - 1) * 2);
+    FixedPointCoordinate previous_coordinate = {0, 0};
+    for (const auto &segment : polyline)
+    {
+        if (segment.necessary)
+        {
+            const int lat_diff = segment.location.lat - previous_coordinate.lat;
+            const int lon_diff = segment.location.lon - previous_coordinate.lon;
+            delta_numbers.emplace_back(lat_diff);
+            delta_numbers.emplace_back(lon_diff);
+            previous_coordinate = segment.location;
+        }
+    }
+    return encode_vector(delta_numbers);
+}
+
+std::vector<FixedPointCoordinate> PolylineCompressor::decode_string(const std::string &geometry_string) const
+{
+    std::vector<FixedPointCoordinate> new_coordinates;
+    int index = 0, len = geometry_string.size();
+    int lat = 0, lng = 0;
+  
+    while (index < len) 
+    {
+        int b, shift = 0, result = 0;
+        do 
+        {
+            b = geometry_string.at(index++) - 63;
+            result |= (b & 0x1f) << shift;
+            shift += 5;
+        } while (b >= 0x20);
+        int dlat = ((result & 1) != 0 ? ~(result >> 1) : (result >> 1));
+        lat += dlat;
+
+        shift = 0;
+        result = 0;
+        do 
+        {
+            b = geometry_string.at(index++) - 63;
+            result |= (b & 0x1f) << shift;
+            shift += 5;
+        } while (b >= 0x20);
+        int dlng = ((result & 1) != 0 ? ~(result >> 1) : (result >> 1));
+        lng += dlng;
+
+        FixedPointCoordinate p;
+        p.lat = COORDINATE_PRECISION * (((double) lat / 1E6));
+        p.lon = COORDINATE_PRECISION * (((double) lng / 1E6));
+        new_coordinates.push_back(p);
+    }
+
+    return new_coordinates;
+}

src/engine/polyline_formatter.cpp

@@ -0,0 +1,56 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "polyline_formatter.hpp"
+
+#include "polyline_compressor.hpp"
+#include "../data_structures/segment_information.hpp"
+
+#include <osrm/coordinate.hpp>
+
+osrm::json::String
+PolylineFormatter::printEncodedString(const std::vector<SegmentInformation> &polyline) const
+{
+    return osrm::json::String(PolylineCompressor().get_encoded_string(polyline));
+}
+
+osrm::json::Array
+PolylineFormatter::printUnencodedString(const std::vector<SegmentInformation> &polyline) const
+{
+    osrm::json::Array json_geometry_array;
+    for (const auto &segment : polyline)
+    {
+        if (segment.necessary)
+        {
+            osrm::json::Array json_coordinate;
+            json_coordinate.values.push_back(segment.location.lat / COORDINATE_PRECISION);
+            json_coordinate.values.push_back(segment.location.lon / COORDINATE_PRECISION);
+            json_geometry_array.values.push_back(json_coordinate);
+        }
+    }
+    return json_geometry_array;
+}

src/engine/route_parameters.cpp

@@ -0,0 +1,179 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include <boost/fusion/container/vector.hpp>
+#include <boost/fusion/sequence/intrinsic.hpp>
+#include <boost/fusion/include/at_c.hpp>
+#include <boost/spirit/include/qi.hpp>
+
+#include <osrm/route_parameters.hpp>
+
+#include "../algorithms/polyline_compressor.hpp"
+
+RouteParameters::RouteParameters()
+    : zoom_level(18), print_instructions(false), alternate_route(true), geometry(true),
+      compression(true), deprecatedAPI(false), uturn_default(false), classify(false),
+      matching_beta(5), gps_precision(5), check_sum(-1), num_results(1)
+{
+}
+
+void RouteParameters::setZoomLevel(const short level)
+{
+    if (18 >= level && 0 <= level)
+    {
+        zoom_level = level;
+    }
+}
+
+void RouteParameters::setNumberOfResults(const short number)
+{
+    if (number > 0 && number <= 100)
+    {
+        num_results = number;
+    }
+}
+
+void RouteParameters::setAlternateRouteFlag(const bool flag) { alternate_route = flag; }
+
+void RouteParameters::setUTurn(const bool flag)
+{
+    // the API grammar should make sure this never happens
+    BOOST_ASSERT(!uturns.empty());
+    uturns.back() = flag;
+}
+
+void RouteParameters::setAllUTurns(const bool flag)
+{
+    // if the flag flips the default, then we erase everything.
+    if (flag)
+    {
+        uturn_default = flag;
+        uturns.clear();
+        uturns.resize(coordinates.size(), uturn_default);
+    }
+}
+
+void RouteParameters::setDeprecatedAPIFlag(const std::string &) { deprecatedAPI = true; }
+
+void RouteParameters::setChecksum(const unsigned sum) { check_sum = sum; }
+
+void RouteParameters::setInstructionFlag(const bool flag) { print_instructions = flag; }
+
+void RouteParameters::setService(const std::string &service_string) { service = service_string; }
+
+void RouteParameters::setClassify(const bool flag) { classify = flag; }
+
+void RouteParameters::setMatchingBeta(const double beta) { matching_beta = beta; }
+
+void RouteParameters::setGPSPrecision(const double precision) { gps_precision = precision; }
+
+void RouteParameters::setOutputFormat(const std::string &format) { output_format = format; }
+
+void RouteParameters::setJSONpParameter(const std::string &parameter)
+{
+    jsonp_parameter = parameter;
+}
+
+void RouteParameters::addHint(const std::string &hint)
+{
+    hints.resize(coordinates.size());
+    if (!hints.empty())
+    {
+        hints.back() = hint;
+    }
+}
+
+void RouteParameters::addTimestamp(const unsigned timestamp)
+{
+    timestamps.resize(coordinates.size());
+    if (!timestamps.empty())
+    {
+        timestamps.back() = timestamp;
+    }
+}
+
+void RouteParameters::addBearing(
+    const boost::fusion::vector<int, boost::optional<int>> &received_bearing,
+        boost::spirit::qi::unused_type /* unused */, bool& pass)
+{
+    pass = false;
+    const int bearing = boost::fusion::at_c<0>(received_bearing);
+    const boost::optional<int> range = boost::fusion::at_c<1>(received_bearing);
+    if (bearing < 0 || bearing > 359) return;
+    if (range && (*range < 0 || *range > 180)) return;
+    bearings.emplace_back(std::make_pair(bearing,range));
+    pass = true;
+}
+
+void RouteParameters::setLanguage(const std::string &language_string)
+{
+    language = language_string;
+}
+
+void RouteParameters::setGeometryFlag(const bool flag) { geometry = flag; }
+
+void RouteParameters::setCompressionFlag(const bool flag) { compression = flag; }
+
+void RouteParameters::addCoordinate(
+    const boost::fusion::vector<double, double> &received_coordinates)
+{
+    coordinates.emplace_back(
+        static_cast<int>(COORDINATE_PRECISION * boost::fusion::at_c<0>(received_coordinates)),
+        static_cast<int>(COORDINATE_PRECISION * boost::fusion::at_c<1>(received_coordinates)));
+    is_source.push_back(true);
+    is_destination.push_back(true);
+    uturns.push_back(uturn_default);
+}
+
+void RouteParameters::addDestination(
+    const boost::fusion::vector<double, double> &received_coordinates)
+{
+    coordinates.emplace_back(
+        static_cast<int>(COORDINATE_PRECISION * boost::fusion::at_c<0>(received_coordinates)),
+        static_cast<int>(COORDINATE_PRECISION * boost::fusion::at_c<1>(received_coordinates)));
+    is_source.push_back(false);
+    is_destination.push_back(true);
+    uturns.push_back(uturn_default);
+}
+
+void RouteParameters::addSource(
+    const boost::fusion::vector<double, double> &received_coordinates)
+{
+    coordinates.emplace_back(
+        static_cast<int>(COORDINATE_PRECISION * boost::fusion::at_c<0>(received_coordinates)),
+        static_cast<int>(COORDINATE_PRECISION * boost::fusion::at_c<1>(received_coordinates)));
+    is_source.push_back(true);
+    is_destination.push_back(false);
+    uturns.push_back(uturn_default);
+}
+
+void RouteParameters::getCoordinatesFromGeometry(const std::string &geometry_string)
+{
+    PolylineCompressor pc;
+    coordinates = pc.decode_string(geometry_string);
+}
+

src/engine/search_engine_data.cpp

@@ -0,0 +1,93 @@
+/*
+
+Copyright (c) 2013, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "search_engine_data.hpp"
+
+#include "binary_heap.hpp"
+
+void SearchEngineData::InitializeOrClearFirstThreadLocalStorage(const unsigned number_of_nodes)
+{
+    if (forward_heap_1.get())
+    {
+        forward_heap_1->Clear();
+    }
+    else
+    {
+        forward_heap_1.reset(new QueryHeap(number_of_nodes));
+    }
+
+    if (reverse_heap_1.get())
+    {
+        reverse_heap_1->Clear();
+    }
+    else
+    {
+        reverse_heap_1.reset(new QueryHeap(number_of_nodes));
+    }
+}
+
+void SearchEngineData::InitializeOrClearSecondThreadLocalStorage(const unsigned number_of_nodes)
+{
+    if (forward_heap_2.get())
+    {
+        forward_heap_2->Clear();
+    }
+    else
+    {
+        forward_heap_2.reset(new QueryHeap(number_of_nodes));
+    }
+
+    if (reverse_heap_2.get())
+    {
+        reverse_heap_2->Clear();
+    }
+    else
+    {
+        reverse_heap_2.reset(new QueryHeap(number_of_nodes));
+    }
+}
+
+void SearchEngineData::InitializeOrClearThirdThreadLocalStorage(const unsigned number_of_nodes)
+{
+    if (forward_heap_3.get())
+    {
+        forward_heap_3->Clear();
+    }
+    else
+    {
+        forward_heap_3.reset(new QueryHeap(number_of_nodes));
+    }
+
+    if (reverse_heap_3.get())
+    {
+        reverse_heap_3->Clear();
+    }
+    else
+    {
+        reverse_heap_3.reset(new QueryHeap(number_of_nodes));
+    }
+}

src/extractor/compressed_edge_container.cpp

@@ -0,0 +1,240 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "compressed_edge_container.hpp"
+#include "../util/simple_logger.hpp"
+
+#include <boost/assert.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
+
+#include <limits>
+#include <string>
+
+#include <iostream>
+
+CompressedEdgeContainer::CompressedEdgeContainer()
+{
+    m_free_list.reserve(100);
+    IncreaseFreeList();
+}
+
+void CompressedEdgeContainer::IncreaseFreeList()
+{
+    m_compressed_geometries.resize(m_compressed_geometries.size() + 100);
+    for (unsigned i = 100; i > 0; --i)
+    {
+        m_free_list.emplace_back(free_list_maximum);
+        ++free_list_maximum;
+    }
+}
+
+bool CompressedEdgeContainer::HasEntryForID(const EdgeID edge_id) const
+{
+    auto iter = m_edge_id_to_list_index_map.find(edge_id);
+    return iter != m_edge_id_to_list_index_map.end();
+}
+
+unsigned CompressedEdgeContainer::GetPositionForID(const EdgeID edge_id) const
+{
+    auto map_iterator = m_edge_id_to_list_index_map.find(edge_id);
+    BOOST_ASSERT(map_iterator != m_edge_id_to_list_index_map.end());
+    BOOST_ASSERT(map_iterator->second < m_compressed_geometries.size());
+    return map_iterator->second;
+}
+
+void CompressedEdgeContainer::SerializeInternalVector(const std::string &path) const
+{
+
+    boost::filesystem::fstream geometry_out_stream(path, std::ios::binary | std::ios::out);
+    const unsigned compressed_geometries = m_compressed_geometries.size() + 1;
+    BOOST_ASSERT(std::numeric_limits<unsigned>::max() != compressed_geometries);
+    geometry_out_stream.write((char *)&compressed_geometries, sizeof(unsigned));
+
+    // write indices array
+    unsigned prefix_sum_of_list_indices = 0;
+    for (const auto &elem : m_compressed_geometries)
+    {
+        geometry_out_stream.write((char *)&prefix_sum_of_list_indices, sizeof(unsigned));
+
+        const std::vector<CompressedNode> &current_vector = elem;
+        const unsigned unpacked_size = current_vector.size();
+        BOOST_ASSERT(std::numeric_limits<unsigned>::max() != unpacked_size);
+        prefix_sum_of_list_indices += unpacked_size;
+    }
+    // sentinel element
+    geometry_out_stream.write((char *)&prefix_sum_of_list_indices, sizeof(unsigned));
+
+    // number of geometry entries to follow, it is the (inclusive) prefix sum
+    geometry_out_stream.write((char *)&prefix_sum_of_list_indices, sizeof(unsigned));
+
+    unsigned control_sum = 0;
+    // write compressed geometries
+    for (auto &elem : m_compressed_geometries)
+    {
+        const std::vector<CompressedNode> &current_vector = elem;
+        const unsigned unpacked_size = current_vector.size();
+        control_sum += unpacked_size;
+        BOOST_ASSERT(std::numeric_limits<unsigned>::max() != unpacked_size);
+        for (const CompressedNode current_node : current_vector)
+        {
+            geometry_out_stream.write((char *)&(current_node.first), sizeof(NodeID));
+        }
+    }
+    BOOST_ASSERT(control_sum == prefix_sum_of_list_indices);
+    // all done, let's close the resource
+    geometry_out_stream.close();
+}
+
+void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1,
+                                      const EdgeID edge_id_2,
+                                      const NodeID via_node_id,
+                                      const NodeID target_node_id,
+                                      const EdgeWeight weight1,
+                                      const EdgeWeight weight2)
+{
+    // remove super-trivial geometries
+    BOOST_ASSERT(SPECIAL_EDGEID != edge_id_1);
+    BOOST_ASSERT(SPECIAL_EDGEID != edge_id_2);
+    BOOST_ASSERT(SPECIAL_NODEID != via_node_id);
+    BOOST_ASSERT(SPECIAL_NODEID != target_node_id);
+    BOOST_ASSERT(INVALID_EDGE_WEIGHT != weight1);
+    BOOST_ASSERT(INVALID_EDGE_WEIGHT != weight2);
+
+    // append list of removed edge_id plus via node to surviving edge id:
+    // <surv_1, .. , surv_n, via_node_id, rem_1, .. rem_n
+    //
+    // General scheme:
+    // 1. append via node id to list of edge_id_1
+    // 2. find list for edge_id_2, if yes add all elements and delete it
+
+    // Add via node id. List is created if it does not exist
+    if (!HasEntryForID(edge_id_1))
+    {
+        // create a new entry in the map
+        if (0 == m_free_list.size())
+        {
+            // make sure there is a place to put the entries
+            IncreaseFreeList();
+        }
+        BOOST_ASSERT(!m_free_list.empty());
+        m_edge_id_to_list_index_map[edge_id_1] = m_free_list.back();
+        m_free_list.pop_back();
+    }
+
+    // find bucket index
+    const auto iter = m_edge_id_to_list_index_map.find(edge_id_1);
+    BOOST_ASSERT(iter != m_edge_id_to_list_index_map.end());
+    const unsigned edge_bucket_id1 = iter->second;
+    BOOST_ASSERT(edge_bucket_id1 == GetPositionForID(edge_id_1));
+    BOOST_ASSERT(edge_bucket_id1 < m_compressed_geometries.size());
+
+    std::vector<CompressedNode> &edge_bucket_list1 = m_compressed_geometries[edge_bucket_id1];
+
+    // note we don't save the start coordinate: it is implicitly given by edge 1
+    // weight1 is the distance to the (currently) last coordinate in the bucket
+    if (edge_bucket_list1.empty())
+    {
+        edge_bucket_list1.emplace_back(via_node_id, weight1);
+    }
+
+    BOOST_ASSERT(0 < edge_bucket_list1.size());
+    BOOST_ASSERT(!edge_bucket_list1.empty());
+
+    if (HasEntryForID(edge_id_2))
+    {
+        // second edge is not atomic anymore
+        const unsigned list_to_remove_index = GetPositionForID(edge_id_2);
+        BOOST_ASSERT(list_to_remove_index < m_compressed_geometries.size());
+
+        std::vector<CompressedNode> &edge_bucket_list2 =
+            m_compressed_geometries[list_to_remove_index];
+
+        // found an existing list, append it to the list of edge_id_1
+        edge_bucket_list1.insert(edge_bucket_list1.end(), edge_bucket_list2.begin(),
+                                 edge_bucket_list2.end());
+
+        // remove the list of edge_id_2
+        m_edge_id_to_list_index_map.erase(edge_id_2);
+        BOOST_ASSERT(m_edge_id_to_list_index_map.end() ==
+                     m_edge_id_to_list_index_map.find(edge_id_2));
+        edge_bucket_list2.clear();
+        BOOST_ASSERT(0 == edge_bucket_list2.size());
+        m_free_list.emplace_back(list_to_remove_index);
+        BOOST_ASSERT(list_to_remove_index == m_free_list.back());
+    }
+    else
+    {
+        // we are certain that the second edge is atomic.
+        edge_bucket_list1.emplace_back(target_node_id, weight2);
+    }
+}
+
+void CompressedEdgeContainer::PrintStatistics() const
+{
+    const uint64_t compressed_edges = m_compressed_geometries.size();
+    BOOST_ASSERT(0 == compressed_edges % 2);
+    BOOST_ASSERT(m_compressed_geometries.size() + m_free_list.size() > 0);
+
+    uint64_t compressed_geometries = 0;
+    uint64_t longest_chain_length = 0;
+    for (const std::vector<CompressedNode> &current_vector : m_compressed_geometries)
+    {
+        compressed_geometries += current_vector.size();
+        longest_chain_length = std::max(longest_chain_length, (uint64_t)current_vector.size());
+    }
+
+    SimpleLogger().Write() << "Geometry successfully removed:"
+                              "\n  compressed edges: " << compressed_edges
+                           << "\n  compressed geometries: " << compressed_geometries
+                           << "\n  longest chain length: " << longest_chain_length
+                           << "\n  cmpr ratio: " << ((float)compressed_edges /
+                                                     std::max(compressed_geometries, (uint64_t)1))
+                           << "\n  avg chain length: "
+                           << (float)compressed_geometries /
+                                  std::max((uint64_t)1, compressed_edges);
+}
+
+const CompressedEdgeContainer::EdgeBucket&
+CompressedEdgeContainer::GetBucketReference(const EdgeID edge_id) const
+{
+    const unsigned index = m_edge_id_to_list_index_map.at(edge_id);
+    return m_compressed_geometries.at(index);
+}
+
+NodeID CompressedEdgeContainer::GetFirstEdgeTargetID(const EdgeID edge_id) const
+{
+    const auto &bucket = GetBucketReference(edge_id);
+    BOOST_ASSERT(bucket.size() >= 2);
+    return bucket.front().first;
+}
+NodeID CompressedEdgeContainer::GetLastEdgeSourceID(const EdgeID edge_id) const
+{
+    const auto &bucket = GetBucketReference(edge_id);
+    BOOST_ASSERT(bucket.size() >= 2);
+    return bucket[bucket.size() - 2].first;
+}

src/extractor/edge_based_graph_factory.cpp

@@ -0,0 +1,689 @@
+/*
+
+Copyright (c) 2015, Project OSRM, Dennis Luxen, others
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "edge_based_graph_factory.hpp"
+#include "../algorithms/coordinate_calculation.hpp"
+#include "../data_structures/percent.hpp"
+#include "../util/compute_angle.hpp"
+#include "../util/integer_range.hpp"
+#include "../util/lua_util.hpp"
+#include "../util/simple_logger.hpp"
+#include "../util/timing_util.hpp"
+#include "../util/osrm_exception.hpp"
+
+#include "../util/debug_geometry.hpp"
+
+#include <boost/assert.hpp>
+
+#include <fstream>
+#include <iomanip>
+#include <limits>
+
+EdgeBasedGraphFactory::EdgeBasedGraphFactory(
+    std::shared_ptr<NodeBasedDynamicGraph> node_based_graph,
+    const CompressedEdgeContainer &compressed_edge_container,
+    const std::unordered_set<NodeID> &barrier_nodes,
+    const std::unordered_set<NodeID> &traffic_lights,
+    std::shared_ptr<const RestrictionMap> restriction_map,
+    const std::vector<QueryNode> &node_info_list,
+    SpeedProfileProperties speed_profile)
+    : m_max_edge_id(0), m_node_info_list(node_info_list), m_node_based_graph(std::move(node_based_graph)),
+      m_restriction_map(std::move(restriction_map)), m_barrier_nodes(barrier_nodes),
+      m_traffic_lights(traffic_lights), m_compressed_edge_container(compressed_edge_container),
+      speed_profile(std::move(speed_profile))
+{
+}
+
+void EdgeBasedGraphFactory::GetEdgeBasedEdges(DeallocatingVector<EdgeBasedEdge> &output_edge_list)
+{
+    BOOST_ASSERT_MSG(0 == output_edge_list.size(), "Vector is not empty");
+    using std::swap; // Koenig swap
+    swap(m_edge_based_edge_list, output_edge_list);
+}
+
+void EdgeBasedGraphFactory::GetEdgeBasedNodes(std::vector<EdgeBasedNode> &nodes)
+{
+#ifndef NDEBUG
+    for (const EdgeBasedNode &node : m_edge_based_node_list)
+    {
+        BOOST_ASSERT(m_node_info_list.at(node.u).lat != INT_MAX);
+        BOOST_ASSERT(m_node_info_list.at(node.u).lon != INT_MAX);
+        BOOST_ASSERT(m_node_info_list.at(node.v).lon != INT_MAX);
+        BOOST_ASSERT(m_node_info_list.at(node.v).lat != INT_MAX);
+    }
+#endif
+    using std::swap; // Koenig swap
+    swap(nodes, m_edge_based_node_list);
+}
+
+void EdgeBasedGraphFactory::GetStartPointMarkers(std::vector<bool> &node_is_startpoint)
+{
+    using std::swap; // Koenig swap
+    swap(m_edge_based_node_is_startpoint, node_is_startpoint);
+}
+
+unsigned EdgeBasedGraphFactory::GetHighestEdgeID()
+{
+    return m_max_edge_id;
+}
+
+void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u,
+                                                const NodeID node_v)
+{
+    // merge edges together into one EdgeBasedNode
+    BOOST_ASSERT(node_u != SPECIAL_NODEID);
+    BOOST_ASSERT(node_v != SPECIAL_NODEID);
+
+    // find forward edge id and
+    const EdgeID edge_id_1 = m_node_based_graph->FindEdge(node_u, node_v);
+    BOOST_ASSERT(edge_id_1 != SPECIAL_EDGEID);
+
+    const EdgeData &forward_data = m_node_based_graph->GetEdgeData(edge_id_1);
+
+    // find reverse edge id and
+    const EdgeID edge_id_2 = m_node_based_graph->FindEdge(node_v, node_u);
+    BOOST_ASSERT(edge_id_2 != SPECIAL_EDGEID);
+
+    const EdgeData &reverse_data = m_node_based_graph->GetEdgeData(edge_id_2);
+
+    if (forward_data.edge_id == SPECIAL_NODEID &&
+        reverse_data.edge_id == SPECIAL_NODEID)
+    {
+        return;
+    }
+
+    BOOST_ASSERT(m_compressed_edge_container.HasEntryForID(edge_id_1) ==
+                 m_compressed_edge_container.HasEntryForID(edge_id_2));
+    if (m_compressed_edge_container.HasEntryForID(edge_id_1))
+    {
+        BOOST_ASSERT(m_compressed_edge_container.HasEntryForID(edge_id_2));
+
+        // reconstruct geometry and put in each individual edge with its offset
+        const auto& forward_geometry = m_compressed_edge_container.GetBucketReference(edge_id_1);
+        const auto& reverse_geometry = m_compressed_edge_container.GetBucketReference(edge_id_2);
+        BOOST_ASSERT(forward_geometry.size() == reverse_geometry.size());
+        BOOST_ASSERT(0 != forward_geometry.size());
+        const unsigned geometry_size = static_cast<unsigned>(forward_geometry.size());
+        BOOST_ASSERT(geometry_size > 1);
+
+        // reconstruct bidirectional edge with individual weights and put each into the NN index
+
+        std::vector<int> forward_dist_prefix_sum(forward_geometry.size(), 0);
+        std::vector<int> reverse_dist_prefix_sum(reverse_geometry.size(), 0);
+
+        // quick'n'dirty prefix sum as std::partial_sum needs addtional casts
+        // TODO: move to lambda function with C++11
+        int temp_sum = 0;
+
+        for (const auto i : osrm::irange(0u, geometry_size))
+        {
+            forward_dist_prefix_sum[i] = temp_sum;
+            temp_sum += forward_geometry[i].second;
+
+            BOOST_ASSERT(forward_data.distance >= temp_sum);
+        }
+
+        temp_sum = 0;
+        for (const auto i : osrm::irange(0u, geometry_size))
+        {
+            temp_sum += reverse_geometry[reverse_geometry.size() - 1 - i].second;
+            reverse_dist_prefix_sum[i] = reverse_data.distance - temp_sum;
+            // BOOST_ASSERT(reverse_data.distance >= temp_sum);
+        }
+
+        NodeID current_edge_source_coordinate_id = node_u;
+
+        // traverse arrays from start and end respectively
+        for (const auto i : osrm::irange(0u, geometry_size))
+        {
+            BOOST_ASSERT(current_edge_source_coordinate_id ==
+                         reverse_geometry[geometry_size - 1 - i].first);
+            const NodeID current_edge_target_coordinate_id = forward_geometry[i].first;
+            BOOST_ASSERT(current_edge_target_coordinate_id != current_edge_source_coordinate_id);
+
+            // build edges
+            m_edge_based_node_list.emplace_back(
+                forward_data.edge_id, reverse_data.edge_id,
+                current_edge_source_coordinate_id, current_edge_target_coordinate_id,
+                forward_data.name_id, forward_geometry[i].second,
+                reverse_geometry[geometry_size - 1 - i].second, forward_dist_prefix_sum[i],
+                reverse_dist_prefix_sum[i], m_compressed_edge_container.GetPositionForID(edge_id_1),
+                false, INVALID_COMPONENTID, i, forward_data.travel_mode, reverse_data.travel_mode);
+            m_edge_based_node_is_startpoint.push_back(forward_data.startpoint || reverse_data.startpoint);
+            current_edge_source_coordinate_id = current_edge_target_coordinate_id;
+
+            BOOST_ASSERT(m_edge_based_node_list.back().IsCompressed());
+
+            BOOST_ASSERT(node_u != m_edge_based_node_list.back().u ||
+                         node_v != m_edge_based_node_list.back().v);
+
+            BOOST_ASSERT(node_u != m_edge_based_node_list.back().v ||
+                         node_v != m_edge_based_node_list.back().u);
+        }
+
+        BOOST_ASSERT(current_edge_source_coordinate_id == node_v);
+        BOOST_ASSERT(m_edge_based_node_list.back().IsCompressed());
+    }
+    else
+    {
+        BOOST_ASSERT(!m_compressed_edge_container.HasEntryForID(edge_id_2));
+
+        if (forward_data.edge_id != SPECIAL_NODEID)
+        {
+            BOOST_ASSERT(!forward_data.reversed);
+        }
+        else
+        {
+            BOOST_ASSERT(forward_data.reversed);
+        }
+
+        if (reverse_data.edge_id != SPECIAL_NODEID)
+        {
+            BOOST_ASSERT(!reverse_data.reversed);
+        }
+        else
+        {
+            BOOST_ASSERT(reverse_data.reversed);
+        }
+
+        BOOST_ASSERT(forward_data.edge_id != SPECIAL_NODEID ||
+                     reverse_data.edge_id != SPECIAL_NODEID);
+
+        m_edge_based_node_list.emplace_back(
+            forward_data.edge_id, reverse_data.edge_id, node_u, node_v,
+            forward_data.name_id, forward_data.distance, reverse_data.distance, 0, 0, SPECIAL_EDGEID,
+            false, INVALID_COMPONENTID, 0, forward_data.travel_mode, reverse_data.travel_mode);
+        m_edge_based_node_is_startpoint.push_back(forward_data.startpoint || reverse_data.startpoint);
+        BOOST_ASSERT(!m_edge_based_node_list.back().IsCompressed());
+    }
+}
+
+void EdgeBasedGraphFactory::FlushVectorToStream(
+    std::ofstream &edge_data_file, std::vector<OriginalEdgeData> &original_edge_data_vector) const
+{
+    if (original_edge_data_vector.empty())
+    {
+        return;
+    }
+    edge_data_file.write((char *)&(original_edge_data_vector[0]),
+                         original_edge_data_vector.size() * sizeof(OriginalEdgeData));
+    original_edge_data_vector.clear();
+}
+
+#ifdef DEBUG_GEOMETRY
+void EdgeBasedGraphFactory::Run(const std::string &original_edge_data_filename,
+                                lua_State *lua_state,
+                                const std::string &edge_segment_lookup_filename,
+                                const std::string &edge_penalty_filename,
+                                const bool generate_edge_lookup,
+                                const std::string &debug_turns_path)
+#else
+void EdgeBasedGraphFactory::Run(const std::string &original_edge_data_filename,
+                                lua_State *lua_state,
+                                const std::string &edge_segment_lookup_filename,
+                                const std::string &edge_penalty_filename,
+                                const bool generate_edge_lookup)
+#endif
+{
+    TIMER_START(renumber);
+    m_max_edge_id = RenumberEdges() - 1;
+    TIMER_STOP(renumber);
+
+    TIMER_START(generate_nodes);
+    GenerateEdgeExpandedNodes();
+    TIMER_STOP(generate_nodes);
+
+    TIMER_START(generate_edges);
+#ifdef DEBUG_GEOMETRY
+    GenerateEdgeExpandedEdges(original_edge_data_filename, lua_state,
+            edge_segment_lookup_filename,edge_penalty_filename, 
+            generate_edge_lookup, debug_turns_path);
+#else
+    GenerateEdgeExpandedEdges(original_edge_data_filename, lua_state,
+            edge_segment_lookup_filename,edge_penalty_filename, 
+            generate_edge_lookup);
+#endif
+
+    TIMER_STOP(generate_edges);
+
+    SimpleLogger().Write() << "Timing statistics for edge-expanded graph:";
+    SimpleLogger().Write() << "Renumbering edges: " << TIMER_SEC(renumber) << "s";
+    SimpleLogger().Write() << "Generating nodes: " << TIMER_SEC(generate_nodes) << "s";
+    SimpleLogger().Write() << "Generating edges: " << TIMER_SEC(generate_edges) << "s";
+}
+
+
+/// Renumbers all _forward_ edges and sets the edge_id.
+/// A specific numbering is not important. Any unique ID will do.
+/// Returns the number of edge based nodes.
+unsigned EdgeBasedGraphFactory::RenumberEdges()
+{
+    // renumber edge based node of outgoing edges
+    unsigned numbered_edges_count = 0;
+    for (const auto current_node : osrm::irange(0u, m_node_based_graph->GetNumberOfNodes()))
+    {
+        for (const auto current_edge : m_node_based_graph->GetAdjacentEdgeRange(current_node))
+        {
+            EdgeData &edge_data = m_node_based_graph->GetEdgeData(current_edge);
+
+            // only number incoming edges
+            if (edge_data.reversed)
+            {
+                continue;
+            }
+
+            BOOST_ASSERT(numbered_edges_count < m_node_based_graph->GetNumberOfEdges());
+            edge_data.edge_id = numbered_edges_count;
+            ++numbered_edges_count;
+
+            BOOST_ASSERT(SPECIAL_NODEID != edge_data.edge_id);
+        }
+    }
+
+    return numbered_edges_count;
+}
+
+/// Creates the nodes in the edge expanded graph from edges in the node-based graph.
+void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes()
+{
+    Percent progress(m_node_based_graph->GetNumberOfNodes());
+
+    // loop over all edges and generate new set of nodes
+    for (const auto node_u : osrm::irange(0u, m_node_based_graph->GetNumberOfNodes()))
+    {
+        BOOST_ASSERT(node_u != SPECIAL_NODEID);
+        BOOST_ASSERT(node_u < m_node_based_graph->GetNumberOfNodes());
+        progress.printStatus(node_u);
+        for (EdgeID e1 : m_node_based_graph->GetAdjacentEdgeRange(node_u))
+        {
+            const EdgeData &edge_data = m_node_based_graph->GetEdgeData(e1);
+            BOOST_ASSERT(e1 != SPECIAL_EDGEID);
+            const NodeID node_v = m_node_based_graph->GetTarget(e1);
+
+            BOOST_ASSERT(SPECIAL_NODEID != node_v);
+            // pick only every other edge, since we have every edge as an outgoing
+            // and incoming egde
+            if (node_u > node_v)
+            {
+                continue;
+            }
+
+            BOOST_ASSERT(node_u < node_v);
+
+            // if we found a non-forward edge reverse and try again
+            if (edge_data.edge_id == SPECIAL_NODEID)
+            {
+                InsertEdgeBasedNode(node_v, node_u);
+            }
+            else
+            {
+                InsertEdgeBasedNode(node_u, node_v);
+            }
+        }
+    }
+
+    BOOST_ASSERT(m_edge_based_node_list.size() == m_edge_based_node_is_startpoint.size());
+
+    SimpleLogger().Write() << "Generated " << m_edge_based_node_list.size()
+                           << " nodes in edge-expanded graph";
+}
+
+/// Actually it also generates OriginalEdgeData and serializes them...
+#ifdef DEBUG_GEOMETRY
+void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
+    const std::string &original_edge_data_filename, lua_State *lua_state,
+    const std::string &edge_segment_lookup_filename,
+    const std::string &edge_fixed_penalties_filename,
+    const bool generate_edge_lookup, 
+    const std::string &debug_turns_path)
+#else
+void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
+    const std::string &original_edge_data_filename, lua_State *lua_state,
+    const std::string &edge_segment_lookup_filename,
+    const std::string &edge_fixed_penalties_filename,
+    const bool generate_edge_lookup)
+#endif
+{
+    SimpleLogger().Write() << "generating edge-expanded edges";
+
+    unsigned node_based_edge_counter = 0;
+    unsigned original_edges_counter = 0;
+
+    std::ofstream edge_data_file(original_edge_data_filename.c_str(), std::ios::binary);
+    std::ofstream edge_segment_file;
+    std::ofstream edge_penalty_file;
+
+    if (generate_edge_lookup)
+    {
+        edge_segment_file.open(edge_segment_lookup_filename.c_str(), std::ios::binary);
+        edge_penalty_file.open(edge_fixed_penalties_filename.c_str(), std::ios::binary);
+    }
+
+    // writes a dummy value that is updated later
+    edge_data_file.write((char *)&original_edges_counter, sizeof(unsigned));
+
+    std::vector<OriginalEdgeData> original_edge_data_vector;
+    original_edge_data_vector.reserve(1024 * 1024);
+
+    // Loop over all turns and generate new set of edges.
+    // Three nested loop look super-linear, but we are dealing with a (kind of)
+    // linear number of turns only.
+    unsigned restricted_turns_counter = 0;
+    unsigned skipped_uturns_counter = 0;
+    unsigned skipped_barrier_turns_counter = 0;
+    unsigned compressed = 0;
+
+    Percent progress(m_node_based_graph->GetNumberOfNodes());
+
+#ifdef DEBUG_GEOMETRY
+    DEBUG_TURNS_START(debug_turns_path);
+#endif
+
+    for (const auto node_u : osrm::irange(0u, m_node_based_graph->GetNumberOfNodes()))
+    {
+        //progress.printStatus(node_u);
+        for (const EdgeID e1 : m_node_based_graph->GetAdjacentEdgeRange(node_u))
+        {
+            if (m_node_based_graph->GetEdgeData(e1).reversed)
+            {
+                continue;
+            }
+
+            ++node_based_edge_counter;
+            const NodeID node_v = m_node_based_graph->GetTarget(e1);
+            const NodeID only_restriction_to_node =
+                m_restriction_map->CheckForEmanatingIsOnlyTurn(node_u, node_v);
+            const bool is_barrier_node = m_barrier_nodes.find(node_v) != m_barrier_nodes.end();
+
+            for (const EdgeID e2 : m_node_based_graph->GetAdjacentEdgeRange(node_v))
+            {
+                if (m_node_based_graph->GetEdgeData(e2).reversed)
+                {
+                    continue;
+                }
+                const NodeID node_w = m_node_based_graph->GetTarget(e2);
+
+                if ((only_restriction_to_node != SPECIAL_NODEID) &&
+                    (node_w != only_restriction_to_node))
+                {
+                    // We are at an only_-restriction but not at the right turn.
+                    ++restricted_turns_counter;
+                    continue;
+                }
+
+                if (is_barrier_node)
+                {
+                    if (node_u != node_w)
+                    {
+                        ++skipped_barrier_turns_counter;
+                        continue;
+                    }
+                }
+                else
+                {
+                    if ((node_u == node_w) && (m_node_based_graph->GetOutDegree(node_v) > 1))
+                    {
+                        ++skipped_uturns_counter;
+                        continue;
+                    }
+                }
+
+                // only add an edge if turn is not a U-turn except when it is
+                // at the end of a dead-end street
+                if (m_restriction_map->CheckIfTurnIsRestricted(node_u, node_v, node_w) &&
+                    (only_restriction_to_node == SPECIAL_NODEID) &&
+                    (node_w != only_restriction_to_node))
+                {
+                    // We are at an only_-restriction but not at the right turn.
+                    ++restricted_turns_counter;
+                    continue;
+                }
+
+                // only add an edge if turn is not prohibited
+                const EdgeData &edge_data1 = m_node_based_graph->GetEdgeData(e1);
+                const EdgeData &edge_data2 = m_node_based_graph->GetEdgeData(e2);
+
+                BOOST_ASSERT(edge_data1.edge_id != edge_data2.edge_id);
+                BOOST_ASSERT(!edge_data1.reversed);
+                BOOST_ASSERT(!edge_data2.reversed);
+
+                // the following is the core of the loop.
+                unsigned distance = edge_data1.distance;
+                if (m_traffic_lights.find(node_v) != m_traffic_lights.end())
+                {
+                    distance += speed_profile.traffic_signal_penalty;
+
+                    DEBUG_SIGNAL(node_v, m_node_info_list, speed_profile.traffic_signal_penalty);
+                }
+
+                // unpack last node of first segment if packed
+                const auto first_coordinate =
+                    m_node_info_list[(m_compressed_edge_container.HasEntryForID(e1)
+                                          ? m_compressed_edge_container.GetLastEdgeSourceID(e1)
+                                          : node_u)];
+
+                // unpack first node of second segment if packed
+                const auto third_coordinate =
+                    m_node_info_list[(m_compressed_edge_container.HasEntryForID(e2)
+                                          ? m_compressed_edge_container.GetFirstEdgeTargetID(e2)
+                                          : node_w)];
+
+                const double turn_angle = ComputeAngle::OfThreeFixedPointCoordinates(
+                    first_coordinate, m_node_info_list[node_v], third_coordinate);
+
+                const int turn_penalty = GetTurnPenalty(turn_angle, lua_state);
+                TurnInstruction turn_instruction = AnalyzeTurn(node_u, node_v, node_w, turn_angle);
+                if (turn_instruction == TurnInstruction::UTurn)
+                {
+                    distance += speed_profile.u_turn_penalty;
+
+                    DEBUG_UTURN(node_v, m_node_info_list, speed_profile.u_turn_penalty);
+                } 
+
+                DEBUG_TURN(node_v, m_node_info_list, first_coordinate, turn_angle, turn_penalty);
+
+                distance += turn_penalty;
+
+                const bool edge_is_compressed = m_compressed_edge_container.HasEntryForID(e1);
+
+                if (edge_is_compressed)
+                {
+                    ++compressed;
+                }
+
+                original_edge_data_vector.emplace_back(
+                    (edge_is_compressed ? m_compressed_edge_container.GetPositionForID(e1) : node_v),
+                    edge_data1.name_id, turn_instruction, edge_is_compressed,
+                    edge_data2.travel_mode);
+
+                ++original_edges_counter;
+
+                if (original_edge_data_vector.size() > 1024 * 1024 * 10)
+                {
+                    FlushVectorToStream(edge_data_file, original_edge_data_vector);
+                }
+
+                BOOST_ASSERT(SPECIAL_NODEID != edge_data1.edge_id);
+                BOOST_ASSERT(SPECIAL_NODEID != edge_data2.edge_id);
+
+
+                // NOTE: potential overflow here if we hit 2^32 routable edges
+                BOOST_ASSERT(m_edge_based_edge_list.size() <= std::numeric_limits<NodeID>::max());
+                m_edge_based_edge_list.emplace_back(edge_data1.edge_id, edge_data2.edge_id,
+                                  m_edge_based_edge_list.size(), distance, true, false);
+
+
+                // Here is where we write out the mapping between the edge-expanded edges, and
+                // the node-based edges that are originally used to calculate the `distance`
+                // for the edge-expanded edges.  About 40 lines back, there is:
+                //
+                //                 unsigned distance = edge_data1.distance;
+                //
+                // This tells us that the weight for an edge-expanded-edge is based on the weight
+                // of the *source* node-based edge.  Therefore, we will look up the individual
+                // segments of the source node-based edge, and write out a mapping between
+                // those and the edge-based-edge ID.
+                // External programs can then use this mapping to quickly perform
+                // updates to the edge-expanded-edge based directly on its ID.
+                if (generate_edge_lookup)
+                {
+                    unsigned fixed_penalty = distance - edge_data1.distance;
+                    edge_penalty_file.write(reinterpret_cast<const char *>(&fixed_penalty), sizeof(fixed_penalty));
+                    if (edge_is_compressed)
+                    {
+                        const auto node_based_edges = m_compressed_edge_container.GetBucketReference(e1);
+                        NodeID previous = node_u;
+
+                        const unsigned node_count = node_based_edges.size()+1;
+                        edge_segment_file.write(reinterpret_cast<const char *>(&node_count), sizeof(node_count));
+                        const QueryNode &first_node = m_node_info_list[previous];
+                        edge_segment_file.write(reinterpret_cast<const char *>(&first_node.node_id), sizeof(first_node.node_id));
+
+                        for (auto target_node : node_based_edges)
+                        {
+                            const QueryNode &from = m_node_info_list[previous];
+                            const QueryNode &to = m_node_info_list[target_node.first];
+                            const double segment_length = coordinate_calculation::great_circle_distance(from.lat, from.lon, to.lat, to.lon);
+
+                            edge_segment_file.write(reinterpret_cast<const char *>(&to.node_id), sizeof(to.node_id));
+                            edge_segment_file.write(reinterpret_cast<const char *>(&segment_length), sizeof(segment_length));
+                            edge_segment_file.write(reinterpret_cast<const char *>(&target_node.second), sizeof(target_node.second));
+                            previous = target_node.first;
+                        }
+                    }
+                    else
+                    {
+                        static const unsigned node_count = 2;
+                        const QueryNode from = m_node_info_list[node_u];
+                        const QueryNode to = m_node_info_list[node_v];
+                        const double segment_length = coordinate_calculation::great_circle_distance(from.lat, from.lon, to.lat, to.lon);
+                        edge_segment_file.write(reinterpret_cast<const char *>(&node_count), sizeof(node_count));
+                        edge_segment_file.write(reinterpret_cast<const char *>(&from.node_id), sizeof(from.node_id));
+                        edge_segment_file.write(reinterpret_cast<const char *>(&to.node_id), sizeof(to.node_id));
+                        edge_segment_file.write(reinterpret_cast<const char *>(&segment_length), sizeof(segment_length));
+                        edge_segment_file.write(reinterpret_cast<const char *>(&edge_data1.distance), sizeof(edge_data1.distance));
+                    }
+                }
+            }
+        }
+    }
+
+    DEBUG_TURNS_STOP();
+
+    FlushVectorToStream(edge_data_file, original_edge_data_vector);
+
+    edge_data_file.seekp(std::ios::beg);
+    edge_data_file.write((char *)&original_edges_counter, sizeof(unsigned));
+    edge_data_file.close();
+
+    SimpleLogger().Write() << "Generated " << m_edge_based_node_list.size() << " edge based nodes";
+    SimpleLogger().Write() << "Node-based graph contains " << node_based_edge_counter << " edges";
+    SimpleLogger().Write() << "Edge-expanded graph ...";
+    SimpleLogger().Write() << "  contains " << m_edge_based_edge_list.size() << " edges";
+    SimpleLogger().Write() << "  skips " << restricted_turns_counter << " turns, "
+                                                                        "defined by "
+                           << m_restriction_map->size() << " restrictions";
+    SimpleLogger().Write() << "  skips " << skipped_uturns_counter << " U turns";
+    SimpleLogger().Write() << "  skips " << skipped_barrier_turns_counter << " turns over barriers";
+}
+
+int EdgeBasedGraphFactory::GetTurnPenalty(double angle, lua_State *lua_state) const
+{
+
+    if (speed_profile.has_turn_penalty_function)
+    {
+        try
+        {
+            // call lua profile to compute turn penalty
+            double penalty = luabind::call_function<double>(lua_state, "turn_function", 180. - angle);
+            return static_cast<int>(penalty);
+        }
+        catch (const luabind::error &er)
+        {
+            SimpleLogger().Write(logWARNING) << er.what();
+        }
+    }
+    return 0;
+}
+
+TurnInstruction EdgeBasedGraphFactory::AnalyzeTurn(const NodeID node_u,
+                                                   const NodeID node_v,
+                                                   const NodeID node_w,
+                                                   const double angle) const
+{
+    if (node_u == node_w)
+    {
+        return TurnInstruction::UTurn;
+    }
+
+    const EdgeID edge1 = m_node_based_graph->FindEdge(node_u, node_v);
+    const EdgeID edge2 = m_node_based_graph->FindEdge(node_v, node_w);
+
+    const EdgeData &data1 = m_node_based_graph->GetEdgeData(edge1);
+    const EdgeData &data2 = m_node_based_graph->GetEdgeData(edge2);
+
+    // roundabouts need to be handled explicitely
+    if (data1.roundabout && data2.roundabout)
+    {
+        // Is a turn possible? If yes, we stay on the roundabout!
+        if (1 == m_node_based_graph->GetDirectedOutDegree(node_v))
+        {
+            // No turn possible.
+            return TurnInstruction::NoTurn;
+        }
+        return TurnInstruction::StayOnRoundAbout;
+    }
+    // Does turn start or end on roundabout?
+    if (data1.roundabout || data2.roundabout)
+    {
+        // We are entering the roundabout
+        if ((!data1.roundabout) && data2.roundabout)
+        {
+            return TurnInstruction::EnterRoundAbout;
+        }
+        // We are leaving the roundabout
+        if (data1.roundabout && (!data2.roundabout))
+        {
+            return TurnInstruction::LeaveRoundAbout;
+        }
+    }
+
+    // If street names stay the same and if we are certain that it is not a
+    // a segment of a roundabout, we skip it.
+    if (data1.name_id == data2.name_id && data1.travel_mode == data2.travel_mode)
+    {
+        // TODO: Here we should also do a small graph exploration to check for
+        //      more complex situations
+        if (0 != data1.name_id || m_node_based_graph->GetOutDegree(node_v) <= 2)
+        {
+            return TurnInstruction::NoTurn;
+        }
+    }
+
+    return TurnInstructionsClass::GetTurnDirectionOfInstruction(angle);
+}
+

src/extractor/external_memory_node.cpp

@@ -0,0 +1,66 @@
+/*
+
+Copyright (c) 2014, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "external_memory_node.hpp"
+#include "query_node.hpp"
+
+#include <limits>
+
+ExternalMemoryNode::ExternalMemoryNode(
+    int lat, int lon, OSMNodeID node_id, bool barrier, bool traffic_lights)
+    : QueryNode(lat, lon, node_id), barrier(barrier), traffic_lights(traffic_lights)
+{
+}
+
+ExternalMemoryNode::ExternalMemoryNode() : barrier(false), traffic_lights(false) {}
+
+ExternalMemoryNode ExternalMemoryNode::min_value()
+{
+    return ExternalMemoryNode(0, 0, MIN_OSM_NODEID, false, false);
+}
+
+ExternalMemoryNode ExternalMemoryNode::max_value()
+{
+    return ExternalMemoryNode(std::numeric_limits<int>::max(), std::numeric_limits<int>::max(),
+                              MAX_OSM_NODEID, false, false);
+}
+
+bool ExternalMemoryNodeSTXXLCompare::operator()(const ExternalMemoryNode &left,
+                                                const ExternalMemoryNode &right) const
+{
+    return left.node_id < right.node_id;
+}
+
+ExternalMemoryNodeSTXXLCompare::value_type ExternalMemoryNodeSTXXLCompare::max_value()
+{
+    return ExternalMemoryNode::max_value();
+}
+
+ExternalMemoryNodeSTXXLCompare::value_type ExternalMemoryNodeSTXXLCompare::min_value()
+{
+    return ExternalMemoryNode::min_value();
+}

src/extractor/extraction_containers.cpp

@@ -0,0 +1,739 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "extraction_containers.hpp"
+#include "extraction_way.hpp"
+
+#include "../algorithms/coordinate_calculation.hpp"
+#include "../data_structures/node_id.hpp"
+#include "../data_structures/range_table.hpp"
+
+#include "../util/osrm_exception.hpp"
+#include "../util/simple_logger.hpp"
+#include "../util/timing_util.hpp"
+#include "../util/fingerprint.hpp"
+#include "../util/lua_util.hpp"
+
+#include <boost/assert.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
+#include <boost/ref.hpp>
+#include <boost/numeric/conversion/cast.hpp>
+
+#include <luabind/luabind.hpp>
+
+#include <stxxl/sort>
+
+#include <chrono>
+#include <limits>
+
+static const int WRITE_BLOCK_BUFFER_SIZE = 8000;
+
+ExtractionContainers::ExtractionContainers()
+{
+    // Check if stxxl can be instantiated
+    stxxl::vector<unsigned> dummy_vector;
+    // Insert the empty string, it has no data and is zero length
+    name_lengths.push_back(0);
+}
+
+ExtractionContainers::~ExtractionContainers()
+{
+    // FIXME isn't this done implicitly of the stxxl::vectors go out of scope?
+    used_node_id_list.clear();
+    all_nodes_list.clear();
+    all_edges_list.clear();
+    name_char_data.clear();
+    name_lengths.clear();
+    restrictions_list.clear();
+    way_start_end_id_list.clear();
+}
+
+/**
+ * Processes the collected data and serializes it.
+ * At this point nodes are still referenced by their OSM id.
+ *
+ * - map start-end nodes of ways to ways used int restrictions to compute compressed
+ *   trippe representation
+ * - filter nodes list to nodes that are referenced by ways
+ * - merge edges with nodes to include location of start/end points and serialize
+ *
+ */
+void ExtractionContainers::PrepareData(const std::string &output_file_name,
+                                       const std::string &restrictions_file_name,
+                                       const std::string &name_file_name,
+                                       lua_State *segment_state)
+{
+    try
+    {
+        std::ofstream file_out_stream;
+        file_out_stream.open(output_file_name.c_str(), std::ios::binary);
+        const FingerPrint fingerprint = FingerPrint::GetValid();
+        file_out_stream.write((char *)&fingerprint, sizeof(FingerPrint));
+
+        PrepareNodes();
+        WriteNodes(file_out_stream);
+        PrepareEdges(segment_state);
+        WriteEdges(file_out_stream);
+
+        file_out_stream.close();
+
+        PrepareRestrictions();
+        WriteRestrictions(restrictions_file_name);
+
+        WriteNames(name_file_name);
+    }
+    catch (const std::exception &e)
+    {
+        std::cerr << "Caught Execption:" << e.what() << std::endl;
+    }
+}
+
+void ExtractionContainers::WriteNames(const std::string& names_file_name) const
+{
+    std::cout << "[extractor] writing street name index ... " << std::flush;
+    TIMER_START(write_name_index);
+    boost::filesystem::ofstream name_file_stream(names_file_name, std::ios::binary);
+
+    unsigned total_length = 0;
+
+    for (const unsigned &name_length : name_lengths)
+    {
+        total_length += name_length;
+    }
+
+    // builds and writes the index
+    RangeTable<> name_index_range(name_lengths);
+    name_file_stream << name_index_range;
+
+    name_file_stream.write((char *)&total_length, sizeof(unsigned));
+
+
+    // write all chars consecutively
+    char write_buffer[WRITE_BLOCK_BUFFER_SIZE];
+    unsigned buffer_len = 0;
+
+    for (const char &c : name_char_data)
+    {
+        write_buffer[buffer_len++] = c;
+
+        if (buffer_len >= WRITE_BLOCK_BUFFER_SIZE)
+        {
+            name_file_stream.write(write_buffer, WRITE_BLOCK_BUFFER_SIZE);
+            buffer_len = 0;
+        }
+    }
+
+    name_file_stream.write(write_buffer, buffer_len);
+
+    name_file_stream.close();
+    TIMER_STOP(write_name_index);
+    std::cout << "ok, after " << TIMER_SEC(write_name_index) << "s" << std::endl;
+}
+
+void ExtractionContainers::PrepareNodes()
+{
+    std::cout << "[extractor] Sorting used nodes        ... " << std::flush;
+    TIMER_START(sorting_used_nodes);
+    stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory);
+    TIMER_STOP(sorting_used_nodes);
+    std::cout << "ok, after " << TIMER_SEC(sorting_used_nodes) << "s" << std::endl;
+
+    std::cout << "[extractor] Erasing duplicate nodes   ... " << std::flush;
+    TIMER_START(erasing_dups);
+    auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end());
+    used_node_id_list.resize(new_end - used_node_id_list.begin());
+    TIMER_STOP(erasing_dups);
+    std::cout << "ok, after " << TIMER_SEC(erasing_dups) << "s" << std::endl;
+
+    std::cout << "[extractor] Sorting all nodes         ... " << std::flush;
+    TIMER_START(sorting_nodes);
+    stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), ExternalMemoryNodeSTXXLCompare(),
+                stxxl_memory);
+    TIMER_STOP(sorting_nodes);
+    std::cout << "ok, after " << TIMER_SEC(sorting_nodes) << "s" << std::endl;
+
+    std::cout << "[extractor] Building node id map      ... " << std::flush;
+    TIMER_START(id_map);
+    external_to_internal_node_id_map.reserve(used_node_id_list.size());
+    auto node_iter = all_nodes_list.begin();
+    auto ref_iter = used_node_id_list.begin();
+    const auto all_nodes_list_end = all_nodes_list.end();
+    const auto used_node_id_list_end = used_node_id_list.end();
+    // Note: despite being able to handle 64 bit OSM node ids, we can't
+    // handle > uint32_t actual usable nodes.  This should be OK for a while
+    // because we usually route on a *lot* less than 2^32 of the OSM
+    // graph nodes.
+    std::size_t internal_id = 0;
+
+    // compute the intersection of nodes that were referenced and nodes we actually have
+    while (node_iter != all_nodes_list_end && ref_iter != used_node_id_list_end)
+    {
+        if (node_iter->node_id < *ref_iter)
+        {
+            node_iter++;
+            continue;
+        }
+        if (node_iter->node_id > *ref_iter)
+        {
+            ref_iter++;
+            continue;
+        }
+        BOOST_ASSERT(node_iter->node_id == *ref_iter);
+        external_to_internal_node_id_map[*ref_iter] = static_cast<NodeID>(internal_id++);
+        node_iter++;
+        ref_iter++;
+    }
+    if (internal_id > std::numeric_limits<NodeID>::max())
+    {
+        throw osrm::exception("There are too many nodes remaining after filtering, OSRM only supports 2^32 unique nodes");
+    }
+    max_internal_node_id = boost::numeric_cast<NodeID>(internal_id);
+    TIMER_STOP(id_map);
+    std::cout << "ok, after " << TIMER_SEC(id_map) << "s" << std::endl;
+
+}
+
+void ExtractionContainers::PrepareEdges(lua_State *segment_state)
+{
+    // Sort edges by start.
+    std::cout << "[extractor] Sorting edges by start    ... " << std::flush;
+    TIMER_START(sort_edges_by_start);
+    stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByOSMStartID(), stxxl_memory);
+    TIMER_STOP(sort_edges_by_start);
+    std::cout << "ok, after " << TIMER_SEC(sort_edges_by_start) << "s" << std::endl;
+
+    std::cout << "[extractor] Setting start coords      ... " << std::flush;
+    TIMER_START(set_start_coords);
+    // Traverse list of edges and nodes in parallel and set start coord
+    auto node_iterator = all_nodes_list.begin();
+    auto edge_iterator = all_edges_list.begin();
+
+    const auto all_edges_list_end = all_edges_list.end();
+    const auto all_nodes_list_end = all_nodes_list.end();
+
+    while (edge_iterator != all_edges_list_end && node_iterator != all_nodes_list_end)
+    {
+        if (edge_iterator->result.osm_source_id < node_iterator->node_id)
+        {
+            SimpleLogger().Write(LogLevel::logWARNING) << "Found invalid node reference " << edge_iterator->result.source;
+            edge_iterator->result.source = SPECIAL_NODEID;
+            ++edge_iterator;
+            continue;
+        }
+        if (edge_iterator->result.osm_source_id > node_iterator->node_id)
+        {
+            node_iterator++;
+            continue;
+        }
+
+        // remove loops
+        if (edge_iterator->result.osm_source_id == edge_iterator->result.osm_target_id)
+        {
+            edge_iterator->result.source = SPECIAL_NODEID;
+            edge_iterator->result.target = SPECIAL_NODEID;
+            ++edge_iterator;
+            continue;
+        }
+
+        BOOST_ASSERT(edge_iterator->result.osm_source_id == node_iterator->node_id);
+
+        // assign new node id
+        auto id_iter = external_to_internal_node_id_map.find(node_iterator->node_id);
+        BOOST_ASSERT(id_iter != external_to_internal_node_id_map.end());
+        edge_iterator->result.source = id_iter->second;
+
+        edge_iterator->source_coordinate.lat = node_iterator->lat;
+        edge_iterator->source_coordinate.lon = node_iterator->lon;
+        ++edge_iterator;
+    }
+
+    // Remove all remaining edges. They are invalid because there are no corresponding nodes for
+    // them. This happens when using osmosis with bbox or polygon to extract smaller areas.
+    auto markSourcesInvalid = [](InternalExtractorEdge &edge)
+    {
+        SimpleLogger().Write(LogLevel::logWARNING) << "Found invalid node reference "
+                                                   << edge.result.source;
+        edge.result.source = SPECIAL_NODEID;
+        edge.result.osm_source_id = SPECIAL_OSM_NODEID;
+    };
+    std::for_each(edge_iterator, all_edges_list_end, markSourcesInvalid);
+    TIMER_STOP(set_start_coords);
+    std::cout << "ok, after " << TIMER_SEC(set_start_coords) << "s" << std::endl;
+
+    // Sort Edges by target
+    std::cout << "[extractor] Sorting edges by target   ... " << std::flush;
+    TIMER_START(sort_edges_by_target);
+    stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByOSMTargetID(),
+                stxxl_memory);
+    TIMER_STOP(sort_edges_by_target);
+    std::cout << "ok, after " << TIMER_SEC(sort_edges_by_target) << "s" << std::endl;
+
+    // Compute edge weights
+    std::cout << "[extractor] Computing edge weights    ... " << std::flush;
+    TIMER_START(compute_weights);
+    node_iterator = all_nodes_list.begin();
+    edge_iterator = all_edges_list.begin();
+    const auto all_edges_list_end_ = all_edges_list.end();
+    const auto all_nodes_list_end_ = all_nodes_list.end();
+
+    while (edge_iterator != all_edges_list_end_ && node_iterator != all_nodes_list_end_)
+    {
+        // skip all invalid edges
+        if (edge_iterator->result.source == SPECIAL_NODEID)
+        {
+            ++edge_iterator;
+            continue;
+        }
+
+        if (edge_iterator->result.osm_target_id < node_iterator->node_id)
+        {
+            SimpleLogger().Write(LogLevel::logWARNING) << "Found invalid node reference " << OSMNodeID_to_uint64_t(edge_iterator->result.osm_target_id);
+            edge_iterator->result.target = SPECIAL_NODEID;
+            ++edge_iterator;
+            continue;
+        }
+        if (edge_iterator->result.osm_target_id > node_iterator->node_id)
+        {
+            ++node_iterator;
+            continue;
+        }
+
+        BOOST_ASSERT(edge_iterator->result.osm_target_id == node_iterator->node_id);
+        BOOST_ASSERT(edge_iterator->weight_data.speed >= 0);
+        BOOST_ASSERT(edge_iterator->source_coordinate.lat != std::numeric_limits<int>::min());
+        BOOST_ASSERT(edge_iterator->source_coordinate.lon != std::numeric_limits<int>::min());
+
+        const double distance = coordinate_calculation::great_circle_distance(
+            edge_iterator->source_coordinate.lat, edge_iterator->source_coordinate.lon,
+            node_iterator->lat, node_iterator->lon);
+
+        if (lua_function_exists(segment_state, "segment_function"))
+        {
+            luabind::call_function<void>(
+                segment_state, "segment_function",
+                boost::cref(edge_iterator->source_coordinate),
+                boost::cref(*node_iterator),
+                distance,
+                boost::ref(edge_iterator->weight_data));
+        }
+
+        const double weight = [distance](const InternalExtractorEdge::WeightData& data) {
+            switch (data.type)
+            {
+                case InternalExtractorEdge::WeightType::EDGE_DURATION:
+                case InternalExtractorEdge::WeightType::WAY_DURATION:
+                    return data.duration * 10.;
+                    break;
+                case InternalExtractorEdge::WeightType::SPEED:
+                    return (distance * 10.) / (data.speed / 3.6);
+                    break;
+                case InternalExtractorEdge::WeightType::INVALID:
+                    osrm::exception("invalid weight type");
+            }
+            return -1.0;
+        }(edge_iterator->weight_data);
+
+        auto& edge = edge_iterator->result;
+        edge.weight = std::max(1, static_cast<int>(std::floor(weight + .5)));
+
+        // assign new node id
+        auto id_iter = external_to_internal_node_id_map.find(node_iterator->node_id);
+        BOOST_ASSERT(id_iter != external_to_internal_node_id_map.end());
+        edge.target = id_iter->second;
+
+        // orient edges consistently: source id < target id
+        // important for multi-edge removal
+        if (edge.source > edge.target)
+        {
+            std::swap(edge.source, edge.target);
+
+            // std::swap does not work with bit-fields
+            bool temp = edge.forward;
+            edge.forward = edge.backward;
+            edge.backward = temp;
+        }
+        ++edge_iterator;
+    }
+
+    // Remove all remaining edges. They are invalid because there are no corresponding nodes for
+    // them. This happens when using osmosis with bbox or polygon to extract smaller areas.
+    auto markTargetsInvalid = [](InternalExtractorEdge &edge)
+    {
+        SimpleLogger().Write(LogLevel::logWARNING) << "Found invalid node reference "
+                                                   << edge.result.target;
+        edge.result.target = SPECIAL_NODEID;
+    };
+    std::for_each(edge_iterator, all_edges_list_end_, markTargetsInvalid);
+    TIMER_STOP(compute_weights);
+    std::cout << "ok, after " << TIMER_SEC(compute_weights) << "s" << std::endl;
+
+    // Sort edges by start.
+    std::cout << "[extractor] Sorting edges by renumbered start ... " << std::flush;
+    TIMER_START(sort_edges_by_renumbered_start);
+    stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByInternalStartThenInternalTargetID(), stxxl_memory);
+    TIMER_STOP(sort_edges_by_renumbered_start);
+    std::cout << "ok, after " << TIMER_SEC(sort_edges_by_renumbered_start) << "s" << std::endl;
+
+    BOOST_ASSERT(all_edges_list.size() > 0);
+    for (unsigned i = 0; i < all_edges_list.size();)
+    {
+        // only invalid edges left
+        if (all_edges_list[i].result.source == SPECIAL_NODEID)
+        {
+            break;
+        }
+        // skip invalid edges
+        if (all_edges_list[i].result.target == SPECIAL_NODEID)
+        {
+            ++i;
+            continue;
+        }
+
+        unsigned start_idx = i;
+        NodeID source = all_edges_list[i].result.source;
+        NodeID target = all_edges_list[i].result.target;
+
+        int min_forward_weight = std::numeric_limits<int>::max();
+        int min_backward_weight = std::numeric_limits<int>::max();
+        unsigned min_forward_idx = std::numeric_limits<unsigned>::max();
+        unsigned min_backward_idx = std::numeric_limits<unsigned>::max();
+
+        // find minimal edge in both directions
+        while (all_edges_list[i].result.source == source &&
+               all_edges_list[i].result.target == target)
+        {
+            if (all_edges_list[i].result.forward && all_edges_list[i].result.weight < min_forward_weight)
+            {
+                min_forward_idx = i;
+            }
+            if (all_edges_list[i].result.backward && all_edges_list[i].result.weight < min_backward_weight)
+            {
+                min_backward_idx = i;
+            }
+
+            // this also increments the outer loop counter!
+            i++;
+        }
+
+        BOOST_ASSERT(min_forward_idx == std::numeric_limits<unsigned>::max() || min_forward_idx < i);
+        BOOST_ASSERT(min_backward_idx == std::numeric_limits<unsigned>::max() || min_backward_idx < i);
+        BOOST_ASSERT(min_backward_idx != std::numeric_limits<unsigned>::max() ||
+                     min_forward_idx != std::numeric_limits<unsigned>::max());
+
+        if (min_backward_idx == min_forward_idx)
+        {
+            all_edges_list[min_forward_idx].result.is_split = false;
+            all_edges_list[min_forward_idx].result.forward = true;
+            all_edges_list[min_forward_idx].result.backward = true;
+        }
+        else
+        {
+            bool has_forward = min_forward_idx != std::numeric_limits<unsigned>::max();
+            bool has_backward = min_backward_idx != std::numeric_limits<unsigned>::max();
+            if (has_forward)
+            {
+                all_edges_list[min_forward_idx].result.forward = true;
+                all_edges_list[min_forward_idx].result.backward = false;
+                all_edges_list[min_forward_idx].result.is_split = has_backward;
+            }
+            if (has_backward)
+            {
+                std::swap(all_edges_list[min_backward_idx].result.source,
+                          all_edges_list[min_backward_idx].result.target);
+                all_edges_list[min_backward_idx].result.forward = true;
+                all_edges_list[min_backward_idx].result.backward = false;
+                all_edges_list[min_backward_idx].result.is_split = has_forward;
+            }
+        }
+
+        // invalidate all unused edges
+        for (unsigned j = start_idx; j < i; j++)
+        {
+            if (j == min_forward_idx || j == min_backward_idx)
+            {
+                continue;
+            }
+            all_edges_list[j].result.source = SPECIAL_NODEID;
+            all_edges_list[j].result.target = SPECIAL_NODEID;
+        }
+    }
+}
+
+void ExtractionContainers::WriteEdges(std::ofstream& file_out_stream) const
+{
+    std::cout << "[extractor] Writing used edges       ... " << std::flush;
+    TIMER_START(write_edges);
+    // Traverse list of edges and nodes in parallel and set target coord
+    std::size_t used_edges_counter = 0;
+    unsigned used_edges_counter_buffer = 0;
+
+    auto start_position = file_out_stream.tellp();
+    file_out_stream.write((char *)&used_edges_counter_buffer, sizeof(unsigned));
+
+    for (const auto& edge : all_edges_list)
+    {
+        if (edge.result.source == SPECIAL_NODEID || edge.result.target == SPECIAL_NODEID)
+        {
+            continue;
+        }
+
+        // IMPORTANT: here, we're using slicing to only write the data from the base
+        // class of NodeBasedEdgeWithOSM
+        NodeBasedEdge tmp = edge.result;
+        file_out_stream.write((char*) &tmp, sizeof(NodeBasedEdge));
+        used_edges_counter++;
+    }
+
+    if (used_edges_counter > std::numeric_limits<unsigned>::max())
+    {
+        throw osrm::exception("There are too many edges, OSRM only supports 2^32");
+    }
+    TIMER_STOP(write_edges);
+    std::cout << "ok, after " << TIMER_SEC(write_edges) << "s" << std::endl;
+
+    std::cout << "[extractor] setting number of edges   ... " << std::flush;
+
+    used_edges_counter_buffer = boost::numeric_cast<unsigned>(used_edges_counter);
+
+    file_out_stream.seekp(start_position);
+    file_out_stream.write((char *)&used_edges_counter_buffer, sizeof(unsigned));
+    std::cout << "ok" << std::endl;
+
+    SimpleLogger().Write() << "Processed " << used_edges_counter << " edges";
+}
+
+void ExtractionContainers::WriteNodes(std::ofstream& file_out_stream) const
+{
+    // write dummy value, will be overwritten later
+    std::cout << "[extractor] setting number of nodes   ... " << std::flush;
+    file_out_stream.write((char *)&max_internal_node_id, sizeof(unsigned));
+    std::cout << "ok" << std::endl;
+
+    std::cout << "[extractor] Confirming/Writing used nodes     ... " << std::flush;
+    TIMER_START(write_nodes);
+    // identify all used nodes by a merging step of two sorted lists
+    auto node_iterator = all_nodes_list.begin();
+    auto node_id_iterator = used_node_id_list.begin();
+    const auto used_node_id_list_end = used_node_id_list.end();
+    const auto all_nodes_list_end = all_nodes_list.end();
+
+    while (node_id_iterator != used_node_id_list_end && node_iterator != all_nodes_list_end)
+    {
+        if (*node_id_iterator < node_iterator->node_id)
+        {
+            ++node_id_iterator;
+            continue;
+        }
+        if (*node_id_iterator > node_iterator->node_id)
+        {
+            ++node_iterator;
+            continue;
+        }
+        BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
+
+        file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode));
+
+        ++node_id_iterator;
+        ++node_iterator;
+    }
+    TIMER_STOP(write_nodes);
+    std::cout << "ok, after " << TIMER_SEC(write_nodes) << "s" << std::endl;
+
+
+    SimpleLogger().Write() << "Processed " << max_internal_node_id << " nodes";
+}
+
+void ExtractionContainers::WriteRestrictions(const std::string& path) const
+{
+    // serialize restrictions
+    std::ofstream restrictions_out_stream;
+    unsigned written_restriction_count = 0;
+    restrictions_out_stream.open(path.c_str(), std::ios::binary);
+    const FingerPrint fingerprint = FingerPrint::GetValid();
+    restrictions_out_stream.write((char *)&fingerprint, sizeof(FingerPrint));
+    const auto count_position = restrictions_out_stream.tellp();
+    restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned));
+
+    for (const auto &restriction_container : restrictions_list)
+    {
+        if (SPECIAL_NODEID != restriction_container.restriction.from.node &&
+            SPECIAL_NODEID != restriction_container.restriction.via.node &&
+            SPECIAL_NODEID != restriction_container.restriction.to.node)
+        {
+            restrictions_out_stream.write((char *)&(restriction_container.restriction),
+                                          sizeof(TurnRestriction));
+            ++written_restriction_count;
+        }
+    }
+    restrictions_out_stream.seekp(count_position);
+    restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned));
+    restrictions_out_stream.close();
+    SimpleLogger().Write() << "usable restrictions: " << written_restriction_count;
+}
+
+void ExtractionContainers::PrepareRestrictions()
+{
+    std::cout << "[extractor] Sorting used ways         ... " << std::flush;
+    TIMER_START(sort_ways);
+    stxxl::sort(way_start_end_id_list.begin(), way_start_end_id_list.end(),
+                FirstAndLastSegmentOfWayStxxlCompare(), stxxl_memory);
+    TIMER_STOP(sort_ways);
+    std::cout << "ok, after " << TIMER_SEC(sort_ways) << "s" << std::endl;
+
+    std::cout << "[extractor] Sorting " << restrictions_list.size()
+              << " restriction. by from... " << std::flush;
+    TIMER_START(sort_restrictions);
+    stxxl::sort(restrictions_list.begin(), restrictions_list.end(),
+                CmpRestrictionContainerByFrom(), stxxl_memory);
+    TIMER_STOP(sort_restrictions);
+    std::cout << "ok, after " << TIMER_SEC(sort_restrictions) << "s" << std::endl;
+
+    std::cout << "[extractor] Fixing restriction starts ... " << std::flush;
+    TIMER_START(fix_restriction_starts);
+    auto restrictions_iterator = restrictions_list.begin();
+    auto way_start_and_end_iterator = way_start_end_id_list.cbegin();
+    const auto restrictions_list_end = restrictions_list.end();
+    const auto way_start_end_id_list_end = way_start_end_id_list.cend();
+
+    while (way_start_and_end_iterator != way_start_end_id_list_end &&
+           restrictions_iterator != restrictions_list_end)
+    {
+        if (way_start_and_end_iterator->way_id < OSMWayID(restrictions_iterator->restriction.from.way))
+        {
+            ++way_start_and_end_iterator;
+            continue;
+        }
+
+        if (way_start_and_end_iterator->way_id > OSMWayID(restrictions_iterator->restriction.from.way))
+        {
+            SimpleLogger().Write(LogLevel::logDEBUG) << "Restriction references invalid way: " << restrictions_iterator->restriction.from.way;
+            restrictions_iterator->restriction.from.node = SPECIAL_NODEID;
+            ++restrictions_iterator;
+            continue;
+        }
+
+        BOOST_ASSERT(way_start_and_end_iterator->way_id ==
+                     OSMWayID(restrictions_iterator->restriction.from.way));
+        // we do not remap the via id yet, since we will need it for the to node as well
+        const OSMNodeID via_node_id = OSMNodeID(restrictions_iterator->restriction.via.node);
+
+        // check if via is actually valid, if not invalidate
+        auto via_id_iter = external_to_internal_node_id_map.find(via_node_id);
+        if(via_id_iter == external_to_internal_node_id_map.end())
+        {
+            SimpleLogger().Write(LogLevel::logDEBUG) << "Restriction references invalid node: " << restrictions_iterator->restriction.via.node;
+            restrictions_iterator->restriction.via.node = SPECIAL_NODEID;
+            ++restrictions_iterator;
+            continue;
+        }
+
+        if (OSMNodeID(way_start_and_end_iterator->first_segment_source_id) == via_node_id)
+        {
+            // assign new from node id
+            auto id_iter = external_to_internal_node_id_map.find(
+                    OSMNodeID(way_start_and_end_iterator->first_segment_target_id));
+            BOOST_ASSERT(id_iter != external_to_internal_node_id_map.end());
+            restrictions_iterator->restriction.from.node = id_iter->second;
+        }
+        else if (OSMNodeID(way_start_and_end_iterator->last_segment_target_id) == via_node_id)
+        {
+            // assign new from node id
+            auto id_iter = external_to_internal_node_id_map.find(
+                    OSMNodeID(way_start_and_end_iterator->last_segment_source_id));
+            BOOST_ASSERT(id_iter != external_to_internal_node_id_map.end());
+            restrictions_iterator->restriction.from.node = id_iter->second;
+        }
+        ++restrictions_iterator;
+    }
+
+    TIMER_STOP(fix_restriction_starts);
+    std::cout << "ok, after " << TIMER_SEC(fix_restriction_starts) << "s" << std::endl;
+
+    std::cout << "[extractor] Sorting restrictions. by to  ... " << std::flush;
+    TIMER_START(sort_restrictions_to);
+    stxxl::sort(restrictions_list.begin(), restrictions_list.end(),
+                CmpRestrictionContainerByTo(), stxxl_memory);
+    TIMER_STOP(sort_restrictions_to);
+    std::cout << "ok, after " << TIMER_SEC(sort_restrictions_to) << "s" << std::endl;
+
+    std::cout << "[extractor] Fixing restriction ends   ... " << std::flush;
+    TIMER_START(fix_restriction_ends);
+    restrictions_iterator = restrictions_list.begin();
+    way_start_and_end_iterator = way_start_end_id_list.cbegin();
+    const auto way_start_end_id_list_end_ = way_start_end_id_list.cend();
+    const auto restrictions_list_end_ = restrictions_list.end();
+
+    while (way_start_and_end_iterator != way_start_end_id_list_end_ &&
+           restrictions_iterator != restrictions_list_end_)
+    {
+        if (way_start_and_end_iterator->way_id < OSMWayID(restrictions_iterator->restriction.to.way))
+        {
+            ++way_start_and_end_iterator;
+            continue;
+        }
+        if (restrictions_iterator->restriction.from.node == SPECIAL_NODEID ||
+            restrictions_iterator->restriction.via.node == SPECIAL_NODEID)
+        {
+            ++restrictions_iterator;
+            continue;
+        }
+        if (way_start_and_end_iterator->way_id > OSMWayID(restrictions_iterator->restriction.to.way))
+        {
+            SimpleLogger().Write(LogLevel::logDEBUG) << "Restriction references invalid way: " << restrictions_iterator->restriction.to.way;
+            restrictions_iterator->restriction.to.way = SPECIAL_NODEID;
+            ++restrictions_iterator;
+            continue;
+        }
+        BOOST_ASSERT(way_start_and_end_iterator->way_id ==
+                     OSMWayID(restrictions_iterator->restriction.to.way));
+        const OSMNodeID via_node_id = OSMNodeID(restrictions_iterator->restriction.via.node);
+
+        // assign new via node id
+        auto via_id_iter = external_to_internal_node_id_map.find(via_node_id);
+        BOOST_ASSERT(via_id_iter != external_to_internal_node_id_map.end());
+        restrictions_iterator->restriction.via.node = via_id_iter->second;
+
+        if (OSMNodeID(way_start_and_end_iterator->first_segment_source_id) == via_node_id)
+        {
+            auto to_id_iter = external_to_internal_node_id_map.find(
+                    OSMNodeID(way_start_and_end_iterator->first_segment_target_id));
+            BOOST_ASSERT(to_id_iter != external_to_internal_node_id_map.end());
+            restrictions_iterator->restriction.to.node = to_id_iter->second;
+        }
+        else if (OSMNodeID(way_start_and_end_iterator->last_segment_target_id) == via_node_id)
+        {
+            auto to_id_iter = external_to_internal_node_id_map.find(
+                    OSMNodeID(way_start_and_end_iterator->last_segment_source_id));
+            BOOST_ASSERT(to_id_iter != external_to_internal_node_id_map.end());
+            restrictions_iterator->restriction.to.node = to_id_iter->second;
+        }
+        ++restrictions_iterator;
+    }
+    TIMER_STOP(fix_restriction_ends);
+    std::cout << "ok, after " << TIMER_SEC(fix_restriction_ends) << "s" << std::endl;
+}

src/extractor/extractor.cpp

@@ -0,0 +1,638 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "extractor.hpp"
+
+#include "extraction_containers.hpp"
+#include "extraction_node.hpp"
+#include "extraction_way.hpp"
+#include "extractor_callbacks.hpp"
+#include "restriction_parser.hpp"
+#include "scripting_environment.hpp"
+
+#include "../data_structures/raster_source.hpp"
+#include "../util/make_unique.hpp"
+#include "../util/simple_logger.hpp"
+#include "../util/timing_util.hpp"
+#include "../util/lua_util.hpp"
+#include "../util/graph_loader.hpp"
+
+#include "../typedefs.h"
+
+#include "../data_structures/static_graph.hpp"
+#include "../data_structures/static_rtree.hpp"
+#include "../data_structures/restriction_map.hpp"
+#include "../data_structures/compressed_edge_container.hpp"
+
+#include "../algorithms/tarjan_scc.hpp"
+#include "../algorithms/crc32_processor.hpp"
+
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
+#include <boost/optional/optional.hpp>
+
+#include <luabind/luabind.hpp>
+
+#include <osmium/io/any_input.hpp>
+
+#include <tbb/parallel_for.h>
+#include <tbb/task_scheduler_init.h>
+
+#include <cstdlib>
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <fstream>
+#include <iostream>
+#include <thread>
+#include <unordered_map>
+#include <vector>
+
+/**
+ * TODO: Refactor this function into smaller functions for better readability.
+ *
+ * This function is the entry point for the whole extraction process. The goal of the extraction
+ * step is to filter and convert the OSM geometry to something more fitting for routing.
+ * That includes:
+ *  - extracting turn restrictions
+ *  - splitting ways into (directional!) edge segments
+ *  - checking if nodes are barriers or traffic signal
+ *  - discarding all tag information: All relevant type information for nodes/ways
+ *    is extracted at this point.
+ *
+ * The result of this process are the following files:
+ *  .names : Names of all streets, stored as long consecutive string with prefix sum based index
+ *  .osrm  : Nodes and edges in a intermediate format that easy to digest for osrm-prepare
+ *  .restrictions : Turn restrictions that are used my osrm-prepare to construct the edge-expanded
+ * graph
+ *
+ */
+int extractor::run()
+{
+    try
+    {
+        LogPolicy::GetInstance().Unmute();
+        TIMER_START(extracting);
+
+        const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
+        const auto number_of_threads =
+            std::min(recommended_num_threads, config.requested_num_threads);
+        tbb::task_scheduler_init init(number_of_threads);
+
+        SimpleLogger().Write() << "Input file: " << config.input_path.filename().string();
+        SimpleLogger().Write() << "Profile: " << config.profile_path.filename().string();
+        SimpleLogger().Write() << "Threads: " << number_of_threads;
+
+        // setup scripting environment
+        ScriptingEnvironment scripting_environment(config.profile_path.string().c_str());
+
+        ExtractionContainers extraction_containers;
+        auto extractor_callbacks = osrm::make_unique<ExtractorCallbacks>(extraction_containers);
+
+        const osmium::io::File input_file(config.input_path.string());
+        osmium::io::Reader reader(input_file);
+        const osmium::io::Header header = reader.header();
+
+        std::atomic<unsigned> number_of_nodes{0};
+        std::atomic<unsigned> number_of_ways{0};
+        std::atomic<unsigned> number_of_relations{0};
+        std::atomic<unsigned> number_of_others{0};
+
+        SimpleLogger().Write() << "Parsing in progress..";
+        TIMER_START(parsing);
+
+        lua_State *segment_state = scripting_environment.get_lua_state();
+
+        if (lua_function_exists(segment_state, "source_function"))
+        {
+            // bind a single instance of SourceContainer class to relevant lua state
+            SourceContainer sources;
+            luabind::globals(segment_state)["sources"] = sources;
+
+            luabind::call_function<void>(segment_state, "source_function");
+        }
+
+        std::string generator = header.get("generator");
+        if (generator.empty())
+        {
+            generator = "unknown tool";
+        }
+        SimpleLogger().Write() << "input file generated by " << generator;
+
+        // write .timestamp data file
+        std::string timestamp = header.get("osmosis_replication_timestamp");
+        if (timestamp.empty())
+        {
+            timestamp = "n/a";
+        }
+        SimpleLogger().Write() << "timestamp: " << timestamp;
+
+        boost::filesystem::ofstream timestamp_out(config.timestamp_file_name);
+        timestamp_out.write(timestamp.c_str(), timestamp.length());
+        timestamp_out.close();
+
+        // initialize vectors holding parsed objects
+        tbb::concurrent_vector<std::pair<std::size_t, ExtractionNode>> resulting_nodes;
+        tbb::concurrent_vector<std::pair<std::size_t, ExtractionWay>> resulting_ways;
+        tbb::concurrent_vector<boost::optional<InputRestrictionContainer>> resulting_restrictions;
+
+        // setup restriction parser
+        const RestrictionParser restriction_parser(scripting_environment.get_lua_state());
+
+        while (const osmium::memory::Buffer buffer = reader.read())
+        {
+            // create a vector of iterators into the buffer
+            std::vector<osmium::memory::Buffer::const_iterator> osm_elements;
+            for (auto iter = std::begin(buffer), end = std::end(buffer); iter != end; ++iter)
+            {
+                osm_elements.push_back(iter);
+            }
+
+            // clear resulting vectors
+            resulting_nodes.clear();
+            resulting_ways.clear();
+            resulting_restrictions.clear();
+
+            // parse OSM entities in parallel, store in resulting vectors
+            tbb::parallel_for(
+                tbb::blocked_range<std::size_t>(0, osm_elements.size()),
+                [&](const tbb::blocked_range<std::size_t> &range)
+                {
+                    ExtractionNode result_node;
+                    ExtractionWay result_way;
+                    lua_State *local_state = scripting_environment.get_lua_state();
+
+                    for (auto x = range.begin(), end = range.end(); x != end; ++x)
+                    {
+                        const auto entity = osm_elements[x];
+
+                        switch (entity->type())
+                        {
+                        case osmium::item_type::node:
+                            result_node.clear();
+                            ++number_of_nodes;
+                            luabind::call_function<void>(
+                                local_state, "node_function",
+                                boost::cref(static_cast<const osmium::Node &>(*entity)),
+                                boost::ref(result_node));
+                            resulting_nodes.push_back(std::make_pair(x, result_node));
+                            break;
+                        case osmium::item_type::way:
+                            result_way.clear();
+                            ++number_of_ways;
+                            luabind::call_function<void>(
+                                local_state, "way_function",
+                                boost::cref(static_cast<const osmium::Way &>(*entity)),
+                                boost::ref(result_way));
+                            resulting_ways.push_back(std::make_pair(x, result_way));
+                            break;
+                        case osmium::item_type::relation:
+                            ++number_of_relations;
+                            resulting_restrictions.push_back(restriction_parser.TryParse(
+                                static_cast<const osmium::Relation &>(*entity)));
+                            break;
+                        default:
+                            ++number_of_others;
+                            break;
+                        }
+                    }
+                });
+
+            // put parsed objects thru extractor callbacks
+            for (const auto &result : resulting_nodes)
+            {
+                extractor_callbacks->ProcessNode(
+                    static_cast<const osmium::Node &>(*(osm_elements[result.first])),
+                    result.second);
+            }
+            for (const auto &result : resulting_ways)
+            {
+                extractor_callbacks->ProcessWay(
+                    static_cast<const osmium::Way &>(*(osm_elements[result.first])), result.second);
+            }
+            for (const auto &result : resulting_restrictions)
+            {
+                extractor_callbacks->ProcessRestriction(result);
+            }
+        }
+        TIMER_STOP(parsing);
+        SimpleLogger().Write() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds";
+
+        SimpleLogger().Write() << "Raw input contains " << number_of_nodes.load() << " nodes, "
+                               << number_of_ways.load() << " ways, and "
+                               << number_of_relations.load() << " relations, and "
+                               << number_of_others.load() << " unknown entities";
+
+        extractor_callbacks.reset();
+
+        if (extraction_containers.all_edges_list.empty())
+        {
+            SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
+            return 1;
+        }
+
+        extraction_containers.PrepareData(config.output_file_name, config.restriction_file_name,
+                                          config.names_file_name, segment_state);
+
+        TIMER_STOP(extracting);
+        SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s";
+    }
+    catch (const std::exception &e)
+    {
+        SimpleLogger().Write(logWARNING) << e.what();
+        return 1;
+    }
+
+    try
+    {
+        // Transform the node-based graph that OSM is based on into an edge-based graph
+        // that is better for routing.  Every edge becomes a node, and every valid
+        // movement (e.g. turn from A->B, and B->A) becomes an edge
+        //
+        //
+        //    // Create a new lua state
+
+        SimpleLogger().Write() << "Generating edge-expanded graph representation";
+
+        TIMER_START(expansion);
+
+        std::vector<EdgeBasedNode> node_based_edge_list;
+        DeallocatingVector<EdgeBasedEdge> edge_based_edge_list;
+        std::vector<bool> node_is_startpoint;
+        std::vector<QueryNode> internal_to_external_node_map;
+        auto graph_size =
+            BuildEdgeExpandedGraph(internal_to_external_node_map, node_based_edge_list,
+                                   node_is_startpoint, edge_based_edge_list);
+
+        auto number_of_node_based_nodes = graph_size.first;
+        auto max_edge_id = graph_size.second;
+
+        TIMER_STOP(expansion);
+
+        SimpleLogger().Write() << "building r-tree ...";
+        TIMER_START(rtree);
+
+        FindComponents(max_edge_id, edge_based_edge_list, node_based_edge_list);
+
+        BuildRTree(std::move(node_based_edge_list), std::move(node_is_startpoint),
+                   internal_to_external_node_map);
+
+        TIMER_STOP(rtree);
+
+        SimpleLogger().Write() << "writing node map ...";
+        WriteNodeMapping(internal_to_external_node_map);
+
+        WriteEdgeBasedGraph(config.edge_graph_output_path, max_edge_id, edge_based_edge_list);
+
+        SimpleLogger().Write() << "Expansion  : "
+                               << (number_of_node_based_nodes / TIMER_SEC(expansion))
+                               << " nodes/sec and " << ((max_edge_id + 1) / TIMER_SEC(expansion))
+                               << " edges/sec";
+        SimpleLogger().Write() << "To prepare the data for routing, run: "
+                               << "./osrm-prepare " << config.output_file_name << std::endl;
+    }
+    catch (const std::exception &e)
+    {
+        SimpleLogger().Write(logWARNING) << e.what();
+        return 1;
+    }
+
+    return 0;
+}
+
+/**
+    \brief Setups scripting environment (lua-scripting)
+    Also initializes speed profile.
+*/
+void extractor::SetupScriptingEnvironment(lua_State *lua_state,
+                                          SpeedProfileProperties &speed_profile)
+{
+    // open utility libraries string library;
+    luaL_openlibs(lua_state);
+
+    // adjust lua load path
+    luaAddScriptFolderToLoadPath(lua_state, config.profile_path.string().c_str());
+
+    // Now call our function in a lua script
+    if (0 != luaL_dofile(lua_state, config.profile_path.string().c_str()))
+    {
+        std::stringstream msg;
+        msg << lua_tostring(lua_state, -1) << " occurred in scripting block";
+        throw osrm::exception(msg.str());
+    }
+
+    if (0 != luaL_dostring(lua_state, "return traffic_signal_penalty\n"))
+    {
+        std::stringstream msg;
+        msg << lua_tostring(lua_state, -1) << " occurred in scripting block";
+        throw osrm::exception(msg.str());
+    }
+    speed_profile.traffic_signal_penalty = 10 * lua_tointeger(lua_state, -1);
+    SimpleLogger().Write(logDEBUG) << "traffic_signal_penalty: "
+                                   << speed_profile.traffic_signal_penalty;
+
+    if (0 != luaL_dostring(lua_state, "return u_turn_penalty\n"))
+    {
+        std::stringstream msg;
+        msg << lua_tostring(lua_state, -1) << " occurred in scripting block";
+        throw osrm::exception(msg.str());
+    }
+
+    speed_profile.u_turn_penalty = 10 * lua_tointeger(lua_state, -1);
+    speed_profile.has_turn_penalty_function = lua_function_exists(lua_state, "turn_function");
+}
+
+void extractor::FindComponents(unsigned max_edge_id,
+                               const DeallocatingVector<EdgeBasedEdge> &input_edge_list,
+                               std::vector<EdgeBasedNode> &input_nodes) const
+{
+    struct UncontractedEdgeData
+    {
+    };
+    struct InputEdge
+    {
+        unsigned source;
+        unsigned target;
+        UncontractedEdgeData data;
+
+        bool operator<(const InputEdge &rhs) const
+        {
+            return source < rhs.source || (source == rhs.source && target < rhs.target);
+        }
+
+        bool operator==(const InputEdge &rhs) const
+        {
+            return source == rhs.source && target == rhs.target;
+        }
+    };
+    using UncontractedGraph = StaticGraph<UncontractedEdgeData>;
+    std::vector<InputEdge> edges;
+    edges.reserve(input_edge_list.size() * 2);
+
+    for (const auto &edge : input_edge_list)
+    {
+        BOOST_ASSERT_MSG(static_cast<unsigned int>(std::max(edge.weight, 1)) > 0,
+                         "edge distance < 1");
+        if (edge.forward)
+        {
+            edges.push_back({edge.source, edge.target, {}});
+        }
+
+        if (edge.backward)
+        {
+            edges.push_back({edge.target, edge.source, {}});
+        }
+    }
+
+    // connect forward and backward nodes of each edge
+    for (const auto &node : input_nodes)
+    {
+        if (node.reverse_edge_based_node_id != SPECIAL_NODEID)
+        {
+            edges.push_back({node.forward_edge_based_node_id, node.reverse_edge_based_node_id, {}});
+            edges.push_back({node.reverse_edge_based_node_id, node.forward_edge_based_node_id, {}});
+        }
+    }
+
+    tbb::parallel_sort(edges.begin(), edges.end());
+    auto new_end = std::unique(edges.begin(), edges.end());
+    edges.resize(new_end - edges.begin());
+
+    auto uncontractor_graph = std::make_shared<UncontractedGraph>(max_edge_id + 1, edges);
+
+    TarjanSCC<UncontractedGraph> component_search(
+        std::const_pointer_cast<const UncontractedGraph>(uncontractor_graph));
+    component_search.run();
+
+    for (auto &node : input_nodes)
+    {
+        auto forward_component = component_search.get_component_id(node.forward_edge_based_node_id);
+        BOOST_ASSERT(node.reverse_edge_based_node_id == SPECIAL_EDGEID ||
+                     forward_component ==
+                         component_search.get_component_id(node.reverse_edge_based_node_id));
+
+        const unsigned component_size = component_search.get_component_size(forward_component);
+        node.component.is_tiny = component_size < config.small_component_size;
+        node.component.id = 1 + forward_component;
+    }
+}
+
+/**
+  \brief Build load restrictions from .restriction file
+  */
+std::shared_ptr<RestrictionMap> extractor::LoadRestrictionMap()
+{
+    boost::filesystem::ifstream input_stream(config.restriction_file_name,
+                                             std::ios::in | std::ios::binary);
+
+    std::vector<TurnRestriction> restriction_list;
+    loadRestrictionsFromFile(input_stream, restriction_list);
+
+    SimpleLogger().Write() << " - " << restriction_list.size() << " restrictions.";
+
+    return std::make_shared<RestrictionMap>(restriction_list);
+}
+
+/**
+  \brief Load node based graph from .osrm file
+  */
+std::shared_ptr<NodeBasedDynamicGraph>
+extractor::LoadNodeBasedGraph(std::unordered_set<NodeID> &barrier_nodes,
+                              std::unordered_set<NodeID> &traffic_lights,
+                              std::vector<QueryNode> &internal_to_external_node_map)
+{
+    std::vector<NodeBasedEdge> edge_list;
+
+    boost::filesystem::ifstream input_stream(config.output_file_name,
+                                             std::ios::in | std::ios::binary);
+
+    std::vector<NodeID> barrier_list;
+    std::vector<NodeID> traffic_light_list;
+    NodeID number_of_node_based_nodes = loadNodesFromFile(
+        input_stream, barrier_list, traffic_light_list, internal_to_external_node_map);
+
+    SimpleLogger().Write() << " - " << barrier_list.size() << " bollard nodes, "
+                           << traffic_light_list.size() << " traffic lights";
+
+    // insert into unordered sets for fast lookup
+    barrier_nodes.insert(barrier_list.begin(), barrier_list.end());
+    traffic_lights.insert(traffic_light_list.begin(), traffic_light_list.end());
+
+    barrier_list.clear();
+    barrier_list.shrink_to_fit();
+    traffic_light_list.clear();
+    traffic_light_list.shrink_to_fit();
+
+    loadEdgesFromFile(input_stream, edge_list);
+
+    if (edge_list.empty())
+    {
+        SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
+        return std::shared_ptr<NodeBasedDynamicGraph>();
+    }
+
+    return NodeBasedDynamicGraphFromEdges(number_of_node_based_nodes, edge_list);
+}
+
+/**
+ \brief Building an edge-expanded graph from node-based input and turn restrictions
+*/
+std::pair<std::size_t, std::size_t>
+extractor::BuildEdgeExpandedGraph(std::vector<QueryNode> &internal_to_external_node_map,
+                                  std::vector<EdgeBasedNode> &node_based_edge_list,
+                                  std::vector<bool> &node_is_startpoint,
+                                  DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list)
+{
+    lua_State *lua_state = luaL_newstate();
+    luabind::open(lua_state);
+
+    SpeedProfileProperties speed_profile;
+    SetupScriptingEnvironment(lua_state, speed_profile);
+
+    std::unordered_set<NodeID> barrier_nodes;
+    std::unordered_set<NodeID> traffic_lights;
+
+    auto restriction_map = LoadRestrictionMap();
+    auto node_based_graph =
+        LoadNodeBasedGraph(barrier_nodes, traffic_lights, internal_to_external_node_map);
+
+    CompressedEdgeContainer compressed_edge_container;
+    GraphCompressor graph_compressor(speed_profile);
+    graph_compressor.Compress(barrier_nodes, traffic_lights, *restriction_map, *node_based_graph,
+                              compressed_edge_container);
+
+    EdgeBasedGraphFactory edge_based_graph_factory(
+        node_based_graph, compressed_edge_container, barrier_nodes, traffic_lights,
+        std::const_pointer_cast<RestrictionMap const>(restriction_map),
+        internal_to_external_node_map, speed_profile);
+
+    compressed_edge_container.SerializeInternalVector(config.geometry_output_path);
+
+    edge_based_graph_factory.Run(config.edge_output_path, lua_state,
+                                 config.edge_segment_lookup_path, config.edge_penalty_path,
+                                 config.generate_edge_lookup
+#ifdef DEBUG_GEOMETRY
+                                 ,
+                                 config.debug_turns_path
+#endif
+                                 );
+    lua_close(lua_state);
+
+    edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list);
+    edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list);
+    edge_based_graph_factory.GetStartPointMarkers(node_is_startpoint);
+    auto max_edge_id = edge_based_graph_factory.GetHighestEdgeID();
+
+    const std::size_t number_of_node_based_nodes = node_based_graph->GetNumberOfNodes();
+    return std::make_pair(number_of_node_based_nodes, max_edge_id);
+}
+
+/**
+  \brief Writing info on original (node-based) nodes
+ */
+void extractor::WriteNodeMapping(const std::vector<QueryNode> &internal_to_external_node_map)
+{
+    boost::filesystem::ofstream node_stream(config.node_output_path, std::ios::binary);
+    const unsigned size_of_mapping = internal_to_external_node_map.size();
+    node_stream.write((char *)&size_of_mapping, sizeof(unsigned));
+    if (size_of_mapping > 0)
+    {
+        node_stream.write((char *)internal_to_external_node_map.data(),
+                          size_of_mapping * sizeof(QueryNode));
+    }
+    node_stream.close();
+}
+
+/**
+    \brief Building rtree-based nearest-neighbor data structure
+
+    Saves tree into '.ramIndex' and leaves into '.fileIndex'.
+ */
+void extractor::BuildRTree(std::vector<EdgeBasedNode> node_based_edge_list,
+                           std::vector<bool> node_is_startpoint,
+                           const std::vector<QueryNode> &internal_to_external_node_map)
+{
+    SimpleLogger().Write() << "constructing r-tree of " << node_based_edge_list.size()
+                           << " edge elements build on-top of "
+                           << internal_to_external_node_map.size() << " coordinates";
+
+    BOOST_ASSERT(node_is_startpoint.size() == node_based_edge_list.size());
+
+    // Filter node based edges based on startpoint
+    auto out_iter = node_based_edge_list.begin();
+    auto in_iter = node_based_edge_list.begin();
+    for (auto index : osrm::irange<std::size_t>(0, node_is_startpoint.size()))
+    {
+        BOOST_ASSERT(in_iter != node_based_edge_list.end());
+        if (node_is_startpoint[index])
+        {
+            *out_iter = *in_iter;
+            out_iter++;
+        }
+        in_iter++;
+    }
+    auto new_size = out_iter - node_based_edge_list.begin();
+    node_based_edge_list.resize(new_size);
+
+    TIMER_START(construction);
+    StaticRTree<EdgeBasedNode>(node_based_edge_list, config.rtree_nodes_output_path,
+                               config.rtree_leafs_output_path, internal_to_external_node_map);
+
+    TIMER_STOP(construction);
+    SimpleLogger().Write() << "finished r-tree construction in " << TIMER_SEC(construction)
+                           << " seconds";
+}
+
+void extractor::WriteEdgeBasedGraph(std::string const &output_file_filename,
+                                    size_t const max_edge_id,
+                                    DeallocatingVector<EdgeBasedEdge> const &edge_based_edge_list)
+{
+
+    std::ofstream file_out_stream;
+    file_out_stream.open(output_file_filename.c_str(), std::ios::binary);
+    const FingerPrint fingerprint = FingerPrint::GetValid();
+    file_out_stream.write((char *)&fingerprint, sizeof(FingerPrint));
+
+    std::cout << "[extractor] Writing edge-based-graph egdes       ... " << std::flush;
+    TIMER_START(write_edges);
+
+    size_t number_of_used_edges = edge_based_edge_list.size();
+    file_out_stream.write((char *)&number_of_used_edges, sizeof(size_t));
+    file_out_stream.write((char *)&max_edge_id, sizeof(size_t));
+
+    for (const auto &edge : edge_based_edge_list)
+    {
+        file_out_stream.write((char *)&edge, sizeof(EdgeBasedEdge));
+    }
+
+    TIMER_STOP(write_edges);
+    std::cout << "ok, after " << TIMER_SEC(write_edges) << "s" << std::endl;
+
+    SimpleLogger().Write() << "Processed " << number_of_used_edges << " edges";
+    file_out_stream.close();
+}

src/extractor/extractor_callbacks.cpp

@@ -0,0 +1,241 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "extractor_callbacks.hpp"
+#include "extraction_containers.hpp"
+#include "extraction_node.hpp"
+#include "extraction_way.hpp"
+
+#include "../data_structures/external_memory_node.hpp"
+#include "../data_structures/restriction.hpp"
+#include "../util/container.hpp"
+#include "../util/simple_logger.hpp"
+
+#include <boost/optional/optional.hpp>
+
+#include <osmium/osm.hpp>
+
+#include <osrm/coordinate.hpp>
+
+#include <limits>
+#include <string>
+#include <vector>
+
+ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers &extraction_containers)
+    : external_memory(extraction_containers)
+{
+    string_map[""] = 0;
+}
+
+/**
+ * Takes the node position from osmium and the filtered properties from the lua
+ * profile and saves them to external memory.
+ *
+ * warning: caller needs to take care of synchronization!
+ */
+void ExtractorCallbacks::ProcessNode(const osmium::Node &input_node,
+                                     const ExtractionNode &result_node)
+{
+    external_memory.all_nodes_list.push_back(
+        {static_cast<int>(input_node.location().lat() * COORDINATE_PRECISION),
+         static_cast<int>(input_node.location().lon() * COORDINATE_PRECISION),
+         OSMNodeID(input_node.id()),
+         result_node.barrier,
+         result_node.traffic_lights});
+}
+
+void ExtractorCallbacks::ProcessRestriction(
+    const boost::optional<InputRestrictionContainer> &restriction)
+{
+    if (restriction)
+    {
+        external_memory.restrictions_list.push_back(restriction.get());
+        // SimpleLogger().Write() << "from: " << restriction.get().restriction.from.node <<
+        //                           ",via: " << restriction.get().restriction.via.node <<
+        //                           ", to: " << restriction.get().restriction.to.node <<
+        //                           ", only: " << (restriction.get().restriction.flags.is_only ?
+        //                           "y" : "n");
+    }
+}
+/**
+ * Takes the geometry contained in the ```input_way``` and the tags computed
+ * by the lua profile inside ```parsed_way``` and computes all edge segments.
+ *
+ * Depending on the forward/backwards weights the edges are split into forward
+ * and backward edges.
+ *
+ * warning: caller needs to take care of synchronization!
+ */
+void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const ExtractionWay &parsed_way)
+{
+    if (((0 >= parsed_way.forward_speed) ||
+         (TRAVEL_MODE_INACCESSIBLE == parsed_way.forward_travel_mode)) &&
+        ((0 >= parsed_way.backward_speed) ||
+         (TRAVEL_MODE_INACCESSIBLE == parsed_way.backward_travel_mode)) &&
+        (0 >= parsed_way.duration))
+    { // Only true if the way is specified by the speed profile
+        return;
+    }
+
+    if (input_way.nodes().size() <= 1)
+    { // safe-guard against broken data
+        return;
+    }
+
+    if (std::numeric_limits<decltype(input_way.id())>::max() == input_way.id())
+    {
+        SimpleLogger().Write(logDEBUG) << "found bogus way with id: " << input_way.id()
+                                       << " of size " << input_way.nodes().size();
+        return;
+    }
+
+    InternalExtractorEdge::WeightData forward_weight_data;
+    InternalExtractorEdge::WeightData backward_weight_data;
+
+    if (0 < parsed_way.duration)
+    {
+        const unsigned num_edges = (input_way.nodes().size() - 1);
+        // FIXME We devide by the numer of nodes here, but should rather consider
+        // the length of each segment. We would eigther have to compute the length
+        // of the whole way here (we can't: no node coordinates) or push that back
+        // to the container and keep a reference to the way.
+        forward_weight_data.duration = parsed_way.duration / num_edges;
+        forward_weight_data.type = InternalExtractorEdge::WeightType::WAY_DURATION;
+        backward_weight_data.duration = parsed_way.duration / num_edges;
+        backward_weight_data.type = InternalExtractorEdge::WeightType::WAY_DURATION;
+    }
+    else
+    {
+        if (parsed_way.forward_speed > 0 &&
+            parsed_way.forward_travel_mode != TRAVEL_MODE_INACCESSIBLE)
+        {
+            forward_weight_data.speed = parsed_way.forward_speed;
+            forward_weight_data.type = InternalExtractorEdge::WeightType::SPEED;
+        }
+        if (parsed_way.backward_speed > 0 &&
+            parsed_way.backward_travel_mode != TRAVEL_MODE_INACCESSIBLE)
+        {
+            backward_weight_data.speed = parsed_way.backward_speed;
+            backward_weight_data.type = InternalExtractorEdge::WeightType::SPEED;
+        }
+    }
+
+    if (forward_weight_data.type == InternalExtractorEdge::WeightType::INVALID &&
+        backward_weight_data.type == InternalExtractorEdge::WeightType::INVALID)
+    {
+        SimpleLogger().Write(logDEBUG) << "found way with bogus speed, id: " << input_way.id();
+        return;
+    }
+
+    // Get the unique identifier for the street name
+    const auto &string_map_iterator = string_map.find(parsed_way.name);
+    unsigned name_id = external_memory.name_lengths.size();
+    if (string_map.end() == string_map_iterator)
+    {
+        auto name_length = std::min<unsigned>(255u, parsed_way.name.size());
+        std::copy(parsed_way.name.c_str(), parsed_way.name.c_str() + name_length, std::back_inserter(external_memory.name_char_data));
+        external_memory.name_lengths.push_back(name_length);
+        string_map.insert(std::make_pair(parsed_way.name, name_id));
+    }
+    else
+    {
+        name_id = string_map_iterator->second;
+    }
+
+    const bool split_edge = (parsed_way.forward_speed > 0) &&
+                            (TRAVEL_MODE_INACCESSIBLE != parsed_way.forward_travel_mode) &&
+                            (parsed_way.backward_speed > 0) &&
+                            (TRAVEL_MODE_INACCESSIBLE != parsed_way.backward_travel_mode) &&
+                            ((parsed_way.forward_speed != parsed_way.backward_speed) ||
+                             (parsed_way.forward_travel_mode != parsed_way.backward_travel_mode));
+
+    std::transform(input_way.nodes().begin(), input_way.nodes().end(),
+                   std::back_inserter(external_memory.used_node_id_list),
+                   [](const osmium::NodeRef &ref)
+                   {
+                       return OSMNodeID(ref.ref());
+                   });
+
+    const bool is_opposite_way = TRAVEL_MODE_INACCESSIBLE == parsed_way.forward_travel_mode;
+
+    // traverse way in reverse in this case
+    if (is_opposite_way)
+    {
+        BOOST_ASSERT(split_edge == false);
+        BOOST_ASSERT(parsed_way.backward_travel_mode != TRAVEL_MODE_INACCESSIBLE);
+        osrm::for_each_pair(input_way.nodes().crbegin(), input_way.nodes().crend(),
+                            [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node)
+                            {
+                                external_memory.all_edges_list.push_back(InternalExtractorEdge(
+                                    OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), name_id,
+                                    backward_weight_data, true, false, parsed_way.roundabout,
+                                    parsed_way.is_access_restricted, parsed_way.is_startpoint,
+                                    parsed_way.backward_travel_mode, false));
+                            });
+
+        external_memory.way_start_end_id_list.push_back(
+            {OSMWayID(input_way.id()),
+             OSMNodeID(input_way.nodes().back().ref()),
+             OSMNodeID(input_way.nodes()[input_way.nodes().size() - 2].ref()),
+             OSMNodeID(input_way.nodes()[1].ref()),
+             OSMNodeID(input_way.nodes()[0].ref())});
+    }
+    else
+    {
+        const bool forward_only =
+            split_edge || TRAVEL_MODE_INACCESSIBLE == parsed_way.backward_travel_mode;
+        osrm::for_each_pair(input_way.nodes().cbegin(), input_way.nodes().cend(),
+                            [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node)
+                            {
+                                external_memory.all_edges_list.push_back(InternalExtractorEdge(
+                                    OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), name_id, forward_weight_data,
+                                    true, !forward_only, parsed_way.roundabout,
+                                    parsed_way.is_access_restricted, parsed_way.is_startpoint, parsed_way.forward_travel_mode,
+                                    split_edge));
+                            });
+        if (split_edge)
+        {
+            BOOST_ASSERT(parsed_way.backward_travel_mode != TRAVEL_MODE_INACCESSIBLE);
+            osrm::for_each_pair(
+                input_way.nodes().cbegin(), input_way.nodes().cend(),
+                [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node)
+                {
+                    external_memory.all_edges_list.push_back(InternalExtractorEdge(
+                        OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), name_id, backward_weight_data, false,
+                        true, parsed_way.roundabout, parsed_way.is_access_restricted,
+                        parsed_way.is_startpoint, parsed_way.backward_travel_mode, true));
+                });
+        }
+
+        external_memory.way_start_end_id_list.push_back(
+            {OSMWayID(input_way.id()),
+             OSMNodeID(input_way.nodes().back().ref()),
+             OSMNodeID(input_way.nodes()[input_way.nodes().size() - 2].ref()),
+             OSMNodeID(input_way.nodes()[1].ref()),
+             OSMNodeID(input_way.nodes()[0].ref())});
+    }
+}

src/extractor/extractor_options.cpp

@@ -0,0 +1,230 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "extractor_options.hpp"
+
+#include "util/version.hpp"
+#include "../util/ini_file.hpp"
+#include "../util/simple_logger.hpp"
+
+#include <boost/filesystem.hpp>
+#include <boost/program_options.hpp>
+
+#include <tbb/task_scheduler_init.h>
+
+return_code
+ExtractorOptions::ParseArguments(int argc, char *argv[], ExtractorConfig &extractor_config)
+{
+    // declare a group of options that will be allowed only on command line
+    boost::program_options::options_description generic_options("Options");
+    generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message")(
+        /*
+         * TODO: re-enable this
+        "restrictions,r",
+        boost::program_options::value<boost::filesystem::path>(&extractor_config.restrictions_path),
+        "Restrictions file in .osrm.restrictions format")(
+        */
+        "config,c", boost::program_options::value<boost::filesystem::path>(
+                        &extractor_config.config_file_path)->default_value("extractor.ini"),
+        "Path to a configuration file.");
+
+    // declare a group of options that will be allowed both on command line and in config file
+    boost::program_options::options_description config_options("Configuration");
+    config_options.add_options()("profile,p",
+                                 boost::program_options::value<boost::filesystem::path>(
+                                     &extractor_config.profile_path)->default_value("profile.lua"),
+                                 "Path to LUA routing profile")(
+        "threads,t",
+        boost::program_options::value<unsigned int>(&extractor_config.requested_num_threads)
+            ->default_value(tbb::task_scheduler_init::default_num_threads()),
+        "Number of threads to use")(
+            "generate-edge-lookup",boost::program_options::value<bool>(
+                                                &extractor_config.generate_edge_lookup)->implicit_value(true)->default_value(false),
+                                 "Generate a lookup table for internal edge-expanded-edge IDs to OSM node pairs")(
+        "small-component-size",
+        boost::program_options::value<unsigned int>(&extractor_config.small_component_size)
+            ->default_value(1000),
+        "Number of nodes required before a strongly-connected-componennt is considered big (affects nearest neighbor snapping)");
+
+#ifdef DEBUG_GEOMETRY
+        config_options.add_options()("debug-turns",
+            boost::program_options::value<std::string>(&extractor_config.debug_turns_path),
+            "Write out GeoJSON with turn penalty data");
+#endif // DEBUG_GEOMETRY
+
+    // hidden options, will be allowed both on command line and in config file, but will not be
+    // shown to the user
+    boost::program_options::options_description hidden_options("Hidden options");
+    hidden_options.add_options()("input,i", boost::program_options::value<boost::filesystem::path>(
+                                                &extractor_config.input_path),
+                                 "Input file in .osm, .osm.bz2 or .osm.pbf format");
+
+
+    // positional option
+    boost::program_options::positional_options_description positional_options;
+    positional_options.add("input", 1);
+
+    // combine above options for parsing
+    boost::program_options::options_description cmdline_options;
+    cmdline_options.add(generic_options).add(config_options).add(hidden_options);
+
+    boost::program_options::options_description config_file_options;
+    config_file_options.add(config_options).add(hidden_options);
+
+    boost::program_options::options_description visible_options(
+        boost::filesystem::basename(argv[0]) + " <input.osm/.osm.bz2/.osm.pbf> [options]");
+    visible_options.add(generic_options).add(config_options);
+
+    // parse command line options
+    try
+    {
+        boost::program_options::variables_map option_variables;
+        boost::program_options::store(boost::program_options::command_line_parser(argc, argv)
+                                          .options(cmdline_options)
+                                          .positional(positional_options)
+                                          .run(),
+                                      option_variables);
+        if (option_variables.count("version"))
+        {
+            SimpleLogger().Write() << OSRM_VERSION;
+            return return_code::exit;
+        }
+
+        if (option_variables.count("help"))
+        {
+            SimpleLogger().Write() << visible_options;
+            return return_code::exit;
+        }
+
+        boost::program_options::notify(option_variables);
+
+        // parse config file
+        if (boost::filesystem::is_regular_file(extractor_config.config_file_path))
+        {
+            SimpleLogger().Write()
+                << "Reading options from: " << extractor_config.config_file_path.string();
+            std::string ini_file_contents =
+                read_file_lower_content(extractor_config.config_file_path);
+            std::stringstream config_stream(ini_file_contents);
+            boost::program_options::store(parse_config_file(config_stream, config_file_options),
+                                          option_variables);
+            boost::program_options::notify(option_variables);
+        }
+
+        if (!option_variables.count("input"))
+        {
+            SimpleLogger().Write() << visible_options;
+            return return_code::exit;
+        }
+    }
+    catch (std::exception &e)
+    {
+        SimpleLogger().Write(logWARNING) << e.what();
+        return return_code::fail;
+    }
+
+    return return_code::ok;
+}
+
+void ExtractorOptions::GenerateOutputFilesNames(ExtractorConfig &extractor_config)
+{
+    boost::filesystem::path &input_path = extractor_config.input_path;
+    extractor_config.output_file_name = input_path.string();
+    extractor_config.restriction_file_name = input_path.string();
+    extractor_config.names_file_name = input_path.string();
+    extractor_config.timestamp_file_name = input_path.string();
+    extractor_config.geometry_output_path = input_path.string();
+    extractor_config.edge_output_path = input_path.string();
+    extractor_config.edge_graph_output_path = input_path.string();
+    extractor_config.node_output_path = input_path.string();
+    extractor_config.rtree_nodes_output_path = input_path.string();
+    extractor_config.rtree_leafs_output_path = input_path.string();
+    extractor_config.edge_segment_lookup_path = input_path.string();
+    extractor_config.edge_penalty_path = input_path.string();
+    std::string::size_type pos = extractor_config.output_file_name.find(".osm.bz2");
+    if (pos == std::string::npos)
+    {
+        pos = extractor_config.output_file_name.find(".osm.pbf");
+        if (pos == std::string::npos)
+        {
+            pos = extractor_config.output_file_name.find(".osm.xml");
+        }
+    }
+    if (pos == std::string::npos)
+    {
+        pos = extractor_config.output_file_name.find(".pbf");
+    }
+    if (pos == std::string::npos)
+    {
+        pos = extractor_config.output_file_name.find(".osm");
+        if (pos == std::string::npos)
+        {
+            extractor_config.output_file_name.append(".osrm");
+            extractor_config.restriction_file_name.append(".osrm.restrictions");
+            extractor_config.names_file_name.append(".osrm.names");
+            extractor_config.timestamp_file_name.append(".osrm.timestamp");
+            extractor_config.geometry_output_path.append(".osrm.geometry");
+            extractor_config.node_output_path.append(".osrm.nodes");
+            extractor_config.edge_output_path.append(".osrm.edges");
+            extractor_config.edge_graph_output_path.append(".osrm.ebg");
+            extractor_config.rtree_nodes_output_path.append(".osrm.ramIndex");
+            extractor_config.rtree_leafs_output_path.append(".osrm.fileIndex");
+            extractor_config.edge_segment_lookup_path.append(".osrm.edge_segment_lookup");
+            extractor_config.edge_penalty_path.append(".osrm.edge_penalties");
+        }
+        else
+        {
+            extractor_config.output_file_name.replace(pos, 5, ".osrm");
+            extractor_config.restriction_file_name.replace(pos, 5, ".osrm.restrictions");
+            extractor_config.names_file_name.replace(pos, 5, ".osrm.names");
+            extractor_config.timestamp_file_name.replace(pos, 5, ".osrm.timestamp");
+            extractor_config.geometry_output_path.replace(pos, 5, ".osrm.geometry");
+            extractor_config.node_output_path.replace(pos, 5, ".osrm.nodes");
+            extractor_config.edge_output_path.replace(pos, 5, ".osrm.edges");
+            extractor_config.edge_graph_output_path.replace(pos, 5, ".osrm.ebg");
+            extractor_config.rtree_nodes_output_path.replace(pos, 5, ".osrm.ramIndex");
+            extractor_config.rtree_leafs_output_path.replace(pos, 5, ".osrm.fileIndex");
+            extractor_config.edge_segment_lookup_path.replace(pos,5, ".osrm.edge_segment_lookup");
+            extractor_config.edge_penalty_path.replace(pos,5, ".osrm.edge_penalties");
+        }
+    }
+    else
+    {
+        extractor_config.output_file_name.replace(pos, 8, ".osrm");
+        extractor_config.restriction_file_name.replace(pos, 8, ".osrm.restrictions");
+        extractor_config.names_file_name.replace(pos, 8, ".osrm.names");
+        extractor_config.timestamp_file_name.replace(pos, 8, ".osrm.timestamp");
+        extractor_config.geometry_output_path.replace(pos, 8, ".osrm.geometry");
+        extractor_config.node_output_path.replace(pos, 8, ".osrm.nodes");
+        extractor_config.edge_output_path.replace(pos, 8, ".osrm.edges");
+        extractor_config.edge_graph_output_path.replace(pos, 8, ".osrm.ebg");
+        extractor_config.rtree_nodes_output_path.replace(pos, 8, ".osrm.ramIndex");
+        extractor_config.rtree_leafs_output_path.replace(pos, 8, ".osrm.fileIndex");
+        extractor_config.edge_segment_lookup_path.replace(pos,8, ".osrm.edge_segment_lookup");
+        extractor_config.edge_penalty_path.replace(pos,8, ".osrm.edge_penalties");
+    }
+}

src/extractor/graph_compressor.cpp

@@ -0,0 +1,188 @@
+#include "graph_compressor.hpp"
+
+#include "../data_structures/compressed_edge_container.hpp"
+#include "../data_structures/dynamic_graph.hpp"
+#include "../data_structures/node_based_graph.hpp"
+#include "../data_structures/restriction_map.hpp"
+#include "../data_structures/percent.hpp"
+
+#include "../util/simple_logger.hpp"
+
+GraphCompressor::GraphCompressor(SpeedProfileProperties speed_profile)
+    : speed_profile(std::move(speed_profile))
+{
+}
+
+
+void GraphCompressor::Compress(const std::unordered_set<NodeID>& barrier_nodes,
+                               const std::unordered_set<NodeID>& traffic_lights,
+                               RestrictionMap& restriction_map,
+                               NodeBasedDynamicGraph& graph,
+                               CompressedEdgeContainer& geometry_compressor)
+{
+    const unsigned original_number_of_nodes = graph.GetNumberOfNodes();
+    const unsigned original_number_of_edges = graph.GetNumberOfEdges();
+
+    Percent progress(original_number_of_nodes);
+
+    for (const NodeID node_v : osrm::irange(0u, original_number_of_nodes))
+    {
+        progress.printStatus(node_v);
+
+        // only contract degree 2 vertices
+        if (2 != graph.GetOutDegree(node_v))
+        {
+            continue;
+        }
+
+        // don't contract barrier node
+        if (barrier_nodes.end() != barrier_nodes.find(node_v))
+        {
+            continue;
+        }
+
+        // check if v is a via node for a turn restriction, i.e. a 'directed' barrier node
+        if (restriction_map.IsViaNode(node_v))
+        {
+            continue;
+        }
+
+        //    reverse_e2   forward_e2
+        // u <---------- v -----------> w
+        //    ----------> <-----------
+        //    forward_e1   reverse_e1
+        //
+        // Will be compressed to:
+        //
+        //    reverse_e1
+        // u <---------- w
+        //    ---------->
+        //    forward_e1
+        //
+        // If the edges are compatible.
+
+        const bool reverse_edge_order = graph.GetEdgeData(graph.BeginEdges(node_v)).reversed;
+        const EdgeID forward_e2 = graph.BeginEdges(node_v) + reverse_edge_order;
+        BOOST_ASSERT(SPECIAL_EDGEID != forward_e2);
+        BOOST_ASSERT(forward_e2 >= graph.BeginEdges(node_v) &&
+                     forward_e2 < graph.EndEdges(node_v));
+        const EdgeID reverse_e2 = graph.BeginEdges(node_v) + 1 - reverse_edge_order;
+        BOOST_ASSERT(SPECIAL_EDGEID != reverse_e2);
+        BOOST_ASSERT(reverse_e2 >= graph.BeginEdges(node_v) &&
+                     reverse_e2 < graph.EndEdges(node_v));
+
+        const EdgeData &fwd_edge_data2 = graph.GetEdgeData(forward_e2);
+        const EdgeData &rev_edge_data2 = graph.GetEdgeData(reverse_e2);
+
+        const NodeID node_w = graph.GetTarget(forward_e2);
+        BOOST_ASSERT(SPECIAL_NODEID != node_w);
+        BOOST_ASSERT(node_v != node_w);
+        const NodeID node_u = graph.GetTarget(reverse_e2);
+        BOOST_ASSERT(SPECIAL_NODEID != node_u);
+        BOOST_ASSERT(node_u != node_v);
+
+        const EdgeID forward_e1 = graph.FindEdge(node_u, node_v);
+        BOOST_ASSERT(SPECIAL_EDGEID != forward_e1);
+        BOOST_ASSERT(node_v == graph.GetTarget(forward_e1));
+        const EdgeID reverse_e1 = graph.FindEdge(node_w, node_v);
+        BOOST_ASSERT(SPECIAL_EDGEID != reverse_e1);
+        BOOST_ASSERT(node_v == graph.GetTarget(reverse_e1));
+
+        const EdgeData &fwd_edge_data1 = graph.GetEdgeData(forward_e1);
+        const EdgeData &rev_edge_data1 = graph.GetEdgeData(reverse_e1);
+
+        if (graph.FindEdgeInEitherDirection(node_u, node_w) != SPECIAL_EDGEID)
+        {
+            continue;
+        }
+
+        // this case can happen if two ways with different names overlap
+        if (fwd_edge_data1.name_id != rev_edge_data1.name_id ||
+            fwd_edge_data2.name_id != rev_edge_data2.name_id)
+        {
+            continue;
+        }
+
+        if (fwd_edge_data1.IsCompatibleTo(fwd_edge_data2) && rev_edge_data1.IsCompatibleTo(rev_edge_data2))
+        {
+            BOOST_ASSERT(graph.GetEdgeData(forward_e1).name_id ==
+                         graph.GetEdgeData(reverse_e1).name_id);
+            BOOST_ASSERT(graph.GetEdgeData(forward_e2).name_id ==
+                         graph.GetEdgeData(reverse_e2).name_id);
+
+            // Get distances before graph is modified
+            const int forward_weight1 = graph.GetEdgeData(forward_e1).distance;
+            const int forward_weight2 = graph.GetEdgeData(forward_e2).distance;
+
+            BOOST_ASSERT(0 != forward_weight1);
+            BOOST_ASSERT(0 != forward_weight2);
+
+            const int reverse_weight1 = graph.GetEdgeData(reverse_e1).distance;
+            const int reverse_weight2 = graph.GetEdgeData(reverse_e2).distance;
+
+            BOOST_ASSERT(0 != reverse_weight1);
+            BOOST_ASSERT(0 != reverse_weight2);
+
+            const bool has_node_penalty = traffic_lights.find(node_v) != traffic_lights.end();
+
+            // add weight of e2's to e1
+            graph.GetEdgeData(forward_e1).distance += fwd_edge_data2.distance;
+            graph.GetEdgeData(reverse_e1).distance += rev_edge_data2.distance;
+            if (has_node_penalty)
+            {
+                graph.GetEdgeData(forward_e1).distance +=
+                    speed_profile.traffic_signal_penalty;
+                graph.GetEdgeData(reverse_e1).distance +=
+                    speed_profile.traffic_signal_penalty;
+            }
+
+            // extend e1's to targets of e2's
+            graph.SetTarget(forward_e1, node_w);
+            graph.SetTarget(reverse_e1, node_u);
+
+            // remove e2's (if bidir, otherwise only one)
+            graph.DeleteEdge(node_v, forward_e2);
+            graph.DeleteEdge(node_v, reverse_e2);
+
+            // update any involved turn restrictions
+            restriction_map.FixupStartingTurnRestriction(node_u, node_v, node_w);
+            restriction_map.FixupArrivingTurnRestriction(node_u, node_v, node_w, graph);
+
+            restriction_map.FixupStartingTurnRestriction(node_w, node_v, node_u);
+            restriction_map.FixupArrivingTurnRestriction(node_w, node_v, node_u, graph);
+
+            // store compressed geometry in container
+            geometry_compressor.CompressEdge(
+                forward_e1, forward_e2, node_v, node_w,
+                forward_weight1 + (has_node_penalty ? speed_profile.traffic_signal_penalty : 0),
+                forward_weight2);
+            geometry_compressor.CompressEdge(
+                reverse_e1, reverse_e2, node_v, node_u, reverse_weight1,
+                reverse_weight2 + (has_node_penalty ? speed_profile.traffic_signal_penalty : 0));
+        }
+    }
+
+    PrintStatistics(original_number_of_nodes, original_number_of_edges, graph);
+}
+
+void GraphCompressor::PrintStatistics(unsigned original_number_of_nodes,
+                                      unsigned original_number_of_edges,
+                                      const NodeBasedDynamicGraph& graph) const
+{
+
+    unsigned new_node_count = 0;
+    unsigned new_edge_count = 0;
+
+    for (const auto i : osrm::irange(0u, graph.GetNumberOfNodes()))
+    {
+        if (graph.GetOutDegree(i) > 0)
+        {
+            ++new_node_count;
+            new_edge_count += (graph.EndEdges(i) - graph.BeginEdges(i));
+        }
+    }
+    SimpleLogger().Write() << "Node compression ratio: "
+                           << new_node_count / (double)original_number_of_nodes;
+    SimpleLogger().Write() << "Edge compression ratio: "
+                           << new_edge_count / (double)original_number_of_edges;
+}

src/extractor/import_edge.cpp

@@ -0,0 +1,113 @@
+/*
+
+Copyright (c) 2014, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "import_edge.hpp"
+
+#include "travel_mode.hpp"
+#include "../typedefs.h"
+
+bool NodeBasedEdge::operator<(const NodeBasedEdge &other) const
+{
+    if (source == other.source)
+    {
+        if (target == other.target)
+        {
+            if (weight == other.weight)
+            {
+                return forward && backward && ((!other.forward) || (!other.backward));
+            }
+            return weight < other.weight;
+        }
+        return target < other.target;
+    }
+    return source < other.source;
+}
+
+NodeBasedEdge::NodeBasedEdge()
+    : source(SPECIAL_NODEID), target(SPECIAL_NODEID), name_id(0), weight(0), forward(false),
+      backward(false), roundabout(false),
+      access_restricted(false), startpoint(true), is_split(false), travel_mode(false)
+{
+}
+
+NodeBasedEdge::NodeBasedEdge(NodeID source,
+                             NodeID target,
+                             NodeID name_id,
+                             EdgeWeight weight,
+                             bool forward,
+                             bool backward,
+                             bool roundabout,
+                             bool access_restricted,
+                             bool startpoint,
+                             TravelMode travel_mode,
+                             bool is_split)
+    : source(source), target(target), name_id(name_id), weight(weight), forward(forward),
+      backward(backward), roundabout(roundabout),
+      access_restricted(access_restricted), startpoint(startpoint), is_split(is_split), travel_mode(travel_mode)
+{
+}
+
+bool EdgeBasedEdge::operator<(const EdgeBasedEdge &other) const
+{
+    if (source == other.source)
+    {
+        if (target == other.target)
+        {
+            if (weight == other.weight)
+            {
+                return forward && backward && ((!other.forward) || (!other.backward));
+            }
+            return weight < other.weight;
+        }
+        return target < other.target;
+    }
+    return source < other.source;
+}
+
+template <class EdgeT>
+EdgeBasedEdge::EdgeBasedEdge(const EdgeT &other)
+    : source(other.source), target(other.target), edge_id(other.data.via),
+      weight(other.data.distance), forward(other.data.forward), backward(other.data.backward)
+{
+}
+
+/** Default constructor. target and weight are set to 0.*/
+EdgeBasedEdge::EdgeBasedEdge()
+    : source(0), target(0), edge_id(0), weight(0), forward(false), backward(false)
+{
+}
+
+EdgeBasedEdge::EdgeBasedEdge(const NodeID source,
+                             const NodeID target,
+                             const NodeID edge_id,
+                             const EdgeWeight weight,
+                             const bool forward,
+                             const bool backward)
+    : source(source), target(target), edge_id(edge_id), weight(weight), forward(forward),
+      backward(backward)
+{
+}

src/extractor/raster_source.cpp

@@ -0,0 +1,178 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "raster_source.hpp"
+
+#include "../util/simple_logger.hpp"
+#include "../util/timing_util.hpp"
+
+#include <osrm/coordinate.hpp>
+
+#include <cmath>
+
+RasterSource::RasterSource(RasterGrid _raster_data,
+                           std::size_t _width,
+                           std::size_t _height,
+                           int _xmin,
+                           int _xmax,
+                           int _ymin,
+                           int _ymax)
+    : xstep(calcSize(_xmin, _xmax, _width)), ystep(calcSize(_ymin, _ymax, _height)),
+      raster_data(std::move(_raster_data)), width(_width), height(_height), xmin(_xmin),
+      xmax(_xmax), ymin(_ymin), ymax(_ymax)
+{
+    BOOST_ASSERT(xstep != 0);
+    BOOST_ASSERT(ystep != 0);
+}
+
+float RasterSource::calcSize(int min, int max, std::size_t count) const
+{
+    BOOST_ASSERT(count > 0);
+    return (max - min) / (static_cast<float>(count) - 1);
+}
+
+// Query raster source for nearest data point
+RasterDatum RasterSource::getRasterData(const int lon, const int lat) const
+{
+    if (lon < xmin || lon > xmax || lat < ymin || lat > ymax)
+    {
+        return {};
+    }
+
+    const std::size_t xth = static_cast<std::size_t>(round((lon - xmin) / xstep));
+    const std::size_t yth = static_cast<std::size_t>(round((ymax - lat) / ystep));
+
+    return {raster_data(xth, yth)};
+}
+
+// Query raster source using bilinear interpolation
+RasterDatum RasterSource::getRasterInterpolate(const int lon, const int lat) const
+{
+    if (lon < xmin || lon > xmax || lat < ymin || lat > ymax)
+    {
+        return {};
+    }
+
+    const auto xthP = (lon - xmin) / xstep;
+    const auto ythP = (ymax - lat) / ystep;
+
+    const std::size_t top = static_cast<std::size_t>(fmax(floor(ythP), 0));
+    const std::size_t bottom = static_cast<std::size_t>(fmin(ceil(ythP), height - 1));
+    const std::size_t left = static_cast<std::size_t>(fmax(floor(xthP), 0));
+    const std::size_t right = static_cast<std::size_t>(fmin(ceil(xthP), width - 1));
+
+    // Calculate distances from corners for bilinear interpolation
+    const float fromLeft = (lon - left * xstep + xmin) / xstep;
+    const float fromTop = (ymax - top * ystep - lat) / ystep;
+    const float fromRight = 1 - fromLeft;
+    const float fromBottom = 1 - fromTop;
+
+    return {static_cast<std::int32_t>(raster_data(left, top) * (fromRight * fromBottom) +
+                                      raster_data(right, top) * (fromLeft * fromBottom) +
+                                      raster_data(left, bottom) * (fromRight * fromTop) +
+                                      raster_data(right, bottom) * (fromLeft * fromTop))};
+}
+
+// Load raster source into memory
+int SourceContainer::loadRasterSource(const std::string &path_string,
+                                      double xmin,
+                                      double xmax,
+                                      double ymin,
+                                      double ymax,
+                                      std::size_t nrows,
+                                      std::size_t ncols)
+{
+    const auto _xmin = static_cast<int>(xmin * COORDINATE_PRECISION);
+    const auto _xmax = static_cast<int>(xmax * COORDINATE_PRECISION);
+    const auto _ymin = static_cast<int>(ymin * COORDINATE_PRECISION);
+    const auto _ymax = static_cast<int>(ymax * COORDINATE_PRECISION);
+
+    const auto itr = LoadedSourcePaths.find(path_string);
+    if (itr != LoadedSourcePaths.end())
+    {
+        SimpleLogger().Write() << "[source loader] Already loaded source '" << path_string
+                               << "' at source_id " << itr->second;
+        return itr->second;
+    }
+
+    int source_id = static_cast<int>(LoadedSources.size());
+
+    SimpleLogger().Write() << "[source loader] Loading from " << path_string << "  ... ";
+    TIMER_START(loading_source);
+
+    boost::filesystem::path filepath(path_string);
+    if (!boost::filesystem::exists(filepath))
+    {
+        throw osrm::exception("error reading: no such path");
+    }
+
+    RasterGrid rasterData{filepath, ncols, nrows};
+
+    RasterSource source{std::move(rasterData), ncols, nrows, _xmin, _xmax, _ymin, _ymax};
+    TIMER_STOP(loading_source);
+    LoadedSourcePaths.emplace(path_string, source_id);
+    LoadedSources.push_back(std::move(source));
+
+    SimpleLogger().Write() << "[source loader] ok, after " << TIMER_SEC(loading_source) << "s";
+
+    return source_id;
+}
+
+// External function for looking up nearest data point from a specified source
+RasterDatum SourceContainer::getRasterDataFromSource(unsigned int source_id, int lon, int lat)
+{
+    if (LoadedSources.size() < source_id + 1)
+    {
+        throw osrm::exception("error reading: no such loaded source");
+    }
+
+    BOOST_ASSERT(lat < (90 * COORDINATE_PRECISION));
+    BOOST_ASSERT(lat > (-90 * COORDINATE_PRECISION));
+    BOOST_ASSERT(lon < (180 * COORDINATE_PRECISION));
+    BOOST_ASSERT(lon > (-180 * COORDINATE_PRECISION));
+
+    const auto &found = LoadedSources[source_id];
+    return found.getRasterData(lon, lat);
+}
+
+// External function for looking up interpolated data from a specified source
+RasterDatum
+SourceContainer::getRasterInterpolateFromSource(unsigned int source_id, int lon, int lat)
+{
+    if (LoadedSources.size() < source_id + 1)
+    {
+        throw osrm::exception("error reading: no such loaded source");
+    }
+
+    BOOST_ASSERT(lat < (90 * COORDINATE_PRECISION));
+    BOOST_ASSERT(lat > (-90 * COORDINATE_PRECISION));
+    BOOST_ASSERT(lon < (180 * COORDINATE_PRECISION));
+    BOOST_ASSERT(lon > (-180 * COORDINATE_PRECISION));
+
+    const auto &found = LoadedSources[source_id];
+    return found.getRasterInterpolate(lon, lat);
+}

src/extractor/restriction_map.cpp

@@ -0,0 +1,184 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "restriction_map.hpp"
+
+RestrictionMap::RestrictionMap(const std::vector<TurnRestriction> &restriction_list) : m_count(0)
+{
+    // decompose restriction consisting of a start, via and end node into a
+    // a pair of starting edge and a list of all end nodes
+    for (auto &restriction : restriction_list)
+    {
+        // This downcasting is OK because when this is called, the node IDs have been
+        // renumbered into internal values, which should be well under 2^32
+        // This will be a problem if we have more than 2^32 actual restrictions
+        BOOST_ASSERT(restriction.from.node < std::numeric_limits<NodeID>::max());
+        BOOST_ASSERT(restriction.via.node < std::numeric_limits<NodeID>::max());
+        m_restriction_start_nodes.insert(restriction.from.node);
+        m_no_turn_via_node_set.insert(restriction.via.node);
+
+        // This explicit downcasting is also OK for the same reason.
+        RestrictionSource restriction_source = {static_cast<NodeID>(restriction.from.node), static_cast<NodeID>(restriction.via.node)};
+
+        std::size_t index;
+        auto restriction_iter = m_restriction_map.find(restriction_source);
+        if (restriction_iter == m_restriction_map.end())
+        {
+            index = m_restriction_bucket_list.size();
+            m_restriction_bucket_list.resize(index + 1);
+            m_restriction_map.emplace(restriction_source, index);
+        }
+        else
+        {
+            index = restriction_iter->second;
+            // Map already contains an is_only_*-restriction
+            if (m_restriction_bucket_list.at(index).begin()->is_only)
+            {
+                continue;
+            }
+            else if (restriction.flags.is_only)
+            {
+                // We are going to insert an is_only_*-restriction. There can be only one.
+                m_count -= m_restriction_bucket_list.at(index).size();
+                m_restriction_bucket_list.at(index).clear();
+            }
+        }
+        ++m_count;
+        BOOST_ASSERT(restriction.to.node < std::numeric_limits<NodeID>::max());
+        m_restriction_bucket_list.at(index)
+            .emplace_back(restriction.to.node, restriction.flags.is_only);
+    }
+}
+
+bool RestrictionMap::IsViaNode(const NodeID node) const
+{
+    return m_no_turn_via_node_set.find(node) != m_no_turn_via_node_set.end();
+}
+
+// Replaces start edge (v, w) with (u, w). Only start node changes.
+void RestrictionMap::FixupStartingTurnRestriction(const NodeID node_u,
+                                                  const NodeID node_v,
+                                                  const NodeID node_w)
+{
+    BOOST_ASSERT(node_u != SPECIAL_NODEID);
+    BOOST_ASSERT(node_v != SPECIAL_NODEID);
+    BOOST_ASSERT(node_w != SPECIAL_NODEID);
+
+    if (!IsSourceNode(node_v))
+    {
+        return;
+    }
+
+    const auto restriction_iterator = m_restriction_map.find({node_v, node_w});
+    if (restriction_iterator != m_restriction_map.end())
+    {
+        const unsigned index = restriction_iterator->second;
+        // remove old restriction start (v,w)
+        m_restriction_map.erase(restriction_iterator);
+        m_restriction_start_nodes.emplace(node_u);
+        // insert new restriction start (u,w) (pointing to index)
+        RestrictionSource new_source = {node_u, node_w};
+        m_restriction_map.emplace(new_source, index);
+    }
+}
+
+// Check if edge (u, v) is the start of any turn restriction.
+// If so returns id of first target node.
+NodeID RestrictionMap::CheckForEmanatingIsOnlyTurn(const NodeID node_u, const NodeID node_v) const
+{
+    BOOST_ASSERT(node_u != SPECIAL_NODEID);
+    BOOST_ASSERT(node_v != SPECIAL_NODEID);
+
+    if (!IsSourceNode(node_u))
+    {
+        return SPECIAL_NODEID;
+    }
+
+    const auto restriction_iter = m_restriction_map.find({node_u, node_v});
+    if (restriction_iter != m_restriction_map.end())
+    {
+        const unsigned index = restriction_iter->second;
+        const auto &bucket = m_restriction_bucket_list.at(index);
+        for (const RestrictionTarget &restriction_target : bucket)
+        {
+            if (restriction_target.is_only)
+            {
+                return restriction_target.target_node;
+            }
+        }
+    }
+    return SPECIAL_NODEID;
+}
+
+// Checks if turn <u,v,w> is actually a turn restriction.
+bool RestrictionMap::CheckIfTurnIsRestricted(const NodeID node_u,
+                                             const NodeID node_v,
+                                             const NodeID node_w) const
+{
+    BOOST_ASSERT(node_u != SPECIAL_NODEID);
+    BOOST_ASSERT(node_v != SPECIAL_NODEID);
+    BOOST_ASSERT(node_w != SPECIAL_NODEID);
+
+    if (!IsSourceNode(node_u))
+    {
+        return false;
+    }
+
+    const auto restriction_iter = m_restriction_map.find({node_u, node_v});
+    if (restriction_iter == m_restriction_map.end())
+    {
+        return false;
+    }
+
+    const unsigned index = restriction_iter->second;
+    const auto &bucket = m_restriction_bucket_list.at(index);
+
+    for (const RestrictionTarget &restriction_target : bucket)
+    {
+        if (node_w == restriction_target.target_node && // target found
+            !restriction_target.is_only)                // and not an only_-restr.
+        {
+            return true;
+        }
+        if (node_w != restriction_target.target_node && // target not found
+            restriction_target.is_only)                 // and is an only restriction
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
+// check of node is the start of any restriction
+bool RestrictionMap::IsSourceNode(const NodeID node) const
+{
+    if (m_restriction_start_nodes.find(node) == m_restriction_start_nodes.end())
+    {
+        return false;
+    }
+    return true;
+}

src/extractor/restriction_parser.cpp

@@ -0,0 +1,256 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "restriction_parser.hpp"
+#include "extraction_way.hpp"
+
+#include "../data_structures/external_memory_node.hpp"
+#include "../util/lua_util.hpp"
+#include "../util/osrm_exception.hpp"
+#include "../util/simple_logger.hpp"
+
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/regex.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/ref.hpp>
+#include <boost/regex.hpp>
+#include <boost/optional/optional.hpp>
+
+#include <osmium/osm.hpp>
+#include <osmium/tags/regex_filter.hpp>
+
+#include <algorithm>
+#include <iterator>
+
+namespace
+{
+int lua_error_callback(lua_State *lua_state)
+{
+    std::string error_msg = lua_tostring(lua_state, -1);
+    throw osrm::exception("ERROR occurred in profile script:\n" + error_msg);
+}
+}
+
+RestrictionParser::RestrictionParser(lua_State *lua_state) : use_turn_restrictions(true)
+{
+    ReadUseRestrictionsSetting(lua_state);
+
+    if (use_turn_restrictions)
+    {
+        ReadRestrictionExceptions(lua_state);
+    }
+}
+
+void RestrictionParser::ReadUseRestrictionsSetting(lua_State *lua_state)
+{
+    if (0 == luaL_dostring(lua_state, "return use_turn_restrictions\n") &&
+        lua_isboolean(lua_state, -1))
+    {
+        use_turn_restrictions = lua_toboolean(lua_state, -1);
+    }
+
+    if (use_turn_restrictions)
+    {
+        SimpleLogger().Write() << "Using turn restrictions";
+    }
+    else
+    {
+        SimpleLogger().Write() << "Ignoring turn restrictions";
+    }
+}
+
+void RestrictionParser::ReadRestrictionExceptions(lua_State *lua_state)
+{
+    if (lua_function_exists(lua_state, "get_exceptions"))
+    {
+        luabind::set_pcall_callback(&lua_error_callback);
+        // get list of turn restriction exceptions
+        luabind::call_function<void>(lua_state, "get_exceptions",
+                                     boost::ref(restriction_exceptions));
+        const unsigned exception_count = restriction_exceptions.size();
+        SimpleLogger().Write() << "Found " << exception_count
+                               << " exceptions to turn restrictions:";
+        for (const std::string &str : restriction_exceptions)
+        {
+            SimpleLogger().Write() << "  " << str;
+        }
+    }
+    else
+    {
+        SimpleLogger().Write() << "Found no exceptions to turn restrictions";
+    }
+}
+
+/**
+ * Tries to parse an relation as turn restriction. This can fail for a number of
+ * reasons, this the return type is a boost::optional<T>.
+ *
+ * Some restrictions can also be ignored: See the ```get_exceptions``` function
+ * in the corresponding profile.
+ */
+boost::optional<InputRestrictionContainer>
+RestrictionParser::TryParse(const osmium::Relation &relation) const
+{
+    // return if turn restrictions should be ignored
+    if (!use_turn_restrictions)
+    {
+        return {};
+    }
+
+    osmium::tags::KeyPrefixFilter filter(false);
+    filter.add(true, "restriction");
+
+    const osmium::TagList &tag_list = relation.tags();
+
+    osmium::tags::KeyPrefixFilter::iterator fi_begin(filter, tag_list.begin(), tag_list.end());
+    osmium::tags::KeyPrefixFilter::iterator fi_end(filter, tag_list.end(), tag_list.end());
+
+    // if it's a restriction, continue;
+    if (std::distance(fi_begin, fi_end) == 0)
+    {
+        return {};
+    }
+
+    // check if the restriction should be ignored
+    const char *except = relation.get_value_by_key("except");
+    if (except != nullptr && ShouldIgnoreRestriction(except))
+    {
+        return {};
+    }
+
+    bool is_only_restriction = false;
+
+    for (; fi_begin != fi_end; ++fi_begin)
+    {
+        const std::string key(fi_begin->key());
+        const std::string value(fi_begin->value());
+
+        if (value.find("only_") == 0)
+        {
+            is_only_restriction = true;
+        }
+
+        // if the "restriction*" key is longer than 11 chars, it is a conditional exception (i.e.
+        // "restriction:<transportation_type>")
+        if (key.size() > 11)
+        {
+            const auto ex_suffix = [&](const std::string &exception)
+            {
+                return boost::algorithm::ends_with(key, exception);
+            };
+            bool is_actually_restricted =
+                std::any_of(begin(restriction_exceptions), end(restriction_exceptions), ex_suffix);
+
+            if (!is_actually_restricted)
+            {
+                return {};
+            }
+        }
+    }
+
+    InputRestrictionContainer restriction_container(is_only_restriction);
+
+    for (const auto &member : relation.members())
+    {
+        const char *role = member.role();
+        if (strcmp("from", role) != 0 && strcmp("to", role) != 0 && strcmp("via", role) != 0)
+        {
+            continue;
+        }
+
+        switch (member.type())
+        {
+        case osmium::item_type::node:
+            // Make sure nodes appear only in the role if a via node
+            if (0 == strcmp("from", role) || 0 == strcmp("to", role))
+            {
+                continue;
+            }
+            BOOST_ASSERT(0 == strcmp("via", role));
+
+            // set via node id
+            restriction_container.restriction.via.node = member.ref();
+            break;
+
+        case osmium::item_type::way:
+            BOOST_ASSERT(0 == strcmp("from", role) || 0 == strcmp("to", role) ||
+                         0 == strcmp("via", role));
+            if (0 == strcmp("from", role))
+            {
+                restriction_container.restriction.from.way = member.ref();
+            }
+            else if (0 == strcmp("to", role))
+            {
+                restriction_container.restriction.to.way = member.ref();
+            }
+            // else if (0 == strcmp("via", role))
+            // {
+            //     not yet suppported
+            //     restriction_container.restriction.via.way = member.ref();
+            // }
+            break;
+        case osmium::item_type::relation:
+            // not yet supported, but who knows what the future holds...
+            break;
+        default:
+            // shouldn't ever happen
+            break;
+        }
+    }
+    return boost::make_optional(std::move(restriction_container));
+}
+
+bool RestrictionParser::ShouldIgnoreRestriction(const std::string &except_tag_string) const
+{
+    // should this restriction be ignored? yes if there's an overlap between:
+    // a) the list of modes in the except tag of the restriction
+    //    (except_tag_string), eg: except=bus;bicycle
+    // b) the lua profile defines a hierachy of modes,
+    //    eg: [access, vehicle, bicycle]
+
+    if (except_tag_string.empty())
+    {
+        return false;
+    }
+
+    // Be warned, this is quadratic work here, but we assume that
+    // only a few exceptions are actually defined.
+    std::vector<std::string> exceptions;
+    boost::algorithm::split_regex(exceptions, except_tag_string, boost::regex("[;][ ]*"));
+
+    return std::any_of(std::begin(exceptions), std::end(exceptions),
+                       [&](const std::string &current_string)
+                       {
+                           if (std::end(restriction_exceptions) !=
+                               std::find(std::begin(restriction_exceptions),
+                                         std::end(restriction_exceptions), current_string))
+                           {
+                               return true;
+                           }
+                           return false;
+                       });
+}

src/extractor/scripting_environment.cpp

@@ -0,0 +1,175 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "scripting_environment.hpp"
+
+#include "extraction_helper_functions.hpp"
+#include "extraction_node.hpp"
+#include "extraction_way.hpp"
+#include "internal_extractor_edge.hpp"
+#include "../data_structures/external_memory_node.hpp"
+#include "../data_structures/raster_source.hpp"
+#include "../util/lua_util.hpp"
+#include "../util/osrm_exception.hpp"
+#include "../util/simple_logger.hpp"
+#include "../typedefs.h"
+
+#include <luabind/tag_function.hpp>
+#include <luabind/operator.hpp>
+
+#include <osmium/osm.hpp>
+
+#include <sstream>
+namespace
+{
+// wrapper method as luabind doesn't automatically overload funcs w/ default parameters
+template <class T>
+auto get_value_by_key(T const &object, const char *key) -> decltype(object.get_value_by_key(key))
+{
+    return object.get_value_by_key(key, "");
+}
+
+int lua_error_callback(lua_State *L) // This is so I can use my own function as an
+// exception handler, pcall_log()
+{
+    std::string error_msg = lua_tostring(L, -1);
+    std::ostringstream error_stream;
+    error_stream << error_msg;
+    throw osrm::exception("ERROR occurred in profile script:\n" + error_stream.str());
+}
+}
+
+ScriptingEnvironment::ScriptingEnvironment(const std::string &file_name) : file_name(file_name)
+{
+    SimpleLogger().Write() << "Using script " << file_name;
+}
+
+void ScriptingEnvironment::init_lua_state(lua_State *lua_state)
+{
+    typedef double (osmium::Location::*location_member_ptr_type)() const;
+
+    luabind::open(lua_state);
+    // open utility libraries string library;
+    luaL_openlibs(lua_state);
+
+    luaAddScriptFolderToLoadPath(lua_state, file_name.c_str());
+
+    // Add our function to the state's global scope
+    luabind::module(lua_state)[
+        luabind::def("print", LUA_print<std::string>),
+        luabind::def("durationIsValid", durationIsValid),
+        luabind::def("parseDuration", parseDuration),
+        luabind::class_<SourceContainer>("sources")
+            .def(luabind::constructor<>())
+            .def("load", &SourceContainer::loadRasterSource)
+            .def("query", &SourceContainer::getRasterDataFromSource)
+            .def("interpolate", &SourceContainer::getRasterInterpolateFromSource),
+        luabind::class_<const float>("constants")
+            .enum_("enums")[luabind::value("precision", COORDINATE_PRECISION)],
+
+        luabind::class_<std::vector<std::string>>("vector")
+            .def("Add", static_cast<void (std::vector<std::string>::*)(const std::string &)>(
+                            &std::vector<std::string>::push_back)),
+
+        luabind::class_<osmium::Location>("Location")
+            .def<location_member_ptr_type>("lat", &osmium::Location::lat)
+            .def<location_member_ptr_type>("lon", &osmium::Location::lon),
+
+        luabind::class_<osmium::Node>("Node")
+            // .def<node_member_ptr_type>("tags", &osmium::Node::tags)
+            .def("location", &osmium::Node::location)
+            .def("get_value_by_key", &osmium::Node::get_value_by_key)
+            .def("get_value_by_key", &get_value_by_key<osmium::Node>)
+            .def("id", &osmium::Node::id),
+
+        luabind::class_<ExtractionNode>("ResultNode")
+            .def_readwrite("traffic_lights", &ExtractionNode::traffic_lights)
+            .def_readwrite("barrier", &ExtractionNode::barrier),
+
+        luabind::class_<ExtractionWay>("ResultWay")
+            // .def(luabind::constructor<>())
+            .def_readwrite("forward_speed", &ExtractionWay::forward_speed)
+            .def_readwrite("backward_speed", &ExtractionWay::backward_speed)
+            .def_readwrite("name", &ExtractionWay::name)
+            .def_readwrite("roundabout", &ExtractionWay::roundabout)
+            .def_readwrite("is_access_restricted", &ExtractionWay::is_access_restricted)
+            .def_readwrite("is_startpoint", &ExtractionWay::is_startpoint)
+            .def_readwrite("duration", &ExtractionWay::duration)
+            .property("forward_mode", &ExtractionWay::get_forward_mode,
+                      &ExtractionWay::set_forward_mode)
+            .property("backward_mode", &ExtractionWay::get_backward_mode,
+                      &ExtractionWay::set_backward_mode)
+            .enum_("constants")[
+                luabind::value("notSure", 0),
+                luabind::value("oneway", 1),
+                luabind::value("bidirectional", 2),
+                luabind::value("opposite", 3)
+            ],
+        luabind::class_<osmium::Way>("Way")
+            .def("get_value_by_key", &osmium::Way::get_value_by_key)
+            .def("get_value_by_key", &get_value_by_key<osmium::Way>)
+            .def("id", &osmium::Way::id),
+        luabind::class_<InternalExtractorEdge>("EdgeSource")
+            .property("source_coordinate", &InternalExtractorEdge::source_coordinate)
+            .property("weight_data", &InternalExtractorEdge::weight_data),
+        luabind::class_<InternalExtractorEdge::WeightData>("WeightData")
+            .def_readwrite("speed", &InternalExtractorEdge::WeightData::speed),
+        luabind::class_<ExternalMemoryNode>("EdgeTarget")
+            .property("lat", &ExternalMemoryNode::lat)
+            .property("lon", &ExternalMemoryNode::lon),
+        luabind::class_<FixedPointCoordinate>("Coordinate")
+            .property("lat", &FixedPointCoordinate::lat)
+            .property("lon", &FixedPointCoordinate::lon),
+        luabind::class_<RasterDatum>("RasterDatum")
+            .property("datum", &RasterDatum::datum)
+            .def("invalid_data", &RasterDatum::get_invalid)
+    ];
+
+    if (0 != luaL_dofile(lua_state, file_name.c_str()))
+    {
+        luabind::object error_msg(luabind::from_stack(lua_state, -1));
+        std::ostringstream error_stream;
+        error_stream << error_msg;
+        throw osrm::exception("ERROR occurred in profile script:\n" + error_stream.str());
+    }
+}
+
+lua_State *ScriptingEnvironment::get_lua_state()
+{
+    std::lock_guard<std::mutex> lock(init_mutex);
+    bool initialized = false;
+    auto &ref = script_contexts.local(initialized);
+    if (!initialized)
+    {
+        std::shared_ptr<lua_State> state(luaL_newstate(), lua_close);
+        ref = state;
+        init_lua_state(ref.get());
+    }
+    luabind::set_pcall_callback(&lua_error_callback);
+
+    return ref.get();
+}

src/server/connection.cpp

@@ -0,0 +1,170 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "connection.hpp"
+#include "request_handler.hpp"
+#include "request_parser.hpp"
+
+#include <boost/assert.hpp>
+#include <boost/bind.hpp>
+#include <boost/iostreams/filtering_stream.hpp>
+#include <boost/iostreams/filter/gzip.hpp>
+
+#include <string>
+#include <vector>
+
+namespace http
+{
+
+Connection::Connection(boost::asio::io_service &io_service, RequestHandler &handler)
+    : strand(io_service), TCP_socket(io_service), request_handler(handler)
+{
+}
+
+boost::asio::ip::tcp::socket &Connection::socket() { return TCP_socket; }
+
+/// Start the first asynchronous operation for the connection.
+void Connection::start()
+{
+    TCP_socket.async_read_some(
+        boost::asio::buffer(incoming_data_buffer),
+        strand.wrap(boost::bind(&Connection::handle_read, this->shared_from_this(),
+                                boost::asio::placeholders::error,
+                                boost::asio::placeholders::bytes_transferred)));
+}
+
+void Connection::handle_read(const boost::system::error_code &error, std::size_t bytes_transferred)
+{
+    if (error)
+    {
+        return;
+    }
+
+    // no error detected, let's parse the request
+    compression_type compression_type(no_compression);
+    osrm::tribool result;
+    std::tie(result, compression_type) =
+        request_parser.parse(current_request, incoming_data_buffer.data(),
+                             incoming_data_buffer.data() + bytes_transferred);
+
+    // the request has been parsed
+    if (result == osrm::tribool::yes)
+    {
+        current_request.endpoint = TCP_socket.remote_endpoint().address();
+        request_handler.handle_request(current_request, current_reply);
+
+        // Header compression_header;
+        std::vector<boost::asio::const_buffer> output_buffer;
+
+        // compress the result w/ gzip/deflate if requested
+        switch (compression_type)
+        {
+        case deflate_rfc1951:
+            // use deflate for compression
+            current_reply.headers.insert(current_reply.headers.begin(),
+                                         {"Content-Encoding", "deflate"});
+            compressed_output = compress_buffers(current_reply.content, compression_type);
+            current_reply.set_size(static_cast<unsigned>(compressed_output.size()));
+            output_buffer = current_reply.headers_to_buffers();
+            output_buffer.push_back(boost::asio::buffer(compressed_output));
+            break;
+        case gzip_rfc1952:
+            // use gzip for compression
+            current_reply.headers.insert(current_reply.headers.begin(),
+                                         {"Content-Encoding", "gzip"});
+            compressed_output = compress_buffers(current_reply.content, compression_type);
+            current_reply.set_size(static_cast<unsigned>(compressed_output.size()));
+            output_buffer = current_reply.headers_to_buffers();
+            output_buffer.push_back(boost::asio::buffer(compressed_output));
+            break;
+        case no_compression:
+            // don't use any compression
+            current_reply.set_uncompressed_size();
+            output_buffer = current_reply.to_buffers();
+            break;
+        }
+        // write result to stream
+        boost::asio::async_write(
+            TCP_socket, output_buffer,
+            strand.wrap(boost::bind(&Connection::handle_write, this->shared_from_this(),
+                                    boost::asio::placeholders::error)));
+    }
+    else if (result == osrm::tribool::no)
+    { // request is not parseable
+        current_reply = reply::stock_reply(reply::bad_request);
+
+        boost::asio::async_write(
+            TCP_socket, current_reply.to_buffers(),
+            strand.wrap(boost::bind(&Connection::handle_write, this->shared_from_this(),
+                                    boost::asio::placeholders::error)));
+    }
+    else
+    {
+        // we don't have a result yet, so continue reading
+        TCP_socket.async_read_some(
+            boost::asio::buffer(incoming_data_buffer),
+            strand.wrap(boost::bind(&Connection::handle_read, this->shared_from_this(),
+                                    boost::asio::placeholders::error,
+                                    boost::asio::placeholders::bytes_transferred)));
+    }
+}
+
+/// Handle completion of a write operation.
+void Connection::handle_write(const boost::system::error_code &error)
+{
+    if (!error)
+    {
+        // Initiate graceful connection closure.
+        boost::system::error_code ignore_error;
+        TCP_socket.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignore_error);
+    }
+}
+
+std::vector<char> Connection::compress_buffers(const std::vector<char> &uncompressed_data,
+                                               const compression_type compression_type)
+{
+    boost::iostreams::gzip_params compression_parameters;
+
+    // there's a trade-off between speed and size. speed wins
+    compression_parameters.level = boost::iostreams::zlib::best_speed;
+    // check which compression flavor is used
+    if (deflate_rfc1951 == compression_type)
+    {
+        compression_parameters.noheader = true;
+    }
+
+    std::vector<char> compressed_data;
+    // plug data into boost's compression stream
+    boost::iostreams::filtering_ostream gzip_stream;
+    gzip_stream.push(boost::iostreams::gzip_compressor(compression_parameters));
+    gzip_stream.push(boost::iostreams::back_inserter(compressed_data));
+    gzip_stream.write(&uncompressed_data[0], uncompressed_data.size());
+    boost::iostreams::close(gzip_stream);
+
+    return compressed_data;
+}
+}

src/server/http/reply.cpp

@@ -0,0 +1,130 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "reply.hpp"
+
+#include <string>
+
+namespace http
+{
+
+const char ok_html[] = "";
+const char bad_request_html[] = "{\"status\": 400,\"status_message\":\"Bad Request\"}";
+const char internal_server_error_html[] =
+    "{\"status\": 500,\"status_message\":\"Internal Server Error\"}";
+const char seperators[] = {':', ' '};
+const char crlf[] = {'\r', '\n'};
+const std::string http_ok_string = "HTTP/1.0 200 OK\r\n";
+const std::string http_bad_request_string = "HTTP/1.0 400 Bad Request\r\n";
+const std::string http_internal_server_error_string = "HTTP/1.0 500 Internal Server Error\r\n";
+
+void reply::set_size(const std::size_t size)
+{
+    for (header &h : headers)
+    {
+        if ("Content-Length" == h.name)
+        {
+            h.value = std::to_string(size);
+        }
+    }
+}
+
+void reply::set_uncompressed_size() { set_size(content.size()); }
+
+std::vector<boost::asio::const_buffer> reply::to_buffers()
+{
+    std::vector<boost::asio::const_buffer> buffers;
+    buffers.push_back(status_to_buffer(status));
+    for (const header &h : headers)
+    {
+        buffers.push_back(boost::asio::buffer(h.name));
+        buffers.push_back(boost::asio::buffer(seperators));
+        buffers.push_back(boost::asio::buffer(h.value));
+        buffers.push_back(boost::asio::buffer(crlf));
+    }
+    buffers.push_back(boost::asio::buffer(crlf));
+    buffers.push_back(boost::asio::buffer(content));
+    return buffers;
+}
+
+std::vector<boost::asio::const_buffer> reply::headers_to_buffers()
+{
+    std::vector<boost::asio::const_buffer> buffers;
+    buffers.push_back(status_to_buffer(status));
+    for (const header &current_header : headers)
+    {
+        buffers.push_back(boost::asio::buffer(current_header.name));
+        buffers.push_back(boost::asio::buffer(seperators));
+        buffers.push_back(boost::asio::buffer(current_header.value));
+        buffers.push_back(boost::asio::buffer(crlf));
+    }
+    buffers.push_back(boost::asio::buffer(crlf));
+    return buffers;
+}
+
+reply reply::stock_reply(const reply::status_type status)
+{
+    reply reply;
+    reply.status = status;
+    reply.content.clear();
+
+    const std::string status_string = reply.status_to_string(status);
+    reply.content.insert(reply.content.end(), status_string.begin(), status_string.end());
+    reply.headers.emplace_back("Access-Control-Allow-Origin", "*");
+    reply.headers.emplace_back("Content-Length", std::to_string(reply.content.size()));
+    reply.headers.emplace_back("Content-Type", "text/html");
+    return reply;
+}
+
+std::string reply::status_to_string(const reply::status_type status)
+{
+    if (reply::ok == status)
+    {
+        return ok_html;
+    }
+    if (reply::bad_request == status)
+    {
+        return bad_request_html;
+    }
+    return internal_server_error_html;
+}
+
+boost::asio::const_buffer reply::status_to_buffer(const reply::status_type status)
+{
+    if (reply::ok == status)
+    {
+        return boost::asio::buffer(http_ok_string);
+    }
+    if (reply::internal_server_error == status)
+    {
+        return boost::asio::buffer(http_internal_server_error_string);
+    }
+    return boost::asio::buffer(http_bad_request_string);
+}
+
+reply::reply() : status(ok) {}
+}

src/server/request_handler.cpp

@@ -0,0 +1,177 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "request_handler.hpp"
+
+#include "api_grammar.hpp"
+#include "http/reply.hpp"
+#include "http/request.hpp"
+
+#include "../util/json_renderer.hpp"
+#include "../util/simple_logger.hpp"
+#include "../util/string_util.hpp"
+#include "../util/xml_renderer.hpp"
+#include "../typedefs.h"
+
+#include <osrm/route_parameters.hpp>
+#include <osrm/json_container.hpp>
+#include <osrm/osrm.hpp>
+
+#include <ctime>
+
+#include <algorithm>
+#include <iostream>
+#include <string>
+
+RequestHandler::RequestHandler() : routing_machine(nullptr) {}
+
+void RequestHandler::handle_request(const http::request &current_request,
+                                    http::reply &current_reply)
+{
+    osrm::json::Object json_result;
+
+    // parse command
+    try
+    {
+        std::string request_string;
+        URIDecode(current_request.uri, request_string);
+
+        // deactivated as GCC apparently does not implement that, not even in 4.9
+        // std::time_t t = std::time(nullptr);
+        // SimpleLogger().Write() << std::put_time(std::localtime(&t), "%m-%d-%Y %H:%M:%S") <<
+        //     " " << current_request.endpoint.to_string() << " " <<
+        //     current_request.referrer << ( 0 == current_request.referrer.length() ? "- " :" ") <<
+        //     current_request.agent << ( 0 == current_request.agent.length() ? "- " :" ") <<
+        //     request;
+
+        time_t ltime;
+        struct tm *time_stamp;
+
+        ltime = time(nullptr);
+        time_stamp = localtime(&ltime);
+
+        // log timestamp
+        SimpleLogger().Write() << (time_stamp->tm_mday < 10 ? "0" : "") << time_stamp->tm_mday
+                               << "-" << (time_stamp->tm_mon + 1 < 10 ? "0" : "")
+                               << (time_stamp->tm_mon + 1) << "-" << 1900 + time_stamp->tm_year
+                               << " " << (time_stamp->tm_hour < 10 ? "0" : "")
+                               << time_stamp->tm_hour << ":" << (time_stamp->tm_min < 10 ? "0" : "")
+                               << time_stamp->tm_min << ":" << (time_stamp->tm_sec < 10 ? "0" : "")
+                               << time_stamp->tm_sec << " " << current_request.endpoint.to_string()
+                               << " " << current_request.referrer
+                               << (0 == current_request.referrer.length() ? "- " : " ")
+                               << current_request.agent
+                               << (0 == current_request.agent.length() ? "- " : " ")
+                               << request_string;
+
+        RouteParameters route_parameters;
+        APIGrammarParser api_parser(&route_parameters);
+
+        auto api_iterator = request_string.begin();
+        const bool result =
+            boost::spirit::qi::parse(api_iterator, request_string.end(), api_parser);
+
+        // check if the was an error with the request
+        if (result && api_iterator == request_string.end())
+        {
+            // parsing done, lets call the right plugin to handle the request
+            BOOST_ASSERT_MSG(routing_machine != nullptr, "pointer not init'ed");
+
+            if (!route_parameters.jsonp_parameter.empty())
+            { // prepend response with jsonp parameter
+                const std::string json_p = (route_parameters.jsonp_parameter + "(");
+                current_reply.content.insert(current_reply.content.end(), json_p.begin(), json_p.end());
+            }
+
+            const int return_code = routing_machine->RunQuery(route_parameters, json_result);
+            json_result.values["status"] = return_code;
+            // 4xx bad request return code
+            if (return_code / 100 == 4)
+            {
+                current_reply.status = http::reply::bad_request;
+                current_reply.content.clear();
+                route_parameters.output_format.clear();
+            }
+            else
+            {
+                // 2xx valid request
+                BOOST_ASSERT(return_code / 100 == 2);
+            }
+        }
+        else
+        {
+            const auto position = std::distance(request_string.begin(), api_iterator);
+
+            current_reply.status = http::reply::bad_request;
+            json_result.values["status"] = http::reply::bad_request;
+            json_result.values["status_message"] = "Query string malformed close to position " + std::to_string(position);
+        }
+
+        current_reply.headers.emplace_back("Access-Control-Allow-Origin", "*");
+        current_reply.headers.emplace_back("Access-Control-Allow-Methods", "GET");
+        current_reply.headers.emplace_back("Access-Control-Allow-Headers",
+                                           "X-Requested-With, Content-Type");
+
+        // set headers
+        current_reply.headers.emplace_back("Content-Length",
+                                           std::to_string(current_reply.content.size()));
+        if ("gpx" == route_parameters.output_format)
+        { // gpx file
+            osrm::json::gpx_render(current_reply.content, json_result.values["route"]);
+            current_reply.headers.emplace_back("Content-Type",
+                                               "application/gpx+xml; charset=UTF-8");
+            current_reply.headers.emplace_back("Content-Disposition",
+                                               "attachment; filename=\"route.gpx\"");
+        }
+        else if (route_parameters.jsonp_parameter.empty())
+        { // json file
+            osrm::json::render(current_reply.content, json_result);
+            current_reply.headers.emplace_back("Content-Type", "application/json; charset=UTF-8");
+            current_reply.headers.emplace_back("Content-Disposition",
+                                               "inline; filename=\"response.json\"");
+        }
+        else
+        { // jsonp
+            osrm::json::render(current_reply.content, json_result);
+            current_reply.headers.emplace_back("Content-Type", "text/javascript; charset=UTF-8");
+            current_reply.headers.emplace_back("Content-Disposition",
+                                               "inline; filename=\"response.js\"");
+        }
+        if (!route_parameters.jsonp_parameter.empty())
+        { // append brace to jsonp response
+            current_reply.content.push_back(')');
+        }
+    }
+    catch (const std::exception &e)
+    {
+        current_reply = http::reply::stock_reply(http::reply::internal_server_error);;
+        SimpleLogger().Write(logWARNING) << "[server error] code: " << e.what()
+                                         << ", uri: " << current_request.uri;
+    }
+}
+
+void RequestHandler::RegisterRoutingMachine(OSRM *osrm) { routing_machine = osrm; }

src/server/request_parser.cpp

@@ -0,0 +1,393 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "request_parser.hpp"
+
+#include "http/compression_type.hpp"
+#include "http/header.hpp"
+#include "http/request.hpp"
+
+#include "../data_structures/tribool.hpp"
+
+#include <boost/algorithm/string/predicate.hpp>
+
+#include <string>
+
+namespace http
+{
+
+RequestParser::RequestParser()
+    : state(internal_state::method_start), current_header({"", ""}),
+      selected_compression(no_compression), is_post_header(false),
+      content_length(0)
+{
+}
+
+std::tuple<osrm::tribool, compression_type>
+RequestParser::parse(request &current_request, char *begin, char *end)
+{
+    while (begin != end)
+    {
+        osrm::tribool result = consume(current_request, *begin++);
+        if (result != osrm::tribool::indeterminate)
+        {
+            return std::make_tuple(result, selected_compression);
+        }
+    }
+    osrm::tribool result = osrm::tribool::indeterminate;
+    
+    if(state == internal_state::post_request && content_length <= 0)
+    {
+        result = osrm::tribool::yes;
+    }
+    return std::make_tuple(result, selected_compression);
+}
+
+osrm::tribool RequestParser::consume(request &current_request, const char input)
+{
+    switch (state)
+    {
+    case internal_state::method_start:
+        if (!is_char(input) || is_CTL(input) || is_special(input))
+        {
+            return osrm::tribool::no;
+        }
+        if(input == 'P')
+        {
+            state = internal_state::post_O;
+            return osrm::tribool::indeterminate;
+        }
+        state = internal_state::method;
+        return osrm::tribool::indeterminate;
+    case internal_state::post_O:
+        if(input == 'O')
+        {
+          state = internal_state::post_S;
+          return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::post_S:
+        if(input == 'S')
+        {
+          state = internal_state::post_T;
+          return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::post_T:
+        if(input == 'T')
+        {
+          is_post_header = true;
+          state = internal_state::method;
+          return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::post_request:
+        current_request.uri.push_back(input);
+        --content_length;
+        return osrm::tribool::indeterminate;
+    case internal_state::method:
+        if (input == ' ')
+        {
+            state = internal_state::uri;
+            return osrm::tribool::indeterminate;
+        }
+        if (!is_char(input) || is_CTL(input) || is_special(input))
+        {
+            return osrm::tribool::no;
+        }
+        return osrm::tribool::indeterminate;
+    case internal_state::uri_start:
+        if (is_CTL(input))
+        {
+            return osrm::tribool::no;
+        }
+        state = internal_state::uri;
+        current_request.uri.push_back(input);
+        return osrm::tribool::indeterminate;
+    case internal_state::uri:
+        if (input == ' ')
+        {
+            state = internal_state::http_version_h;
+            return osrm::tribool::indeterminate;
+        }
+        if (is_CTL(input))
+        {
+            return osrm::tribool::no;
+        }
+        current_request.uri.push_back(input);
+        return osrm::tribool::indeterminate;
+    case internal_state::http_version_h:
+        if (input == 'H')
+        {
+            state = internal_state::http_version_t_1;
+            return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::http_version_t_1:
+        if (input == 'T')
+        {
+            state = internal_state::http_version_t_2;
+            return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::http_version_t_2:
+        if (input == 'T')
+        {
+            state = internal_state::http_version_p;
+            return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::http_version_p:
+        if (input == 'P')
+        {
+            state = internal_state::http_version_slash;
+            return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::http_version_slash:
+        if (input == '/')
+        {
+            state = internal_state::http_version_major_start;
+            return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::http_version_major_start:
+        if (is_digit(input))
+        {
+            state = internal_state::http_version_major;
+            return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::http_version_major:
+        if (input == '.')
+        {
+            state = internal_state::http_version_minor_start;
+            return osrm::tribool::indeterminate;
+        }
+        if (is_digit(input))
+        {
+            return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::http_version_minor_start:
+        if (is_digit(input))
+        {
+            state = internal_state::http_version_minor;
+            return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::http_version_minor:
+        if (input == '\r')
+        {
+            state = internal_state::expecting_newline_1;
+            return osrm::tribool::indeterminate;
+        }
+        if (is_digit(input))
+        {
+            return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::expecting_newline_1:
+        if (input == '\n')
+        {
+            state = internal_state::header_line_start;
+            return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::header_line_start:
+        if (boost::iequals(current_header.name, "Accept-Encoding"))
+        {
+            /* giving gzip precedence over deflate */
+            if (boost::icontains(current_header.value, "deflate"))
+            {
+                selected_compression = deflate_rfc1951;
+            }
+            if (boost::icontains(current_header.value, "gzip"))
+            {
+                selected_compression = gzip_rfc1952;
+            }
+        }
+
+        if (boost::iequals(current_header.name, "Referer"))
+        {
+            current_request.referrer = current_header.value;
+        }
+
+        if (boost::iequals(current_header.name, "User-Agent"))
+        {
+            current_request.agent = current_header.value;
+        }
+        if (boost::iequals(current_header.name, "Content-Length"))
+        {
+            try 
+            {
+                content_length = std::stoi(current_header.value);
+            }
+            catch (const std::exception &e)
+            {
+                // Ignore the header if the parameter isn't an int
+            }
+        }
+        if (boost::iequals(current_header.name, "Content-Type"))
+        {
+            if (!boost::icontains(current_header.value, "application/x-www-form-urlencoded"))
+            {
+                return osrm::tribool::no;
+            }
+        }
+
+        if (input == '\r')
+        {
+            state = internal_state::expecting_newline_3;
+            return osrm::tribool::indeterminate;
+        }
+        if (!is_char(input) || is_CTL(input) || is_special(input))
+        {
+            return osrm::tribool::no;
+        }
+        state = internal_state::header_name;
+        current_header.clear();
+        current_header.name.push_back(input);
+        return osrm::tribool::indeterminate;
+    case internal_state::header_lws:
+        if (input == '\r')
+        {
+            state = internal_state::expecting_newline_2;
+            return osrm::tribool::indeterminate;
+        }
+        if (input == ' ' || input == '\t')
+        {
+            return osrm::tribool::indeterminate;
+        }
+        if (is_CTL(input))
+        {
+            return osrm::tribool::no;
+        }
+        state = internal_state::header_value;
+        return osrm::tribool::indeterminate;
+    case internal_state::header_name:
+        if (input == ':')
+        {
+            state = internal_state::space_before_header_value;
+            return osrm::tribool::indeterminate;
+        }
+        if (!is_char(input) || is_CTL(input) || is_special(input))
+        {
+            return osrm::tribool::no;
+        }
+        current_header.name.push_back(input);
+        return osrm::tribool::indeterminate;
+    case internal_state::space_before_header_value:
+        if (input == ' ')
+        {
+            state = internal_state::header_value;
+            return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::header_value:
+        if (input == '\r')
+        {
+            state = internal_state::expecting_newline_2;
+            return osrm::tribool::indeterminate;
+        }
+        if (is_CTL(input))
+        {
+            return osrm::tribool::no;
+        }
+        current_header.value.push_back(input);
+        return osrm::tribool::indeterminate;
+    case internal_state::expecting_newline_2:
+        if (input == '\n')
+        {
+            state = internal_state::header_line_start;
+            return osrm::tribool::indeterminate;
+        }
+        return osrm::tribool::no;
+    case internal_state::expecting_newline_3:
+        if (input == '\n')
+        {
+            if (is_post_header)
+            {
+		if (content_length > 0)
+		{
+		    current_request.uri.push_back('?');
+		}
+                state = internal_state::post_request;
+                return osrm::tribool::indeterminate;
+            }
+            return osrm::tribool::yes;
+        }
+        return osrm::tribool::no;
+    default: // should never be reached
+        return input == '\n' ? osrm::tribool::yes : osrm::tribool::no;
+    }
+}
+
+bool RequestParser::is_char(const int character) const
+{
+    return character >= 0 && character <= 127;
+}
+
+bool RequestParser::is_CTL(const int character) const
+{
+    return (character >= 0 && character <= 31) || (character == 127);
+}
+
+bool RequestParser::is_special(const int character) const
+{
+    switch (character)
+    {
+    case '(':
+    case ')':
+    case '<':
+    case '>':
+    case '@':
+    case ',':
+    case ';':
+    case ':':
+    case '\\':
+    case '"':
+    case '/':
+    case '[':
+    case ']':
+    case '?':
+    case '=':
+    case '{':
+    case '}':
+    case ' ':
+    case '\t':
+        return true;
+    default:
+        return false;
+    }
+}
+
+bool RequestParser::is_digit(const int character) const
+{
+    return character >= '0' && character <= '9';
+}
+}

src/tools/.gitignore

@@ -0,0 +1 @@
+/osrm-component

src/tools/check-hsgr.cpp

@@ -0,0 +1,112 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+#include "../data_structures/percent.hpp"
+#include "../data_structures/query_edge.hpp"
+#include "../data_structures/static_graph.hpp"
+#include "../util/integer_range.hpp"
+#include "../util/graph_loader.hpp"
+#include "../util/simple_logger.hpp"
+#include "../util/osrm_exception.hpp"
+
+#include <boost/assert.hpp>
+#include <boost/filesystem.hpp>
+
+#include <memory>
+#include <vector>
+
+using EdgeData = QueryEdge::EdgeData;
+using QueryGraph = StaticGraph<EdgeData>;
+
+int main(int argc, char *argv[])
+{
+    LogPolicy::GetInstance().Unmute();
+    try
+    {
+        if (argc != 2)
+        {
+            SimpleLogger().Write(logWARNING) << "usage: " << argv[0] << " <file.hsgr>";
+            return 1;
+        }
+
+        boost::filesystem::path hsgr_path(argv[1]);
+
+        std::vector<QueryGraph::NodeArrayEntry> node_list;
+        std::vector<QueryGraph::EdgeArrayEntry> edge_list;
+        SimpleLogger().Write() << "loading graph from " << hsgr_path.string();
+
+        unsigned m_check_sum = 0;
+        unsigned m_number_of_nodes =
+            readHSGRFromStream(hsgr_path, node_list, edge_list, &m_check_sum);
+        SimpleLogger().Write() << "expecting " << m_number_of_nodes
+                               << " nodes, checksum: " << m_check_sum;
+        BOOST_ASSERT_MSG(0 != node_list.size(), "node list empty");
+        SimpleLogger().Write() << "loaded " << node_list.size() << " nodes and " << edge_list.size()
+                               << " edges";
+        auto m_query_graph = std::make_shared<QueryGraph>(node_list, edge_list);
+
+        BOOST_ASSERT_MSG(0 == node_list.size(), "node list not flushed");
+        BOOST_ASSERT_MSG(0 == edge_list.size(), "edge list not flushed");
+
+        Percent progress(m_query_graph->GetNumberOfNodes());
+        for (const auto node_u : osrm::irange(0u, m_query_graph->GetNumberOfNodes()))
+        {
+            for (const auto eid : m_query_graph->GetAdjacentEdgeRange(node_u))
+            {
+                const EdgeData &data = m_query_graph->GetEdgeData(eid);
+                if (!data.shortcut)
+                {
+                    continue;
+                }
+                const unsigned node_v = m_query_graph->GetTarget(eid);
+                const EdgeID edge_id_1 = m_query_graph->FindEdgeInEitherDirection(node_u, data.id);
+                if (SPECIAL_EDGEID == edge_id_1)
+                {
+                    throw osrm::exception("cannot find first segment of edge (" +
+                                          std::to_string(node_u) + "," + std::to_string(data.id) +
+                                          "," + std::to_string(node_v) + "), eid: " +
+                                          std::to_string(eid));
+                }
+                const EdgeID edge_id_2 = m_query_graph->FindEdgeInEitherDirection(data.id, node_v);
+                if (SPECIAL_EDGEID == edge_id_2)
+                {
+                    throw osrm::exception("cannot find second segment of edge (" +
+                                          std::to_string(node_u) + "," + std::to_string(data.id) +
+                                          "," + std::to_string(node_v) + "), eid: " +
+                                          std::to_string(eid));
+                }
+            }
+            progress.printStatus(node_u);
+        }
+        m_query_graph.reset();
+        SimpleLogger().Write() << "Data file " << argv[0] << " appears to be OK";
+    }
+    catch (const std::exception &e)
+    {
+        SimpleLogger().Write(logWARNING) << "[exception] " << e.what();
+    }
+    return 0;
+}

src/tools/components.cpp

@@ -0,0 +1,257 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "../typedefs.h"
+#include "../algorithms/tarjan_scc.hpp"
+#include "../algorithms/coordinate_calculation.hpp"
+#include "../data_structures/dynamic_graph.hpp"
+#include "../data_structures/static_graph.hpp"
+#include "../util/fingerprint.hpp"
+#include "../util/graph_loader.hpp"
+#include "../util/make_unique.hpp"
+#include "../util/osrm_exception.hpp"
+#include "../util/simple_logger.hpp"
+
+#include <boost/filesystem.hpp>
+
+#if defined(__APPLE__) || defined(_WIN32)
+#include <gdal.h>
+#include <ogrsf_frmts.h>
+#else
+#include <gdal/gdal.h>
+#include <gdal/ogrsf_frmts.h>
+#endif
+
+#include <osrm/coordinate.hpp>
+
+#include <fstream>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace
+{
+
+struct TarjanEdgeData
+{
+    TarjanEdgeData() : distance(INVALID_EDGE_WEIGHT), name_id(INVALID_NAMEID) {}
+    TarjanEdgeData(unsigned distance, unsigned name_id) : distance(distance), name_id(name_id) {}
+    unsigned distance;
+    unsigned name_id;
+};
+
+using TarjanGraph = StaticGraph<TarjanEdgeData>;
+using TarjanEdge = TarjanGraph::InputEdge;
+
+void DeleteFileIfExists(const std::string &file_name)
+{
+    if (boost::filesystem::exists(file_name))
+    {
+        boost::filesystem::remove(file_name);
+    }
+}
+}
+
+std::size_t LoadGraph(const char *path,
+                      std::vector<QueryNode> &coordinate_list,
+                      std::vector<TarjanEdge> &graph_edge_list)
+{
+    std::ifstream input_stream(path, std::ifstream::in | std::ifstream::binary);
+    if (!input_stream.is_open())
+    {
+        throw osrm::exception("Cannot open osrm file");
+    }
+
+    // load graph data
+    std::vector<NodeBasedEdge> edge_list;
+    std::vector<NodeID> traffic_light_node_list;
+    std::vector<NodeID> barrier_node_list;
+
+    auto number_of_nodes = loadNodesFromFile(input_stream, barrier_node_list,
+                                             traffic_light_node_list, coordinate_list);
+
+    loadEdgesFromFile(input_stream, edge_list);
+
+    traffic_light_node_list.clear();
+    traffic_light_node_list.shrink_to_fit();
+
+    // Building an node-based graph
+    for (const auto &input_edge : edge_list)
+    {
+        if (input_edge.source == input_edge.target)
+        {
+            continue;
+        }
+
+        if (input_edge.forward)
+        {
+            graph_edge_list.emplace_back(input_edge.source, input_edge.target,
+                                         (std::max)(input_edge.weight, 1), input_edge.name_id);
+        }
+        if (input_edge.backward)
+        {
+            graph_edge_list.emplace_back(input_edge.target, input_edge.source,
+                                         (std::max)(input_edge.weight, 1), input_edge.name_id);
+        }
+    }
+
+    return number_of_nodes;
+}
+
+int main(int argc, char *argv[])
+{
+    std::vector<QueryNode> coordinate_list;
+
+    LogPolicy::GetInstance().Unmute();
+    try
+    {
+        // enable logging
+        if (argc < 2)
+        {
+            SimpleLogger().Write(logWARNING) << "usage:\n" << argv[0] << " <osrm>";
+            return -1;
+        }
+
+        std::vector<TarjanEdge> graph_edge_list;
+        auto number_of_nodes = LoadGraph(argv[1], coordinate_list, graph_edge_list);
+
+        tbb::parallel_sort(graph_edge_list.begin(), graph_edge_list.end());
+        const auto graph = std::make_shared<TarjanGraph>(number_of_nodes, graph_edge_list);
+        graph_edge_list.clear();
+        graph_edge_list.shrink_to_fit();
+
+        SimpleLogger().Write() << "Starting SCC graph traversal";
+
+        auto tarjan = osrm::make_unique<TarjanSCC<TarjanGraph>>(graph);
+        tarjan->run();
+        SimpleLogger().Write() << "identified: " << tarjan->get_number_of_components()
+                               << " many components";
+        SimpleLogger().Write() << "identified " << tarjan->get_size_one_count() << " size 1 SCCs";
+
+        // output
+        TIMER_START(SCC_RUN_SETUP);
+
+        // remove files from previous run if exist
+        DeleteFileIfExists("component.dbf");
+        DeleteFileIfExists("component.shx");
+        DeleteFileIfExists("component.shp");
+
+        Percent percentage(graph->GetNumberOfNodes());
+
+        OGRRegisterAll();
+
+        const char *pszDriverName = "ESRI Shapefile";
+        OGRSFDriver *poDriver =
+            OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(pszDriverName);
+        if (nullptr == poDriver)
+        {
+            throw osrm::exception("ESRI Shapefile driver not available");
+        }
+        OGRDataSource *poDS = poDriver->CreateDataSource("component.shp", nullptr);
+
+        if (nullptr == poDS)
+        {
+            throw osrm::exception("Creation of output file failed");
+        }
+
+        OGRSpatialReference *poSRS = new OGRSpatialReference();
+        poSRS->importFromEPSG(4326);
+
+        OGRLayer *poLayer = poDS->CreateLayer("component", poSRS, wkbLineString, nullptr);
+
+        if (nullptr == poLayer)
+        {
+            throw osrm::exception("Layer creation failed.");
+        }
+        TIMER_STOP(SCC_RUN_SETUP);
+        SimpleLogger().Write() << "shapefile setup took " << TIMER_MSEC(SCC_RUN_SETUP) / 1000.
+                               << "s";
+
+        uint64_t total_network_length = 0;
+        percentage.reinit(graph->GetNumberOfNodes());
+        TIMER_START(SCC_OUTPUT);
+        for (const NodeID source : osrm::irange(0u, graph->GetNumberOfNodes()))
+        {
+            percentage.printIncrement();
+            for (const auto current_edge : graph->GetAdjacentEdgeRange(source))
+            {
+                const TarjanGraph::NodeIterator target = graph->GetTarget(current_edge);
+
+                if (source < target || SPECIAL_EDGEID == graph->FindEdge(target, source))
+                {
+                    total_network_length +=
+                        100 * coordinate_calculation::great_circle_distance(
+                                  coordinate_list[source].lat, coordinate_list[source].lon,
+                                  coordinate_list[target].lat, coordinate_list[target].lon);
+
+                    BOOST_ASSERT(current_edge != SPECIAL_EDGEID);
+                    BOOST_ASSERT(source != SPECIAL_NODEID);
+                    BOOST_ASSERT(target != SPECIAL_NODEID);
+
+                    const unsigned size_of_containing_component =
+                        std::min(tarjan->get_component_size(tarjan->get_component_id(source)),
+                                 tarjan->get_component_size(tarjan->get_component_id(target)));
+
+                    // edges that end on bollard nodes may actually be in two distinct components
+                    if (size_of_containing_component < 1000)
+                    {
+                        OGRLineString lineString;
+                        lineString.addPoint(coordinate_list[source].lon / COORDINATE_PRECISION,
+                                            coordinate_list[source].lat / COORDINATE_PRECISION);
+                        lineString.addPoint(coordinate_list[target].lon / COORDINATE_PRECISION,
+                                            coordinate_list[target].lat / COORDINATE_PRECISION);
+
+                        OGRFeature *poFeature = OGRFeature::CreateFeature(poLayer->GetLayerDefn());
+
+                        poFeature->SetGeometry(&lineString);
+                        if (OGRERR_NONE != poLayer->CreateFeature(poFeature))
+                        {
+                            throw osrm::exception("Failed to create feature in shapefile.");
+                        }
+                        OGRFeature::DestroyFeature(poFeature);
+                    }
+                }
+            }
+        }
+        OGRSpatialReference::DestroySpatialReference(poSRS);
+        OGRDataSource::DestroyDataSource(poDS);
+        TIMER_STOP(SCC_OUTPUT);
+        SimpleLogger().Write() << "generating output took: " << TIMER_MSEC(SCC_OUTPUT) / 1000.
+                               << "s";
+
+        SimpleLogger().Write() << "total network distance: "
+                               << static_cast<uint64_t>(total_network_length / 100 / 1000.)
+                               << " km";
+
+        SimpleLogger().Write() << "finished component analysis";
+    }
+    catch (const std::exception &e)
+    {
+        SimpleLogger().Write(logWARNING) << "[exception] " << e.what();
+    }
+    return 0;
+}

src/tools/contract.cpp

@@ -0,0 +1,109 @@
+/*
+
+Copyright (c) 2013, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "contractor/processing_chain.hpp"
+#include "contractor/contractor_options.hpp"
+#include "util/simple_logger.hpp"
+
+#include <boost/program_options/errors.hpp>
+
+#include <tbb/task_scheduler_init.h>
+
+#include <cstdlib>
+#include <exception>
+#include <ostream>
+#include <new>
+
+int main(int argc, char *argv[]) try
+{
+    LogPolicy::GetInstance().Unmute();
+    ContractorConfig contractor_config;
+
+    const return_code result = ContractorOptions::ParseArguments(argc, argv, contractor_config);
+
+    if (return_code::fail == result)
+    {
+        return EXIT_FAILURE;
+    }
+
+    if (return_code::exit == result)
+    {
+        return EXIT_SUCCESS;
+    }
+
+    ContractorOptions::GenerateOutputFilesNames(contractor_config);
+
+    if (1 > contractor_config.requested_num_threads)
+    {
+        SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger";
+        return EXIT_FAILURE;
+    }
+
+    const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
+
+    if (recommended_num_threads != contractor_config.requested_num_threads)
+    {
+        SimpleLogger().Write(logWARNING) << "The recommended number of threads is "
+                                         << recommended_num_threads
+                                         << "! This setting may have performance side-effects.";
+    }
+
+    if (!boost::filesystem::is_regular_file(contractor_config.osrm_input_path))
+    {
+        SimpleLogger().Write(logWARNING)
+            << "Input file " << contractor_config.osrm_input_path.string() << " not found!";
+        return EXIT_FAILURE;
+    }
+
+    if (!boost::filesystem::is_regular_file(contractor_config.profile_path))
+    {
+        SimpleLogger().Write(logWARNING) << "Profile " << contractor_config.profile_path.string()
+                                         << " not found!";
+        return EXIT_FAILURE;
+    }
+
+    SimpleLogger().Write() << "Input file: "
+                           << contractor_config.osrm_input_path.filename().string();
+    SimpleLogger().Write() << "Profile: " << contractor_config.profile_path.filename().string();
+    SimpleLogger().Write() << "Threads: " << contractor_config.requested_num_threads;
+
+    tbb::task_scheduler_init init(contractor_config.requested_num_threads);
+
+    return Prepare(contractor_config).Run();
+}
+catch (const std::bad_alloc &e)
+{
+    SimpleLogger().Write(logWARNING) << "[exception] " << e.what();
+    SimpleLogger().Write(logWARNING)
+        << "Please provide more memory or consider using a larger swapfile";
+    return EXIT_FAILURE;
+}
+catch (const std::exception &e)
+{
+    SimpleLogger().Write(logWARNING) << "[exception] " << e.what();
+    return EXIT_FAILURE;
+}

src/tools/datastore.cpp

@@ -0,0 +1,598 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "data_structures/original_edge_data.hpp"
+#include "data_structures/range_table.hpp"
+#include "data_structures/query_edge.hpp"
+#include "data_structures/query_node.hpp"
+#include "data_structures/shared_memory_factory.hpp"
+#include "data_structures/shared_memory_vector_wrapper.hpp"
+#include "data_structures/static_graph.hpp"
+#include "data_structures/static_rtree.hpp"
+#include "data_structures/travel_mode.hpp"
+#include "data_structures/turn_instructions.hpp"
+#include "server/data_structures/datafacade_base.hpp"
+#include "server/data_structures/shared_datatype.hpp"
+#include "server/data_structures/shared_barriers.hpp"
+#include "util/datastore_options.hpp"
+#include "util/simple_logger.hpp"
+#include "util/osrm_exception.hpp"
+#include "util/fingerprint.hpp"
+#include "typedefs.h"
+
+#include <osrm/coordinate.hpp>
+
+using RTreeLeaf = BaseDataFacade<QueryEdge::EdgeData>::RTreeLeaf;
+using RTreeNode = StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>::TreeNode;
+using QueryGraph = StaticGraph<QueryEdge::EdgeData>;
+
+#ifdef __linux__
+#include <sys/mman.h>
+#endif
+
+#include <boost/filesystem/fstream.hpp>
+#include <boost/iostreams/seek.hpp>
+
+#include <cstdint>
+
+#include <fstream>
+#include <string>
+#include <new>
+
+// delete a shared memory region. report warning if it could not be deleted
+void delete_region(const SharedDataType region)
+{
+    if (SharedMemory::RegionExists(region) && !SharedMemory::Remove(region))
+    {
+        const std::string name = [&]
+        {
+            switch (region)
+            {
+            case CURRENT_REGIONS:
+                return "CURRENT_REGIONS";
+            case LAYOUT_1:
+                return "LAYOUT_1";
+            case DATA_1:
+                return "DATA_1";
+            case LAYOUT_2:
+                return "LAYOUT_2";
+            case DATA_2:
+                return "DATA_2";
+            case LAYOUT_NONE:
+                return "LAYOUT_NONE";
+            default: // DATA_NONE:
+                return "DATA_NONE";
+            }
+        }();
+
+        SimpleLogger().Write(logWARNING) << "could not delete shared memory region " << name;
+    }
+}
+
+int main(const int argc, const char *argv[]) try
+{
+    LogPolicy::GetInstance().Unmute();
+    SharedBarriers barrier;
+
+#ifdef __linux__
+    // try to disable swapping on Linux
+    const bool lock_flags = MCL_CURRENT | MCL_FUTURE;
+    if (-1 == mlockall(lock_flags))
+    {
+        SimpleLogger().Write(logWARNING) << "Process " << argv[0] << " could not request RAM lock";
+    }
+#endif
+
+    try
+    {
+        boost::interprocess::scoped_lock<boost::interprocess::named_mutex> pending_lock(
+            barrier.pending_update_mutex);
+    }
+    catch (...)
+    {
+        // hard unlock in case of any exception.
+        barrier.pending_update_mutex.unlock();
+    }
+
+    SimpleLogger().Write(logDEBUG) << "Checking input parameters";
+
+    std::unordered_map<std::string, boost::filesystem::path> server_paths;
+    if (!GenerateDataStoreOptions(argc, argv, server_paths))
+    {
+        return EXIT_SUCCESS;
+    }
+
+    if (server_paths.find("hsgrdata") == server_paths.end())
+    {
+        throw osrm::exception("no hsgr file found");
+    }
+    if (server_paths.find("ramindex") == server_paths.end())
+    {
+        throw osrm::exception("no ram index file found");
+    }
+    if (server_paths.find("fileindex") == server_paths.end())
+    {
+        throw osrm::exception("no leaf index file found");
+    }
+    if (server_paths.find("nodesdata") == server_paths.end())
+    {
+        throw osrm::exception("no nodes file found");
+    }
+    if (server_paths.find("edgesdata") == server_paths.end())
+    {
+        throw osrm::exception("no edges file found");
+    }
+    if (server_paths.find("namesdata") == server_paths.end())
+    {
+        throw osrm::exception("no names file found");
+    }
+    if (server_paths.find("geometry") == server_paths.end())
+    {
+        throw osrm::exception("no geometry file found");
+    }
+    if (server_paths.find("core") == server_paths.end())
+    {
+        throw osrm::exception("no core file found");
+    }
+
+    auto paths_iterator = server_paths.find("hsgrdata");
+    BOOST_ASSERT(server_paths.end() != paths_iterator);
+    BOOST_ASSERT(!paths_iterator->second.empty());
+    const boost::filesystem::path &hsgr_path = paths_iterator->second;
+    paths_iterator = server_paths.find("timestamp");
+    BOOST_ASSERT(server_paths.end() != paths_iterator);
+    BOOST_ASSERT(!paths_iterator->second.empty());
+    const boost::filesystem::path &timestamp_path = paths_iterator->second;
+    paths_iterator = server_paths.find("ramindex");
+    BOOST_ASSERT(server_paths.end() != paths_iterator);
+    BOOST_ASSERT(!paths_iterator->second.empty());
+    const boost::filesystem::path &ram_index_path = paths_iterator->second;
+    paths_iterator = server_paths.find("fileindex");
+    BOOST_ASSERT(server_paths.end() != paths_iterator);
+    BOOST_ASSERT(!paths_iterator->second.empty());
+    const boost::filesystem::path index_file_path_absolute =
+        boost::filesystem::canonical(paths_iterator->second);
+    const std::string &file_index_path = index_file_path_absolute.string();
+    paths_iterator = server_paths.find("nodesdata");
+    BOOST_ASSERT(server_paths.end() != paths_iterator);
+    BOOST_ASSERT(!paths_iterator->second.empty());
+    const boost::filesystem::path &nodes_data_path = paths_iterator->second;
+    paths_iterator = server_paths.find("edgesdata");
+    BOOST_ASSERT(server_paths.end() != paths_iterator);
+    BOOST_ASSERT(!paths_iterator->second.empty());
+    const boost::filesystem::path &edges_data_path = paths_iterator->second;
+    paths_iterator = server_paths.find("namesdata");
+    BOOST_ASSERT(server_paths.end() != paths_iterator);
+    BOOST_ASSERT(!paths_iterator->second.empty());
+    const boost::filesystem::path &names_data_path = paths_iterator->second;
+    paths_iterator = server_paths.find("geometry");
+    BOOST_ASSERT(server_paths.end() != paths_iterator);
+    BOOST_ASSERT(!paths_iterator->second.empty());
+    const boost::filesystem::path &geometries_data_path = paths_iterator->second;
+    paths_iterator = server_paths.find("core");
+    BOOST_ASSERT(server_paths.end() != paths_iterator);
+    BOOST_ASSERT(!paths_iterator->second.empty());
+    const boost::filesystem::path &core_marker_path = paths_iterator->second;
+
+    // determine segment to use
+    bool segment2_in_use = SharedMemory::RegionExists(LAYOUT_2);
+    const SharedDataType layout_region = [&]
+    {
+        return segment2_in_use ? LAYOUT_1 : LAYOUT_2;
+    }();
+    const SharedDataType data_region = [&]
+    {
+        return segment2_in_use ? DATA_1 : DATA_2;
+    }();
+    const SharedDataType previous_layout_region = [&]
+    {
+        return segment2_in_use ? LAYOUT_2 : LAYOUT_1;
+    }();
+    const SharedDataType previous_data_region = [&]
+    {
+        return segment2_in_use ? DATA_2 : DATA_1;
+    }();
+
+    // Allocate a memory layout in shared memory, deallocate previous
+    auto *layout_memory = SharedMemoryFactory::Get(layout_region, sizeof(SharedDataLayout));
+    auto *shared_layout_ptr = new (layout_memory->Ptr()) SharedDataLayout();
+
+    shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::FILE_INDEX_PATH,
+                                          file_index_path.length() + 1);
+
+    // collect number of elements to store in shared memory object
+    SimpleLogger().Write() << "load names from: " << names_data_path;
+    // number of entries in name index
+    boost::filesystem::ifstream name_stream(names_data_path, std::ios::binary);
+    unsigned name_blocks = 0;
+    name_stream.read((char *)&name_blocks, sizeof(unsigned));
+    shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::NAME_OFFSETS, name_blocks);
+    shared_layout_ptr->SetBlockSize<typename RangeTable<16, true>::BlockT>(
+        SharedDataLayout::NAME_BLOCKS, name_blocks);
+    SimpleLogger().Write() << "name offsets size: " << name_blocks;
+    BOOST_ASSERT_MSG(0 != name_blocks, "name file broken");
+
+    unsigned number_of_chars = 0;
+    name_stream.read((char *)&number_of_chars, sizeof(unsigned));
+    shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::NAME_CHAR_LIST, number_of_chars);
+
+    // Loading information for original edges
+    boost::filesystem::ifstream edges_input_stream(edges_data_path, std::ios::binary);
+    unsigned number_of_original_edges = 0;
+    edges_input_stream.read((char *)&number_of_original_edges, sizeof(unsigned));
+
+    // note: settings this all to the same size is correct, we extract them from the same struct
+    shared_layout_ptr->SetBlockSize<NodeID>(SharedDataLayout::VIA_NODE_LIST,
+                                            number_of_original_edges);
+    shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::NAME_ID_LIST,
+                                              number_of_original_edges);
+    shared_layout_ptr->SetBlockSize<TravelMode>(SharedDataLayout::TRAVEL_MODE,
+                                                number_of_original_edges);
+    shared_layout_ptr->SetBlockSize<TurnInstruction>(SharedDataLayout::TURN_INSTRUCTION,
+                                                     number_of_original_edges);
+    // note: there are 32 geometry indicators in one unsigned block
+    shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::GEOMETRIES_INDICATORS,
+                                              number_of_original_edges);
+
+    boost::filesystem::ifstream hsgr_input_stream(hsgr_path, std::ios::binary);
+
+    FingerPrint fingerprint_valid = FingerPrint::GetValid();
+    FingerPrint fingerprint_loaded;
+    hsgr_input_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint));
+    if (fingerprint_loaded.TestGraphUtil(fingerprint_valid))
+    {
+        SimpleLogger().Write(logDEBUG) << "Fingerprint checked out ok";
+    }
+    else
+    {
+        SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build. "
+                                            "Reprocess to get rid of this warning.";
+    }
+
+    // load checksum
+    unsigned checksum = 0;
+    hsgr_input_stream.read((char *)&checksum, sizeof(unsigned));
+    shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::HSGR_CHECKSUM, 1);
+    // load graph node size
+    unsigned number_of_graph_nodes = 0;
+    hsgr_input_stream.read((char *)&number_of_graph_nodes, sizeof(unsigned));
+
+    BOOST_ASSERT_MSG((0 != number_of_graph_nodes), "number of nodes is zero");
+    shared_layout_ptr->SetBlockSize<QueryGraph::NodeArrayEntry>(SharedDataLayout::GRAPH_NODE_LIST,
+                                                                number_of_graph_nodes);
+
+    // load graph edge size
+    unsigned number_of_graph_edges = 0;
+    hsgr_input_stream.read((char *)&number_of_graph_edges, sizeof(unsigned));
+    // BOOST_ASSERT_MSG(0 != number_of_graph_edges, "number of graph edges is zero");
+    shared_layout_ptr->SetBlockSize<QueryGraph::EdgeArrayEntry>(SharedDataLayout::GRAPH_EDGE_LIST,
+                                                                number_of_graph_edges);
+
+    // load rsearch tree size
+    boost::filesystem::ifstream tree_node_file(ram_index_path, std::ios::binary);
+
+    uint32_t tree_size = 0;
+    tree_node_file.read((char *)&tree_size, sizeof(uint32_t));
+    shared_layout_ptr->SetBlockSize<RTreeNode>(SharedDataLayout::R_SEARCH_TREE, tree_size);
+
+    // load timestamp size
+    std::string m_timestamp;
+    if (boost::filesystem::exists(timestamp_path))
+    {
+        boost::filesystem::ifstream timestamp_stream(timestamp_path);
+        if (!timestamp_stream)
+        {
+            SimpleLogger().Write(logWARNING) << timestamp_path << " not found. setting to default";
+        }
+        else
+        {
+            getline(timestamp_stream, m_timestamp);
+            timestamp_stream.close();
+        }
+    }
+    if (m_timestamp.empty())
+    {
+        m_timestamp = "n/a";
+    }
+    if (25 < m_timestamp.length())
+    {
+        m_timestamp.resize(25);
+    }
+    shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::TIMESTAMP, m_timestamp.length());
+
+    // load core marker size
+    boost::filesystem::ifstream core_marker_file(core_marker_path, std::ios::binary);
+
+    uint32_t number_of_core_markers = 0;
+    core_marker_file.read((char *)&number_of_core_markers, sizeof(uint32_t));
+    shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::CORE_MARKER,
+                                              number_of_core_markers);
+
+    // load coordinate size
+    boost::filesystem::ifstream nodes_input_stream(nodes_data_path, std::ios::binary);
+    unsigned coordinate_list_size = 0;
+    nodes_input_stream.read((char *)&coordinate_list_size, sizeof(unsigned));
+    shared_layout_ptr->SetBlockSize<FixedPointCoordinate>(SharedDataLayout::COORDINATE_LIST,
+                                                          coordinate_list_size);
+
+    // load geometries sizes
+    std::ifstream geometry_input_stream(geometries_data_path.string().c_str(), std::ios::binary);
+    unsigned number_of_geometries_indices = 0;
+    unsigned number_of_compressed_geometries = 0;
+
+    geometry_input_stream.read((char *)&number_of_geometries_indices, sizeof(unsigned));
+    shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::GEOMETRIES_INDEX,
+                                              number_of_geometries_indices);
+    boost::iostreams::seek(geometry_input_stream, number_of_geometries_indices * sizeof(unsigned),
+                           BOOST_IOS::cur);
+    geometry_input_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned));
+    shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::GEOMETRIES_LIST,
+                                              number_of_compressed_geometries);
+    // allocate shared memory block
+    SimpleLogger().Write() << "allocating shared memory of " << shared_layout_ptr->GetSizeOfLayout()
+                           << " bytes";
+    SharedMemory *shared_memory =
+        SharedMemoryFactory::Get(data_region, shared_layout_ptr->GetSizeOfLayout());
+    char *shared_memory_ptr = static_cast<char *>(shared_memory->Ptr());
+
+    // read actual data into shared memory object //
+
+    // hsgr checksum
+    unsigned *checksum_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(
+        shared_memory_ptr, SharedDataLayout::HSGR_CHECKSUM);
+    *checksum_ptr = checksum;
+
+    // ram index file name
+    char *file_index_path_ptr = shared_layout_ptr->GetBlockPtr<char, true>(
+        shared_memory_ptr, SharedDataLayout::FILE_INDEX_PATH);
+    // make sure we have 0 ending
+    std::fill(file_index_path_ptr,
+              file_index_path_ptr +
+                  shared_layout_ptr->GetBlockSize(SharedDataLayout::FILE_INDEX_PATH),
+              0);
+    std::copy(file_index_path.begin(), file_index_path.end(), file_index_path_ptr);
+
+    // Loading street names
+    unsigned *name_offsets_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(
+        shared_memory_ptr, SharedDataLayout::NAME_OFFSETS);
+    if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_OFFSETS) > 0)
+    {
+        name_stream.read((char *)name_offsets_ptr,
+                         shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_OFFSETS));
+    }
+
+    unsigned *name_blocks_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(
+        shared_memory_ptr, SharedDataLayout::NAME_BLOCKS);
+    if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_BLOCKS) > 0)
+    {
+        name_stream.read((char *)name_blocks_ptr,
+                         shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_BLOCKS));
+    }
+
+    char *name_char_ptr = shared_layout_ptr->GetBlockPtr<char, true>(
+        shared_memory_ptr, SharedDataLayout::NAME_CHAR_LIST);
+    unsigned temp_length;
+    name_stream.read((char *)&temp_length, sizeof(unsigned));
+
+    BOOST_ASSERT_MSG(temp_length ==
+                         shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST),
+                     "Name file corrupted!");
+
+    if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST) > 0)
+    {
+        name_stream.read(name_char_ptr,
+                         shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST));
+    }
+
+    name_stream.close();
+
+    // load original edge information
+    NodeID *via_node_ptr = shared_layout_ptr->GetBlockPtr<NodeID, true>(
+        shared_memory_ptr, SharedDataLayout::VIA_NODE_LIST);
+
+    unsigned *name_id_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(
+        shared_memory_ptr, SharedDataLayout::NAME_ID_LIST);
+
+    TravelMode *travel_mode_ptr = shared_layout_ptr->GetBlockPtr<TravelMode, true>(
+        shared_memory_ptr, SharedDataLayout::TRAVEL_MODE);
+
+    TurnInstruction *turn_instructions_ptr = shared_layout_ptr->GetBlockPtr<TurnInstruction, true>(
+        shared_memory_ptr, SharedDataLayout::TURN_INSTRUCTION);
+
+    unsigned *geometries_indicator_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(
+        shared_memory_ptr, SharedDataLayout::GEOMETRIES_INDICATORS);
+
+    OriginalEdgeData current_edge_data;
+    for (unsigned i = 0; i < number_of_original_edges; ++i)
+    {
+        edges_input_stream.read((char *)&(current_edge_data), sizeof(OriginalEdgeData));
+        via_node_ptr[i] = current_edge_data.via_node;
+        name_id_ptr[i] = current_edge_data.name_id;
+        travel_mode_ptr[i] = current_edge_data.travel_mode;
+        turn_instructions_ptr[i] = current_edge_data.turn_instruction;
+
+        const unsigned bucket = i / 32;
+        const unsigned offset = i % 32;
+        const unsigned value = [&]
+        {
+            unsigned return_value = 0;
+            if (0 != offset)
+            {
+                return_value = geometries_indicator_ptr[bucket];
+            }
+            return return_value;
+        }();
+        if (current_edge_data.compressed_geometry)
+        {
+            geometries_indicator_ptr[bucket] = (value | (1 << offset));
+        }
+    }
+    edges_input_stream.close();
+
+    // load compressed geometry
+    unsigned temporary_value;
+    unsigned *geometries_index_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(
+        shared_memory_ptr, SharedDataLayout::GEOMETRIES_INDEX);
+    geometry_input_stream.seekg(0, geometry_input_stream.beg);
+    geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned));
+    BOOST_ASSERT(temporary_value ==
+                 shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_INDEX]);
+
+    if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_INDEX) > 0)
+    {
+        geometry_input_stream.read(
+            (char *)geometries_index_ptr,
+            shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_INDEX));
+    }
+    unsigned *geometries_list_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(
+        shared_memory_ptr, SharedDataLayout::GEOMETRIES_LIST);
+
+    geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned));
+    BOOST_ASSERT(temporary_value ==
+                 shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_LIST]);
+
+    if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST) > 0)
+    {
+        geometry_input_stream.read(
+            (char *)geometries_list_ptr,
+            shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST));
+    }
+
+    // Loading list of coordinates
+    FixedPointCoordinate *coordinates_ptr =
+        shared_layout_ptr->GetBlockPtr<FixedPointCoordinate, true>(
+            shared_memory_ptr, SharedDataLayout::COORDINATE_LIST);
+
+    QueryNode current_node;
+    for (unsigned i = 0; i < coordinate_list_size; ++i)
+    {
+        nodes_input_stream.read((char *)&current_node, sizeof(QueryNode));
+        coordinates_ptr[i] = FixedPointCoordinate(current_node.lat, current_node.lon);
+    }
+    nodes_input_stream.close();
+
+    // store timestamp
+    char *timestamp_ptr =
+        shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr, SharedDataLayout::TIMESTAMP);
+    std::copy(m_timestamp.c_str(), m_timestamp.c_str() + m_timestamp.length(), timestamp_ptr);
+
+    // store search tree portion of rtree
+    char *rtree_ptr = shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr,
+                                                                 SharedDataLayout::R_SEARCH_TREE);
+
+    if (tree_size > 0)
+    {
+        tree_node_file.read(rtree_ptr, sizeof(RTreeNode) * tree_size);
+    }
+    tree_node_file.close();
+
+    // load core markers
+    std::vector<char> unpacked_core_markers(number_of_core_markers);
+    core_marker_file.read((char *)unpacked_core_markers.data(),
+                          sizeof(char) * number_of_core_markers);
+
+    unsigned *core_marker_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(
+        shared_memory_ptr, SharedDataLayout::CORE_MARKER);
+
+    for (auto i = 0u; i < number_of_core_markers; ++i)
+    {
+        BOOST_ASSERT(unpacked_core_markers[i] == 0 || unpacked_core_markers[i] == 1);
+
+        if (unpacked_core_markers[i] == 1)
+        {
+            const unsigned bucket = i / 32;
+            const unsigned offset = i % 32;
+            const unsigned value = [&]
+            {
+                unsigned return_value = 0;
+                if (0 != offset)
+                {
+                    return_value = core_marker_ptr[bucket];
+                }
+                return return_value;
+            }();
+
+            core_marker_ptr[bucket] = (value | (1 << offset));
+        }
+    }
+
+    // load the nodes of the search graph
+    QueryGraph::NodeArrayEntry *graph_node_list_ptr =
+        shared_layout_ptr->GetBlockPtr<QueryGraph::NodeArrayEntry, true>(
+            shared_memory_ptr, SharedDataLayout::GRAPH_NODE_LIST);
+    if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST) > 0)
+    {
+        hsgr_input_stream.read((char *)graph_node_list_ptr,
+                               shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST));
+    }
+
+    // load the edges of the search graph
+    QueryGraph::EdgeArrayEntry *graph_edge_list_ptr =
+        shared_layout_ptr->GetBlockPtr<QueryGraph::EdgeArrayEntry, true>(
+            shared_memory_ptr, SharedDataLayout::GRAPH_EDGE_LIST);
+    if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST) > 0)
+    {
+        hsgr_input_stream.read((char *)graph_edge_list_ptr,
+                               shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST));
+    }
+    hsgr_input_stream.close();
+
+    // acquire lock
+    SharedMemory *data_type_memory =
+        SharedMemoryFactory::Get(CURRENT_REGIONS, sizeof(SharedDataTimestamp), true, false);
+    SharedDataTimestamp *data_timestamp_ptr =
+        static_cast<SharedDataTimestamp *>(data_type_memory->Ptr());
+
+    boost::interprocess::scoped_lock<boost::interprocess::named_mutex> query_lock(
+        barrier.query_mutex);
+
+    // notify all processes that were waiting for this condition
+    if (0 < barrier.number_of_queries)
+    {
+        barrier.no_running_queries_condition.wait(query_lock);
+    }
+
+    data_timestamp_ptr->layout = layout_region;
+    data_timestamp_ptr->data = data_region;
+    data_timestamp_ptr->timestamp += 1;
+    delete_region(previous_data_region);
+    delete_region(previous_layout_region);
+    SimpleLogger().Write() << "all data loaded";
+
+    shared_layout_ptr->PrintInformation();
+}
+catch (const std::bad_alloc &e)
+{
+    SimpleLogger().Write(logWARNING) << "[exception] " << e.what();
+    SimpleLogger().Write(logWARNING) << "Please provide more memory or disable locking the virtual "
+                                        "address space (note: this makes OSRM swap, i.e. slow)";
+    return EXIT_FAILURE;
+}
+catch (const std::exception &e)
+{
+    SimpleLogger().Write(logWARNING) << "caught exception: " << e.what();
+}

src/tools/extract.cpp

@@ -0,0 +1,89 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "extractor/extractor.hpp"
+#include "extractor/extractor_options.hpp"
+#include "util/simple_logger.hpp"
+
+#include <boost/filesystem.hpp>
+
+#include <cstdlib>
+#include <exception>
+#include <new>
+
+int main(int argc, char *argv[]) try
+{
+    LogPolicy::GetInstance().Unmute();
+    ExtractorConfig extractor_config;
+
+    const return_code result = ExtractorOptions::ParseArguments(argc, argv, extractor_config);
+
+    if (return_code::fail == result)
+    {
+        return EXIT_FAILURE;
+    }
+
+    if (return_code::exit == result)
+    {
+        return EXIT_SUCCESS;
+    }
+
+    ExtractorOptions::GenerateOutputFilesNames(extractor_config);
+
+    if (1 > extractor_config.requested_num_threads)
+    {
+        SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger";
+        return EXIT_FAILURE;
+    }
+
+    if (!boost::filesystem::is_regular_file(extractor_config.input_path))
+    {
+        SimpleLogger().Write(logWARNING) << "Input file " << extractor_config.input_path.string()
+                                         << " not found!";
+        return EXIT_FAILURE;
+    }
+
+    if (!boost::filesystem::is_regular_file(extractor_config.profile_path))
+    {
+        SimpleLogger().Write(logWARNING) << "Profile " << extractor_config.profile_path.string()
+                                         << " not found!";
+        return EXIT_FAILURE;
+    }
+    return extractor(extractor_config).run();
+}
+catch (const std::bad_alloc &e)
+{
+    SimpleLogger().Write(logWARNING) << "[exception] " << e.what();
+    SimpleLogger().Write(logWARNING)
+        << "Please provide more memory or consider using a larger swapfile";
+    return EXIT_FAILURE;
+}
+catch (const std::exception &e)
+{
+    SimpleLogger().Write(logWARNING) << "[exception] " << e.what();
+    return EXIT_FAILURE;
+}

src/tools/io-benchmark.cpp

@@ -0,0 +1,348 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "util/version.hpp"
+#include "../util/osrm_exception.hpp"
+#include "../util/simple_logger.hpp"
+#include "../util/timing_util.hpp"
+
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
+
+#include <cmath>
+#include <cstdio>
+#include <fcntl.h>
+#ifdef __linux__
+#include <malloc.h>
+#endif
+
+#include <algorithm>
+#include <chrono>
+#include <iomanip>
+#include <numeric>
+#include <random>
+#include <vector>
+
+const unsigned number_of_elements = 268435456;
+
+struct Statistics
+{
+    double min, max, med, mean, dev;
+};
+
+void RunStatistics(std::vector<double> &timings_vector, Statistics &stats)
+{
+    std::sort(timings_vector.begin(), timings_vector.end());
+    stats.min = timings_vector.front();
+    stats.max = timings_vector.back();
+    stats.med = timings_vector[timings_vector.size() / 2];
+    double primary_sum = std::accumulate(timings_vector.begin(), timings_vector.end(), 0.0);
+    stats.mean = primary_sum / timings_vector.size();
+
+    double primary_sq_sum = std::inner_product(timings_vector.begin(), timings_vector.end(),
+                                               timings_vector.begin(), 0.0);
+    stats.dev = std::sqrt(primary_sq_sum / timings_vector.size() - (stats.mean * stats.mean));
+}
+
+int main(int argc, char *argv[])
+{
+
+#ifdef __FreeBSD__
+    SimpleLogger().Write() << "Not supported on FreeBSD";
+    return 0;
+#endif
+#ifdef _WIN32
+    SimpleLogger().Write() << "Not supported on Windows";
+    return 0;
+#else
+
+    LogPolicy::GetInstance().Unmute();
+    boost::filesystem::path test_path;
+    try
+    {
+        SimpleLogger().Write() << "starting up engines, " << OSRM_VERSION;
+
+        if (1 == argc)
+        {
+            SimpleLogger().Write(logWARNING) << "usage: " << argv[0] << " /path/on/device";
+            return -1;
+        }
+
+        test_path = boost::filesystem::path(argv[1]);
+        test_path /= "osrm.tst";
+        SimpleLogger().Write(logDEBUG) << "temporary file: " << test_path.string();
+
+        // create files for testing
+        if (2 == argc)
+        {
+            // create file to test
+            if (boost::filesystem::exists(test_path))
+            {
+                throw osrm::exception("Data file already exists");
+            }
+
+            int *random_array = new int[number_of_elements];
+            std::generate(random_array, random_array + number_of_elements, std::rand);
+#ifdef __APPLE__
+            FILE *fd = fopen(test_path.string().c_str(), "w");
+            fcntl(fileno(fd), F_NOCACHE, 1);
+            fcntl(fileno(fd), F_RDAHEAD, 0);
+            TIMER_START(write_1gb);
+            write(fileno(fd), (char *)random_array, number_of_elements * sizeof(unsigned));
+            TIMER_STOP(write_1gb);
+            fclose(fd);
+#endif
+#ifdef __linux__
+            int file_desc =
+                open(test_path.string().c_str(), O_CREAT | O_TRUNC | O_WRONLY | O_SYNC, S_IRWXU);
+            if (-1 == file_desc)
+            {
+                throw osrm::exception("Could not open random data file");
+            }
+            TIMER_START(write_1gb);
+            int ret = write(file_desc, random_array, number_of_elements * sizeof(unsigned));
+            if (0 > ret)
+            {
+                throw osrm::exception("could not write random data file");
+            }
+            TIMER_STOP(write_1gb);
+            close(file_desc);
+#endif
+            delete[] random_array;
+            SimpleLogger().Write(logDEBUG) << "writing raw 1GB took " << TIMER_SEC(write_1gb)
+                                           << "s";
+            SimpleLogger().Write() << "raw write performance: " << std::setprecision(5)
+                                   << std::fixed << 1024 * 1024 / TIMER_SEC(write_1gb) << "MB/sec";
+
+            SimpleLogger().Write(logDEBUG)
+                << "finished creation of random data. Flush disk cache now!";
+        }
+        else
+        {
+            // Run Non-Cached I/O benchmarks
+            if (!boost::filesystem::exists(test_path))
+            {
+                throw osrm::exception("data file does not exist");
+            }
+
+            // volatiles do not get optimized
+            Statistics stats;
+
+#ifdef __APPLE__
+            volatile unsigned single_block[1024];
+            char *raw_array = new char[number_of_elements * sizeof(unsigned)];
+            FILE *fd = fopen(test_path.string().c_str(), "r");
+            fcntl(fileno(fd), F_NOCACHE, 1);
+            fcntl(fileno(fd), F_RDAHEAD, 0);
+#endif
+#ifdef __linux__
+            char *single_block = (char *)memalign(512, 1024 * sizeof(unsigned));
+
+            int file_desc = open(test_path.string().c_str(), O_RDONLY | O_DIRECT | O_SYNC);
+            if (-1 == file_desc)
+            {
+                SimpleLogger().Write(logDEBUG) << "opened, error: " << strerror(errno);
+                return -1;
+            }
+            char *raw_array = (char *)memalign(512, number_of_elements * sizeof(unsigned));
+#endif
+            TIMER_START(read_1gb);
+#ifdef __APPLE__
+            read(fileno(fd), raw_array, number_of_elements * sizeof(unsigned));
+            close(fileno(fd));
+            fd = fopen(test_path.string().c_str(), "r");
+#endif
+#ifdef __linux__
+            int ret = read(file_desc, raw_array, number_of_elements * sizeof(unsigned));
+            SimpleLogger().Write(logDEBUG) << "read " << ret
+                                           << " bytes, error: " << strerror(errno);
+            close(file_desc);
+            file_desc = open(test_path.string().c_str(), O_RDONLY | O_DIRECT | O_SYNC);
+            SimpleLogger().Write(logDEBUG) << "opened, error: " << strerror(errno);
+#endif
+            TIMER_STOP(read_1gb);
+
+            SimpleLogger().Write(logDEBUG) << "reading raw 1GB took " << TIMER_SEC(read_1gb) << "s";
+            SimpleLogger().Write() << "raw read performance: " << std::setprecision(5) << std::fixed
+                                   << 1024 * 1024 / TIMER_SEC(read_1gb) << "MB/sec";
+
+            std::vector<double> timing_results_raw_random;
+            SimpleLogger().Write(logDEBUG) << "running 1000 random I/Os of 4KB";
+
+#ifdef __APPLE__
+            fseek(fd, 0, SEEK_SET);
+#endif
+#ifdef __linux__
+            lseek(file_desc, 0, SEEK_SET);
+#endif
+            // make 1000 random access, time each I/O seperately
+            unsigned number_of_blocks = (number_of_elements * sizeof(unsigned) - 1) / 4096;
+            std::random_device rd;
+            std::default_random_engine e1(rd());
+            std::uniform_int_distribution<unsigned> uniform_dist(0, number_of_blocks - 1);
+            for (unsigned i = 0; i < 1000; ++i)
+            {
+                unsigned block_to_read = uniform_dist(e1);
+                off_t current_offset = block_to_read * 4096;
+                TIMER_START(random_access);
+#ifdef __APPLE__
+                int ret1 = fseek(fd, current_offset, SEEK_SET);
+                int ret2 = read(fileno(fd), (char *)&single_block[0], 4096);
+#endif
+
+#ifdef __FreeBSD__
+                int ret1 = 0;
+                int ret2 = 0;
+#endif
+
+#ifdef __linux__
+                int ret1 = lseek(file_desc, current_offset, SEEK_SET);
+                int ret2 = read(file_desc, (char *)single_block, 4096);
+#endif
+                TIMER_STOP(random_access);
+                if (((off_t)-1) == ret1)
+                {
+                    SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
+                    SimpleLogger().Write(logWARNING) << "seek error " << strerror(errno);
+                    throw osrm::exception("seek error");
+                }
+                if (-1 == ret2)
+                {
+                    SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
+                    SimpleLogger().Write(logWARNING) << "read error " << strerror(errno);
+                    throw osrm::exception("read error");
+                }
+                timing_results_raw_random.push_back(TIMER_SEC(random_access));
+            }
+
+            // Do statistics
+            SimpleLogger().Write(logDEBUG) << "running raw random I/O statistics";
+            std::ofstream random_csv("random.csv", std::ios::trunc);
+            for (unsigned i = 0; i < timing_results_raw_random.size(); ++i)
+            {
+                random_csv << i << ", " << timing_results_raw_random[i] << std::endl;
+            }
+            random_csv.close();
+            RunStatistics(timing_results_raw_random, stats);
+
+            SimpleLogger().Write() << "raw random I/O: " << std::setprecision(5) << std::fixed
+                                   << "min: " << stats.min << "ms, "
+                                   << "mean: " << stats.mean << "ms, "
+                                   << "med: " << stats.med << "ms, "
+                                   << "max: " << stats.max << "ms, "
+                                   << "dev: " << stats.dev << "ms";
+
+            std::vector<double> timing_results_raw_seq;
+#ifdef __APPLE__
+            fseek(fd, 0, SEEK_SET);
+#endif
+#ifdef __linux__
+            lseek(file_desc, 0, SEEK_SET);
+#endif
+
+            // read every 100th block
+            for (unsigned i = 0; i < 1000; ++i)
+            {
+                off_t current_offset = i * 4096;
+                TIMER_START(read_every_100);
+#ifdef __APPLE__
+                int ret1 = fseek(fd, current_offset, SEEK_SET);
+                int ret2 = read(fileno(fd), (char *)&single_block, 4096);
+#endif
+
+#ifdef __FreeBSD__
+                int ret1 = 0;
+                int ret2 = 0;
+#endif
+
+#ifdef __linux__
+                int ret1 = lseek(file_desc, current_offset, SEEK_SET);
+
+                int ret2 = read(file_desc, (char *)single_block, 4096);
+#endif
+                TIMER_STOP(read_every_100);
+                if (((off_t)-1) == ret1)
+                {
+                    SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
+                    SimpleLogger().Write(logWARNING) << "seek error " << strerror(errno);
+                    throw osrm::exception("seek error");
+                }
+                if (-1 == ret2)
+                {
+                    SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
+                    SimpleLogger().Write(logWARNING) << "read error " << strerror(errno);
+                    throw osrm::exception("read error");
+                }
+                timing_results_raw_seq.push_back(TIMER_SEC(read_every_100));
+            }
+#ifdef __APPLE__
+            fclose(fd);
+            // free(single_element);
+            free(raw_array);
+// free(single_block);
+#endif
+#ifdef __linux__
+            close(file_desc);
+#endif
+            // Do statistics
+            SimpleLogger().Write(logDEBUG) << "running sequential I/O statistics";
+            // print simple statistics: min, max, median, variance
+            std::ofstream seq_csv("sequential.csv", std::ios::trunc);
+            for (unsigned i = 0; i < timing_results_raw_seq.size(); ++i)
+            {
+                seq_csv << i << ", " << timing_results_raw_seq[i] << std::endl;
+            }
+            seq_csv.close();
+            RunStatistics(timing_results_raw_seq, stats);
+            SimpleLogger().Write() << "raw sequential I/O: " << std::setprecision(5) << std::fixed
+                                   << "min: " << stats.min << "ms, "
+                                   << "mean: " << stats.mean << "ms, "
+                                   << "med: " << stats.med << "ms, "
+                                   << "max: " << stats.max << "ms, "
+                                   << "dev: " << stats.dev << "ms";
+
+            if (boost::filesystem::exists(test_path))
+            {
+                boost::filesystem::remove(test_path);
+                SimpleLogger().Write(logDEBUG) << "removing temporary files";
+            }
+        }
+    }
+    catch (const std::exception &e)
+    {
+        SimpleLogger().Write(logWARNING) << "caught exception: " << e.what();
+        SimpleLogger().Write(logWARNING) << "cleaning up, and exiting";
+        if (boost::filesystem::exists(test_path))
+        {
+            boost::filesystem::remove(test_path);
+            SimpleLogger().Write(logWARNING) << "removing temporary files";
+        }
+        return -1;
+    }
+    return 0;
+#endif
+}

src/tools/routed.cpp

@@ -0,0 +1,198 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "server/server.hpp"
+#include "util/version.hpp"
+#include "util/routed_options.hpp"
+#include "util/simple_logger.hpp"
+
+#include <osrm/osrm.hpp>
+#include <osrm/libosrm_config.hpp>
+
+#ifdef __linux__
+#include <sys/mman.h>
+#endif
+
+#include <cstdlib>
+
+#include <signal.h>
+
+#include <chrono>
+#include <future>
+#include <iostream>
+#include <thread>
+#include <new>
+
+#ifdef _WIN32
+boost::function0<void> console_ctrl_function;
+
+BOOL WINAPI console_ctrl_handler(DWORD ctrl_type)
+{
+    switch (ctrl_type)
+    {
+    case CTRL_C_EVENT:
+    case CTRL_BREAK_EVENT:
+    case CTRL_CLOSE_EVENT:
+    case CTRL_SHUTDOWN_EVENT:
+        console_ctrl_function();
+        return TRUE;
+    default:
+        return FALSE;
+    }
+}
+#endif
+
+int main(int argc, const char *argv[]) try
+{
+    LogPolicy::GetInstance().Unmute();
+
+    bool trial_run = false;
+    std::string ip_address;
+    int ip_port, requested_thread_num;
+
+    LibOSRMConfig lib_config;
+    const unsigned init_result = GenerateServerProgramOptions(
+        argc, argv, lib_config.server_paths, ip_address, ip_port, requested_thread_num,
+        lib_config.use_shared_memory, trial_run, lib_config.max_locations_trip, lib_config.max_locations_viaroute,
+        lib_config.max_locations_distance_table,
+        lib_config.max_locations_map_matching);
+    if (init_result == INIT_OK_DO_NOT_START_ENGINE)
+    {
+        return EXIT_SUCCESS;
+    }
+    if (init_result == INIT_FAILED)
+    {
+        return EXIT_FAILURE;
+    }
+
+#ifdef __linux__
+    struct MemoryLocker final
+    {
+        explicit MemoryLocker(bool shouldLock_) : shouldLock(shouldLock_)
+        {
+            if (shouldLock && -1 == mlockall(MCL_CURRENT | MCL_FUTURE))
+            {
+                couldLock = false;
+                SimpleLogger().Write(logWARNING) << "memory could not be locked to RAM";
+            }
+        }
+        ~MemoryLocker()
+        {
+            if (shouldLock && couldLock)
+                (void)munlockall();
+        }
+        bool shouldLock = false, couldLock = true;
+    } memoryLocker(lib_config.use_shared_memory);
+#endif
+    SimpleLogger().Write() << "starting up engines, " << OSRM_VERSION;
+
+    if (lib_config.use_shared_memory)
+    {
+        SimpleLogger().Write(logDEBUG) << "Loading from shared memory";
+    }
+
+    SimpleLogger().Write(logDEBUG) << "Threads:\t" << requested_thread_num;
+    SimpleLogger().Write(logDEBUG) << "IP address:\t" << ip_address;
+    SimpleLogger().Write(logDEBUG) << "IP port:\t" << ip_port;
+
+#ifndef _WIN32
+    int sig = 0;
+    sigset_t new_mask;
+    sigset_t old_mask;
+    sigfillset(&new_mask);
+    pthread_sigmask(SIG_BLOCK, &new_mask, &old_mask);
+#endif
+
+    OSRM osrm_lib(lib_config);
+    auto routing_server = Server::CreateServer(ip_address, ip_port, requested_thread_num);
+
+    routing_server->GetRequestHandlerPtr().RegisterRoutingMachine(&osrm_lib);
+
+    if (trial_run)
+    {
+        SimpleLogger().Write() << "trial run, quitting after successful initialization";
+    }
+    else
+    {
+        std::packaged_task<int()> server_task([&]() -> int
+                                              {
+                                                  routing_server->Run();
+                                                  return 0;
+                                              });
+        auto future = server_task.get_future();
+        std::thread server_thread(std::move(server_task));
+
+#ifndef _WIN32
+        sigset_t wait_mask;
+        pthread_sigmask(SIG_SETMASK, &old_mask, nullptr);
+        sigemptyset(&wait_mask);
+        sigaddset(&wait_mask, SIGINT);
+        sigaddset(&wait_mask, SIGQUIT);
+        sigaddset(&wait_mask, SIGTERM);
+        pthread_sigmask(SIG_BLOCK, &wait_mask, nullptr);
+        SimpleLogger().Write() << "running and waiting for requests";
+        sigwait(&wait_mask, &sig);
+#else
+        // Set console control handler to allow server to be stopped.
+        console_ctrl_function = std::bind(&Server::Stop, routing_server);
+        SetConsoleCtrlHandler(console_ctrl_handler, TRUE);
+        SimpleLogger().Write() << "running and waiting for requests";
+        routing_server->Run();
+#endif
+        SimpleLogger().Write() << "initiating shutdown";
+        routing_server->Stop();
+        SimpleLogger().Write() << "stopping threads";
+
+        auto status = future.wait_for(std::chrono::seconds(2));
+
+        if (status == std::future_status::ready)
+        {
+            server_thread.join();
+        }
+        else
+        {
+            SimpleLogger().Write(logWARNING) << "Didn't exit within 2 seconds. Hard abort!";
+            server_task.reset(); // just kill it
+        }
+    }
+
+    SimpleLogger().Write() << "freeing objects";
+    routing_server.reset();
+    SimpleLogger().Write() << "shutdown completed";
+}
+catch (const std::bad_alloc &e)
+{
+    SimpleLogger().Write(logWARNING) << "[exception] " << e.what();
+    SimpleLogger().Write(logWARNING)
+        << "Please provide more memory or consider using a larger swapfile";
+    return EXIT_FAILURE;
+}
+catch (const std::exception &e)
+{
+    SimpleLogger().Write(logWARNING) << "[exception] " << e.what();
+    return EXIT_FAILURE;
+}

src/tools/simpleclient.cpp

@@ -0,0 +1,97 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "util/version.hpp"
+#include "../util/json_renderer.hpp"
+#include "../util/routed_options.hpp"
+#include "../util/simple_logger.hpp"
+
+#include <osrm/json_container.hpp>
+#include <osrm/libosrm_config.hpp>
+#include <osrm/route_parameters.hpp>
+#include <osrm/osrm.hpp>
+
+#include <string>
+
+int main(int argc, const char *argv[])
+{
+    LogPolicy::GetInstance().Unmute();
+    try
+    {
+        std::string ip_address;
+        int ip_port, requested_thread_num;
+        bool trial_run = false;
+        LibOSRMConfig lib_config;
+        const unsigned init_result = GenerateServerProgramOptions(
+            argc, argv, lib_config.server_paths, ip_address, ip_port, requested_thread_num,
+            lib_config.use_shared_memory, trial_run, lib_config.max_locations_trip,
+            lib_config.max_locations_viaroute, lib_config.max_locations_distance_table,
+            lib_config.max_locations_map_matching);
+
+        if (init_result == INIT_OK_DO_NOT_START_ENGINE)
+        {
+            return 0;
+        }
+        if (init_result == INIT_FAILED)
+        {
+            return 1;
+        }
+        SimpleLogger().Write() << "starting up engines, " << OSRM_VERSION;
+
+        OSRM routing_machine(lib_config);
+
+        RouteParameters route_parameters;
+        route_parameters.zoom_level = 18;           // no generalization
+        route_parameters.print_instructions = true; // turn by turn instructions
+        route_parameters.alternate_route = true;    // get an alternate route, too
+        route_parameters.geometry = true;           // retrieve geometry of route
+        route_parameters.compression = true;        // polyline encoding
+        route_parameters.check_sum = -1;            // see wiki
+        route_parameters.service = "viaroute";      // that's routing
+        route_parameters.output_format = "json";
+        route_parameters.jsonp_parameter = ""; // set for jsonp wrapping
+        route_parameters.language = "";        // unused atm
+        // route_parameters.hints.push_back(); // see wiki, saves I/O if done properly
+
+        // start_coordinate
+        route_parameters.coordinates.emplace_back(52.519930 * COORDINATE_PRECISION,
+                                                  13.438640 * COORDINATE_PRECISION);
+        // target_coordinate
+        route_parameters.coordinates.emplace_back(52.513191 * COORDINATE_PRECISION,
+                                                  13.415852 * COORDINATE_PRECISION);
+        osrm::json::Object json_result;
+        const int result_code = routing_machine.RunQuery(route_parameters, json_result);
+        SimpleLogger().Write() << "http code: " << result_code;
+        osrm::json::render(SimpleLogger().Write(), json_result);
+    }
+    catch (std::exception &current_exception)
+    {
+        SimpleLogger().Write(logWARNING) << "caught exception: " << current_exception.what();
+        return -1;
+    }
+    return 0;
+}

src/tools/springclean.cpp

@@ -0,0 +1,102 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include <cstdio>
+
+#include "../data_structures/shared_memory_factory.hpp"
+#include "../server/data_structures/shared_datatype.hpp"
+#include "util/version.hpp"
+#include "../util/simple_logger.hpp"
+
+void delete_region(const SharedDataType region)
+{
+    if (SharedMemory::RegionExists(region) && !SharedMemory::Remove(region))
+    {
+        const std::string name = [&]
+        {
+            switch (region)
+            {
+            case CURRENT_REGIONS:
+                return "CURRENT_REGIONS";
+            case LAYOUT_1:
+                return "LAYOUT_1";
+            case DATA_1:
+                return "DATA_1";
+            case LAYOUT_2:
+                return "LAYOUT_2";
+            case DATA_2:
+                return "DATA_2";
+            case LAYOUT_NONE:
+                return "LAYOUT_NONE";
+            default: // DATA_NONE:
+                return "DATA_NONE";
+            }
+        }();
+
+        SimpleLogger().Write(logWARNING) << "could not delete shared memory region " << name;
+    }
+}
+
+// find all existing shmem regions and remove them.
+void springclean()
+{
+    SimpleLogger().Write() << "spring-cleaning all shared memory regions";
+    delete_region(DATA_1);
+    delete_region(LAYOUT_1);
+    delete_region(DATA_2);
+    delete_region(LAYOUT_2);
+    delete_region(CURRENT_REGIONS);
+}
+
+int main()
+{
+    LogPolicy::GetInstance().Unmute();
+    try
+    {
+        SimpleLogger().Write() << "starting up engines, " << OSRM_VERSION << "\n\n";
+        SimpleLogger().Write() << "Releasing all locks";
+        SimpleLogger().Write() << "ATTENTION! BE CAREFUL!";
+        SimpleLogger().Write() << "----------------------";
+        SimpleLogger().Write() << "This tool may put osrm-routed into an undefined state!";
+        SimpleLogger().Write() << "Type 'Y' to acknowledge that you know what your are doing.";
+        SimpleLogger().Write() << "\n\nDo you want to purge all shared memory allocated "
+                               << "by osrm-datastore? [type 'Y' to confirm]";
+
+        const auto letter = getchar();
+        if (letter != 'Y')
+        {
+            SimpleLogger().Write() << "aborted.";
+            return 0;
+        }
+        springclean();
+    }
+    catch (const std::exception &e)
+    {
+        SimpleLogger().Write(logWARNING) << "[excpetion] " << e.what();
+    }
+    return 0;
+}

src/tools/unlock_all_mutexes.cpp

@@ -0,0 +1,51 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "util/version.hpp"
+#include "../util/simple_logger.hpp"
+#include "../server/data_structures/shared_barriers.hpp"
+
+#include <iostream>
+
+int main()
+{
+    LogPolicy::GetInstance().Unmute();
+    try
+    {
+        SimpleLogger().Write() << "starting up engines, " << OSRM_VERSION;
+        SimpleLogger().Write() << "Releasing all locks";
+        SharedBarriers barrier;
+        barrier.pending_update_mutex.unlock();
+        barrier.query_mutex.unlock();
+        barrier.update_mutex.unlock();
+    }
+    catch (const std::exception &e)
+    {
+        SimpleLogger().Write(logWARNING) << "[excpetion] " << e.what();
+    }
+    return 0;
+}

src/util/compute_angle.cpp

@@ -0,0 +1,54 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "compute_angle.hpp"
+
+#include "trigonometry_table.hpp"
+#include "../util/mercator.hpp"
+
+#include <osrm/coordinate.hpp>
+
+#include <cmath>
+
+double ComputeAngle::OfThreeFixedPointCoordinates(const FixedPointCoordinate &first,
+                                                  const FixedPointCoordinate &second,
+                                                  const FixedPointCoordinate &third) noexcept
+{
+    const double v1x = (first.lon - second.lon) / COORDINATE_PRECISION;
+    const double v1y = mercator::lat2y(first.lat / COORDINATE_PRECISION) -
+                       mercator::lat2y(second.lat / COORDINATE_PRECISION);
+    const double v2x = (third.lon - second.lon) / COORDINATE_PRECISION;
+    const double v2y = mercator::lat2y(third.lat / COORDINATE_PRECISION) -
+                       mercator::lat2y(second.lat / COORDINATE_PRECISION);
+
+    double angle = (atan2_lookup(v2y, v2x) - atan2_lookup(v1y, v1x)) * 180. / M_PI;
+    while (angle < 0.)
+    {
+        angle += 360.;
+    }
+    return angle;
+}

src/util/coordinate.cpp

@@ -0,0 +1,87 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "../algorithms/coordinate_calculation.hpp"
+
+#ifndef NDEBUG
+#include "../util/simple_logger.hpp"
+#endif
+#include <osrm/coordinate.hpp>
+
+#ifndef NDEBUG
+#include <bitset>
+#endif
+#include <iostream>
+#include <limits>
+
+FixedPointCoordinate::FixedPointCoordinate()
+    : lat(std::numeric_limits<int>::min()), lon(std::numeric_limits<int>::min())
+{
+}
+
+FixedPointCoordinate::FixedPointCoordinate(int lat, int lon) : lat(lat), lon(lon)
+{
+#ifndef NDEBUG
+    if (0 != (std::abs(lat) >> 30))
+    {
+        std::bitset<32> y_coordinate_vector(lat);
+        SimpleLogger().Write(logDEBUG) << "broken lat: " << lat
+                                       << ", bits: " << y_coordinate_vector;
+    }
+    if (0 != (std::abs(lon) >> 30))
+    {
+        std::bitset<32> x_coordinate_vector(lon);
+        SimpleLogger().Write(logDEBUG) << "broken lon: " << lon
+                                       << ", bits: " << x_coordinate_vector;
+    }
+#endif
+}
+
+bool FixedPointCoordinate::is_valid() const
+{
+    if (lat > 90 * COORDINATE_PRECISION || lat < -90 * COORDINATE_PRECISION ||
+        lon > 180 * COORDINATE_PRECISION || lon < -180 * COORDINATE_PRECISION)
+    {
+        return false;
+    }
+    return true;
+}
+
+bool FixedPointCoordinate::operator==(const FixedPointCoordinate &other) const
+{
+    return lat == other.lat && lon == other.lon;
+}
+
+void FixedPointCoordinate::output(std::ostream &out) const
+{
+    out << "(" << lat / COORDINATE_PRECISION << "," << lon / COORDINATE_PRECISION << ")";
+}
+
+float FixedPointCoordinate::bearing(const FixedPointCoordinate &other) const
+{
+    return coordinate_calculation::bearing(other, *this);
+}

src/util/coordinate_calculation.cpp

@@ -0,0 +1,273 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "coordinate_calculation.hpp"
+
+#include "../util/mercator.hpp"
+#include "../util/string_util.hpp"
+
+#include <boost/assert.hpp>
+
+#include <osrm/coordinate.hpp>
+
+#include <cmath>
+
+#include <limits>
+
+namespace
+{
+constexpr static const float RAD = 0.017453292519943295769236907684886f;
+// earth radius varies between 6,356.750-6,378.135 km (3,949.901-3,963.189mi)
+// The IUGG value for the equatorial radius is 6378.137 km (3963.19 miles)
+constexpr static const float earth_radius = 6372797.560856f;
+}
+
+namespace coordinate_calculation
+{
+
+double haversine_distance(const int lat1,
+                                                     const int lon1,
+                                                     const int lat2,
+                                                     const int lon2)
+{
+    BOOST_ASSERT(lat1 != std::numeric_limits<int>::min());
+    BOOST_ASSERT(lon1 != std::numeric_limits<int>::min());
+    BOOST_ASSERT(lat2 != std::numeric_limits<int>::min());
+    BOOST_ASSERT(lon2 != std::numeric_limits<int>::min());
+    const double lt1 = lat1 / COORDINATE_PRECISION;
+    const double ln1 = lon1 / COORDINATE_PRECISION;
+    const double lt2 = lat2 / COORDINATE_PRECISION;
+    const double ln2 = lon2 / COORDINATE_PRECISION;
+    const double dlat1 = lt1 * (RAD);
+
+    const double dlong1 = ln1 * (RAD);
+    const double dlat2 = lt2 * (RAD);
+    const double dlong2 = ln2 * (RAD);
+
+    const double dLong = dlong1 - dlong2;
+    const double dLat = dlat1 - dlat2;
+
+    const double aHarv = std::pow(std::sin(dLat / 2.0), 2.0) +
+                         std::cos(dlat1) * std::cos(dlat2) * std::pow(std::sin(dLong / 2.), 2);
+    const double cHarv = 2. * std::atan2(std::sqrt(aHarv), std::sqrt(1.0 - aHarv));
+    return earth_radius * cHarv;
+}
+
+double haversine_distance(const FixedPointCoordinate &coordinate_1,
+                                                     const FixedPointCoordinate &coordinate_2)
+{
+    return haversine_distance(coordinate_1.lat, coordinate_1.lon, coordinate_2.lat,
+                                 coordinate_2.lon);
+}
+
+float great_circle_distance(const FixedPointCoordinate &coordinate_1,
+                                                 const FixedPointCoordinate &coordinate_2)
+{
+    return great_circle_distance(coordinate_1.lat, coordinate_1.lon, coordinate_2.lat,
+                              coordinate_2.lon);
+}
+
+float great_circle_distance(const int lat1,
+                                                 const int lon1,
+                                                 const int lat2,
+                                                 const int lon2)
+{
+    BOOST_ASSERT(lat1 != std::numeric_limits<int>::min());
+    BOOST_ASSERT(lon1 != std::numeric_limits<int>::min());
+    BOOST_ASSERT(lat2 != std::numeric_limits<int>::min());
+    BOOST_ASSERT(lon2 != std::numeric_limits<int>::min());
+
+    const float float_lat1 = (lat1 / COORDINATE_PRECISION) * RAD;
+    const float float_lon1 = (lon1 / COORDINATE_PRECISION) * RAD;
+    const float float_lat2 = (lat2 / COORDINATE_PRECISION) * RAD;
+    const float float_lon2 = (lon2 / COORDINATE_PRECISION) * RAD;
+
+    const float x_value = (float_lon2 - float_lon1) * std::cos((float_lat1 + float_lat2) / 2.f);
+    const float y_value = float_lat2 - float_lat1;
+    return std::hypot(x_value, y_value) * earth_radius;
+}
+
+float perpendicular_distance(const FixedPointCoordinate &source_coordinate,
+                                                     const FixedPointCoordinate &target_coordinate,
+                                                     const FixedPointCoordinate &query_location)
+{
+    float ratio;
+    FixedPointCoordinate nearest_location;
+
+    return perpendicular_distance(source_coordinate, target_coordinate, query_location,
+                                  nearest_location, ratio);
+}
+
+float perpendicular_distance(const FixedPointCoordinate &segment_source,
+                                                     const FixedPointCoordinate &segment_target,
+                                                     const FixedPointCoordinate &query_location,
+                                                     FixedPointCoordinate &nearest_location,
+                                                     float &ratio)
+{
+    return perpendicular_distance_from_projected_coordinate(
+        segment_source, segment_target, query_location,
+        {mercator::lat2y(query_location.lat / COORDINATE_PRECISION),
+         query_location.lon / COORDINATE_PRECISION},
+        nearest_location, ratio);
+}
+
+float perpendicular_distance_from_projected_coordinate(
+    const FixedPointCoordinate &source_coordinate,
+    const FixedPointCoordinate &target_coordinate,
+    const FixedPointCoordinate &query_location,
+    const std::pair<double, double> &projected_coordinate)
+{
+    float ratio;
+    FixedPointCoordinate nearest_location;
+
+    return perpendicular_distance_from_projected_coordinate(source_coordinate, target_coordinate,
+                                                            query_location, projected_coordinate,
+                                                            nearest_location, ratio);
+}
+
+float perpendicular_distance_from_projected_coordinate(
+    const FixedPointCoordinate &segment_source,
+    const FixedPointCoordinate &segment_target,
+    const FixedPointCoordinate &query_location,
+    const std::pair<double, double> &projected_coordinate,
+    FixedPointCoordinate &nearest_location,
+    float &ratio)
+{
+    BOOST_ASSERT(query_location.is_valid());
+
+    // initialize values
+    const double x = projected_coordinate.first;
+    const double y = projected_coordinate.second;
+    const double a = mercator::lat2y(segment_source.lat / COORDINATE_PRECISION);
+    const double b = segment_source.lon / COORDINATE_PRECISION;
+    const double c = mercator::lat2y(segment_target.lat / COORDINATE_PRECISION);
+    const double d = segment_target.lon / COORDINATE_PRECISION;
+    double p, q /*,mX*/, nY;
+    if (std::abs(a - c) > std::numeric_limits<double>::epsilon())
+    {
+        const double m = (d - b) / (c - a); // slope
+        // Projection of (x,y) on line joining (a,b) and (c,d)
+        p = ((x + (m * y)) + (m * m * a - m * b)) / (1.f + m * m);
+        q = b + m * (p - a);
+    }
+    else
+    {
+        p = c;
+        q = y;
+    }
+    nY = (d * p - c * q) / (a * d - b * c);
+
+    // discretize the result to coordinate precision. it's a hack!
+    if (std::abs(nY) < (1.f / COORDINATE_PRECISION))
+    {
+        nY = 0.f;
+    }
+
+    // compute ratio
+    ratio =
+        static_cast<float>((p - nY * a) / c); // These values are actually n/m+n and m/m+n , we need
+    // not calculate the explicit values of m an n as we
+    // are just interested in the ratio
+    if (std::isnan(ratio))
+    {
+        ratio = (segment_target == query_location ? 1.f : 0.f);
+    }
+    else if (std::abs(ratio) <= std::numeric_limits<float>::epsilon())
+    {
+        ratio = 0.f;
+    }
+    else if (std::abs(ratio - 1.f) <= std::numeric_limits<float>::epsilon())
+    {
+        ratio = 1.f;
+    }
+
+    // compute nearest location
+    BOOST_ASSERT(!std::isnan(ratio));
+    if (ratio <= 0.f)
+    {
+        nearest_location = segment_source;
+    }
+    else if (ratio >= 1.f)
+    {
+        nearest_location = segment_target;
+    }
+    else
+    {
+        // point lies in between
+        nearest_location.lat = static_cast<int>(mercator::y2lat(p) * COORDINATE_PRECISION);
+        nearest_location.lon = static_cast<int>(q * COORDINATE_PRECISION);
+    }
+    BOOST_ASSERT(nearest_location.is_valid());
+
+    const float approximate_distance =
+        great_circle_distance(query_location, nearest_location);
+    BOOST_ASSERT(0.f <= approximate_distance);
+    return approximate_distance;
+}
+
+void lat_or_lon_to_string(const int value, std::string &output)
+{
+    char buffer[12];
+    buffer[11] = 0; // zero termination
+    output = printInt<11, 6>(buffer, value);
+}
+
+float deg_to_rad(const float degree)
+{
+    return degree * (static_cast<float>(M_PI) / 180.f);
+}
+
+float rad_to_deg(const float radian)
+{
+    return radian * (180.f * static_cast<float>(M_1_PI));
+}
+
+float bearing(const FixedPointCoordinate &first_coordinate,
+                                      const FixedPointCoordinate &second_coordinate)
+{
+    const float lon_diff =
+        second_coordinate.lon / COORDINATE_PRECISION - first_coordinate.lon / COORDINATE_PRECISION;
+    const float lon_delta = deg_to_rad(lon_diff);
+    const float lat1 = deg_to_rad(first_coordinate.lat / COORDINATE_PRECISION);
+    const float lat2 = deg_to_rad(second_coordinate.lat / COORDINATE_PRECISION);
+    const float y = std::sin(lon_delta) * std::cos(lat2);
+    const float x =
+        std::cos(lat1) * std::sin(lat2) - std::sin(lat1) * std::cos(lat2) * std::cos(lon_delta);
+    float result = rad_to_deg(std::atan2(y, x));
+    while (result < 0.f)
+    {
+        result += 360.f;
+    }
+
+    while (result >= 360.f)
+    {
+        result -= 360.f;
+    }
+    return result;
+}
+
+}

src/util/fingerprint.cpp

@@ -0,0 +1,29 @@
+/*
+
+Copyright (c) 2015, Project OSRM, Dennis Luxen, others
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "util/fingerprint.hpp"
+#include "util/fingerprint_impl.hpp"

src/util/hilbert_value.cpp

@@ -0,0 +1,100 @@
+/*
+
+Copyright (c) 2014, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "hilbert_value.hpp"
+
+#include <osrm/coordinate.hpp>
+
+uint64_t HilbertCode::operator()(const FixedPointCoordinate &current_coordinate) const
+{
+    unsigned location[2];
+    location[0] = current_coordinate.lat + static_cast<int>(90 * COORDINATE_PRECISION);
+    location[1] = current_coordinate.lon + static_cast<int>(180 * COORDINATE_PRECISION);
+
+    TransposeCoordinate(location);
+    return BitInterleaving(location[0], location[1]);
+}
+
+uint64_t HilbertCode::BitInterleaving(const uint32_t latitude, const uint32_t longitude) const
+{
+    uint64_t result = 0;
+    for (int8_t index = 31; index >= 0; --index)
+    {
+        result |= (latitude >> index) & 1;
+        result <<= 1;
+        result |= (longitude >> index) & 1;
+        if (0 != index)
+        {
+            result <<= 1;
+        }
+    }
+    return result;
+}
+
+void HilbertCode::TransposeCoordinate(uint32_t *X) const
+{
+    uint32_t M = 1u << (32 - 1), P, Q, t;
+    int i;
+    // Inverse undo
+    for (Q = M; Q > 1; Q >>= 1)
+    {
+        P = Q - 1;
+        for (i = 0; i < 2; ++i)
+        {
+
+            const bool condition = (X[i] & Q);
+            if (condition)
+            {
+                X[0] ^= P; // invert
+            }
+            else
+            {
+                t = (X[0] ^ X[i]) & P;
+                X[0] ^= t;
+                X[i] ^= t;
+            }
+        } // exchange
+    }
+    // Gray encode
+    for (i = 1; i < 2; ++i)
+    {
+        X[i] ^= X[i - 1];
+    }
+    t = 0;
+    for (Q = M; Q > 1; Q >>= 1)
+    {
+        const bool condition = (X[2 - 1] & Q);
+        if (condition)
+        {
+            t ^= Q - 1;
+        }
+    } // check if this for loop is wrong
+    for (i = 0; i < 2; ++i)
+    {
+        X[i] ^= t;
+    }
+}

src/util/mercator.cpp

@@ -0,0 +1,40 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "mercator.hpp"
+
+#include <cmath>
+
+double mercator::y2lat(const double value) noexcept
+{
+    return 180. * M_1_PI * (2. * std::atan(std::exp(value * M_PI / 180.)) - M_PI_2);
+}
+
+double mercator::lat2y(const double latitude) noexcept
+{
+    return 180. * M_1_PI * std::log(std::tan(M_PI_4 + latitude * (M_PI / 180.) / 2.));
+}

src/util/osrm_exception.cpp

@@ -0,0 +1,43 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "osrm_exception.hpp"
+
+namespace osrm
+{
+// This function exists to 'anchor' the class, and stop the compiler from
+// copying vtable and RTTI info into every object file that includes
+// this header. (Caught by -Wweak-vtables under Clang.)
+
+// More information from the LLVM Coding Standards:
+// If a class is defined in a header file and has a vtable (either it has
+// virtual methods or it derives from classes with virtual methods), it must
+// always have at least one out-of-line virtual method in the class. Without
+// this, the compiler will copy the vtable and RTTI into every .o file that
+// #includes the header, bloating .o file sizes and increasing link times.
+void exception::anchor() const {}
+}

src/util/simple_logger.cpp

@@ -0,0 +1,121 @@
+/*
+
+Copyright (c) 2015, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "simple_logger.hpp"
+#ifdef _MSC_VER
+#include <io.h>
+#define isatty _isatty
+#define fileno _fileno
+#else
+#include <unistd.h>
+#endif
+#include <cstdio>
+#include <iostream>
+#include <mutex>
+#include <string>
+
+namespace
+{
+static const char COL_RESET[]{"\x1b[0m"};
+static const char RED[]{"\x1b[31m"};
+#ifndef NDEBUG
+static const char YELLOW[]{"\x1b[33m"};
+#endif
+// static const char GREEN[] { "\x1b[32m"};
+// static const char BLUE[] { "\x1b[34m"};
+// static const char MAGENTA[] { "\x1b[35m"};
+// static const char CYAN[] { "\x1b[36m"};
+}
+
+void LogPolicy::Unmute() { m_is_mute = false; }
+
+void LogPolicy::Mute() { m_is_mute = true; }
+
+bool LogPolicy::IsMute() const { return m_is_mute; }
+
+LogPolicy &LogPolicy::GetInstance()
+{
+    static LogPolicy runningInstance;
+    return runningInstance;
+}
+
+SimpleLogger::SimpleLogger() : level(logINFO) {}
+
+std::mutex &SimpleLogger::get_mutex()
+{
+    static std::mutex mtx;
+    return mtx;
+}
+
+std::ostringstream &SimpleLogger::Write(LogLevel lvl) noexcept
+{
+    std::lock_guard<std::mutex> lock(get_mutex());
+    level = lvl;
+    os << "[";
+    switch (level)
+    {
+    case logWARNING:
+        os << "warn";
+        break;
+    case logDEBUG:
+#ifndef NDEBUG
+        os << "debug";
+#endif
+        break;
+    default: // logINFO:
+        os << "info";
+        break;
+    }
+    os << "] ";
+    return os;
+}
+
+SimpleLogger::~SimpleLogger()
+{
+    std::lock_guard<std::mutex> lock(get_mutex());
+    if (!LogPolicy::GetInstance().IsMute())
+    {
+        const bool is_terminal = static_cast<bool>(isatty(fileno(stdout)));
+        switch (level)
+        {
+        case logWARNING:
+            std::cerr << (is_terminal ? RED : "") << os.str() << (is_terminal ? COL_RESET : "")
+                      << std::endl;
+            break;
+        case logDEBUG:
+#ifndef NDEBUG
+            std::cout << (is_terminal ? YELLOW : "") << os.str() << (is_terminal ? COL_RESET : "")
+                      << std::endl;
+#endif
+            break;
+        case logINFO:
+        default:
+            std::cout << os.str() << (is_terminal ? COL_RESET : "") << std::endl;
+            break;
+        }
+    }
+}