Improvement of raster source loading to reduce consumed memory size.

include/extractor/raster_source.hpp

@@ -5,15 +5,22 @@
 #include "util/exception.hpp"
 
 #include <boost/algorithm/string/trim.hpp>
+#include <boost/algorithm/string.hpp>
 #include <boost/assert.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
 #include <boost/spirit/include/qi.hpp>
 #include <boost/spirit/include/qi_int.hpp>
+#include <boost/foreach.hpp>
+
 #include <storage/io.hpp>
 
 #include <iterator>
 #include <unordered_map>
+#include <string>
+#include <list>
+#include <iostream>
+using namespace std;
 
 namespace osrm
 {
@@ -45,8 +52,29 @@ class RasterGrid
         _data.reserve(ydim * xdim);
         BOOST_ASSERT(_data.capacity() >= ydim * xdim);
 
+        // Construct FileReader
         storage::io::FileReader file_reader(filepath, storage::io::FileReader::HasNoFingerprint);
+        std::string buf;
+        buf.resize(xdim * 11); // INT32_MAX = 2147483647 = 10 chars + 1 white space = 11
+        BOOST_ASSERT(buf.size() >= xdim * 11);
 
+        for (unsigned int y = 0 ; y < ydim ; y++) {
+            // read one line from file.
+            file_reader.ReadLine(&buf[0], xdim * 11);
+            boost::algorithm::trim(buf);
+
+            std::vector<std::string> result;
+            std::string delim (" ");
+            //boost::split(result, buf, boost::is_any_of(delim), boost::algorithm::token_compress_on);
+            boost::split(result, buf, boost::is_any_of(delim));
+            unsigned int x = 0;
+            BOOST_FOREACH(std::string s, result) {
+                _data[(y * xdim) + x] = atoi(s.c_str());
+                ++x;
+            }
+            BOOST_ASSERT(x == xdim);
+        }
+/*
         std::string buffer;
         buffer.resize(file_reader.GetSize());
 
@@ -76,6 +104,7 @@ class RasterGrid
             throw util::exception("Failed to parse raster source: " + filepath.string() +
                                   SOURCE_REF);
         }
+*/
     }
 
     RasterGrid(const RasterGrid &) = default;
@@ -144,8 +173,8 @@ class RasterContainer
     RasterDatum GetRasterInterpolateFromSource(unsigned int source_id, double lon, double lat);
 
   private:
-    std::vector<RasterSource> LoadedSources;
+    static std::vector<RasterSource> LoadedSources;
-    std::unordered_map<std::string, int> LoadedSourcePaths;
+    static std::unordered_map<std::string, int> LoadedSourcePaths;
 };
 }
 }

include/storage/io.hpp

@@ -69,7 +69,10 @@ class FileReader
             std::cout << ex.what() << std::endl;
             throw;
         }
+    }
 /*
+    std::size_t GetSize()
+    {
         const boost::filesystem::ifstream::pos_type position = input_stream.tellg();
         input_stream.seekg(0, std::ios::end);
         const boost::filesystem::ifstream::pos_type file_size = input_stream.tellg();
@@ -94,7 +97,16 @@ class FileReader
         {
             return file_size;
         }
- */
+    }
+*/
+    /* Read one line */
+    template <typename T> void ReadLine(T *dest, const std::size_t count) {
+        if (0 < count) {
+            const auto &ios = input_stream.getline(reinterpret_cast<char *>(dest), count * sizeof(T));
+            for (std::size_t n = ios.gcount(); n < count; ++n) {
+                reinterpret_cast<char *>(dest)[n] = '\0';
+            }
+        }
     }
 
     /* Read count objects of type T into pointer dest */

src/extractor/raster_source.cpp

@@ -78,6 +78,10 @@ RasterDatum RasterSource::GetRasterInterpolate(const int lon, const int lat) con
                                       raster_data(right, bottom) * (fromLeft * fromTop))};
 }
 
+// static member of Raster Container
+std::vector<RasterSource> RasterContainer::LoadedSources;
+std::unordered_map<std::string, int> RasterContainer::LoadedSourcePaths;
+
 // Load raster source into memory
 int RasterContainer::LoadRasterSource(const std::string &path_string,
                                       double xmin,
@@ -91,7 +95,13 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
     const auto _xmax = static_cast<std::int32_t>(util::toFixed(util::FloatLongitude{xmax}));
     const auto _ymin = static_cast<std::int32_t>(util::toFixed(util::FloatLatitude{ymin}));
     const auto _ymax = static_cast<std::int32_t>(util::toFixed(util::FloatLatitude{ymax}));
-
+/*
+    // for debug : list up all keys and values
+    util::Log() << "Num of Raster Sources : " << LoadedSourcePaths.size();
+    for (auto i = LoadedSourcePaths.begin(); i != LoadedSourcePaths.end(); ++i) {
+        util::Log() << "Key : " << i->first << " Value: " << i->second;
+    }
+*/
     const auto itr = LoadedSourcePaths.find(path_string);
     if (itr != LoadedSourcePaths.end())
     {
@@ -120,7 +130,13 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
     LoadedSources.push_back(std::move(source));
 
     util::Log() << "[source loader] ok, after " << TIMER_SEC(loading_source) << "s";
-
+/*
+    // for debug : list up all keys and values
+    util::Log() << "Num of Raster Sources : " << LoadedSourcePaths.size();
+    for (auto i = LoadedSourcePaths.begin(); i != LoadedSourcePaths.end(); ++i) {
+        util::Log() << "Key : " << i->first << " Value: " << i->second;
+    }
+*/
     return source_id;
 }