osrm-backend/features/support/cache.js

'use strict';

const d3 = require('d3-queue');
const fs = require('fs');
const util = require('util');
const path = require('path');
const mkdirp = require('mkdirp');
const hash = require('../lib/hash');
const rimraf = require('rimraf');

module.exports = function() {
    this.initializeCache = (callback) => {
        this.getOSRMHash((err, osrmHash) => {
            if (err) return callback(err);
            this.osrmHash = osrmHash;
            callback();
        });
    };

    // computes all paths for every feature
    this.setupFeatures = (features, callback) => {
        this.featureIDs = {};
        this.featureCacheDirectories = {};
        this.featureProcessedCacheDirectories = {};
        let queue = d3.queue();

        function initializeFeature(feature, callback) {
            let uri = feature.getUri();

            // setup cache for feature data
            // if OSRM_PROFILE is set to force a specific profile, then
             // include the profile name in the hash of the profile file
            hash.hashOfFile(uri, this.OSRM_PROFILE, (err, hash) => {
                if (err) return callback(err);

                // shorten uri to be realtive to 'features/'
                let featurePath = path.relative(path.resolve('./features'), uri);
                // bicycle/bollards/{HASH}/
                let featureID = path.join(featurePath, hash);

                let featureCacheDirectory = this.getFeatureCacheDirectory(featureID);
                let featureProcessedCacheDirectory = this.getFeatureProcessedCacheDirectory(featureCacheDirectory, this.osrmHash);
                this.featureIDs[uri] = featureID;
                this.featureCacheDirectories[uri] = featureCacheDirectory;
                this.featureProcessedCacheDirectories[uri] = featureProcessedCacheDirectory;

                d3.queue(1)
                  .defer(mkdirp, featureProcessedCacheDirectory)
                  .defer(this.cleanupFeatureCache.bind(this), featureCacheDirectory, hash)
                  .defer(this.cleanupProcessedFeatureCache.bind(this), featureProcessedCacheDirectory, this.osrmHash)
                  .awaitAll(callback);
            });
        }

        for (let i = 0; i < features.length; ++i) {
            queue.defer(initializeFeature.bind(this), features[i]);
        }
        queue.awaitAll(callback);
    };

    this.cleanupProcessedFeatureCache = (directory, osrmHash, callback) => {
        let parentPath = path.resolve(path.join(directory, '..'));
        fs.readdir(parentPath, (err, files) => {
            let q = d3.queue();
            function runStats(path, callback) {
                fs.stat(path, (err, stat) => {
                    if (err) return callback(err);
                    callback(null, {file: path, stat: stat});
                });
            }
            files.map(f => { q.defer(runStats, path.join(parentPath, f)); });
            q.awaitAll((err, results) => {
                if (err) return callback(err);
                let q = d3.queue();
                results.forEach(r => {
                    if (r.stat.isDirectory() && r.file.search(osrmHash) < 0) {
                        q.defer(rimraf, r.file);
                    }
                });
                q.awaitAll(callback);
            });
        });
    };

    this.cleanupFeatureCache = (directory, featureHash, callback) => {
        let parentPath = path.resolve(path.join(directory, '..'));
        fs.readdir(parentPath, (err, files) => {
            let q = d3.queue();
            files.filter(name => { return name !== featureHash;})
                 .map((f) => { q.defer(rimraf, path.join(parentPath, f)); });
            q.awaitAll(callback);
        });
    };

    this.setupFeatureCache = (feature) => {
        let uri = feature.getUri();
        this.featureID = this.featureIDs[uri];
        this.featureCacheDirectory = this.featureCacheDirectories[uri];
        this.featureProcessedCacheDirectory = this.featureProcessedCacheDirectories[uri];
    };

    this.setupScenarioCache = (scenarioID) => {
        this.scenarioCacheFile = this.getScenarioCacheFile(this.featureCacheDirectory, scenarioID);
        this.processedCacheFile = this.getProcessedCacheFile(this.featureProcessedCacheDirectory, scenarioID);
        this.inputCacheFile = this.getInputCacheFile(this.featureProcessedCacheDirectory, scenarioID);
        this.rasterCacheFile = this.getRasterCacheFile(this.featureProcessedCacheDirectory, scenarioID);
        this.speedsCacheFile = this.getSpeedsCacheFile(this.featureProcessedCacheDirectory, scenarioID);
        this.penaltiesCacheFile = this.getPenaltiesCacheFile(this.featureProcessedCacheDirectory, scenarioID);
        this.profileCacheFile = this.getProfileCacheFile(this.featureProcessedCacheDirectory, scenarioID);
    };

    // returns a hash of all OSRM code side dependencies
    this.getOSRMHash = (callback) => {
        let dependencies = [
            this.OSRM_EXTRACT_PATH,
            this.OSRM_CONTRACT_PATH,
            this.LIB_OSRM_EXTRACT_PATH,
            this.LIB_OSRM_GUIDANCE_PATH,
            this.LIB_OSRM_CONTRACT_PATH
        ];

        var addLuaFiles = (directory, callback) => {
            fs.readdir(path.normalize(directory), (err, files) => {
                if (err) return callback(err);

                var luaFiles = files.filter(f => !!f.match(/\.lua$/)).map(f => path.normalize(directory + '/' + f));
                Array.prototype.push.apply(dependencies, luaFiles);

                callback();
            });
        };

        // Note: we need a serialized queue here to ensure that the order of the files
        // passed is stable. Otherwise the hash will not be stable
        d3.queue(1)
            .defer(addLuaFiles, this.PROFILES_PATH)
            .defer(addLuaFiles, this.PROFILES_PATH + '/lib')
            .awaitAll(hash.hashOfFiles.bind(hash, dependencies, callback));
    };

    // test/cache/bicycle/bollards/{HASH}/
    this.getFeatureCacheDirectory = (featureID) => {
        return path.join(this.CACHE_PATH, featureID);
    };

    // converts the scenario titles in file prefixes
    this.getScenarioID = (scenario) => {
        let name = scenario.getName().toLowerCase().replace(/[\/\-'=,\(\):\*#]/g, '')
            .replace(/\s/g, '_').replace(/__/g, '_').replace(/\.\./g, '.')
            .substring(0, 64);
        return util.format('%d_%s', scenario.getLine(), name);
    };

    // test/cache/{feature_path}/{feature_hash}/{scenario}_raster.asc
    this.getRasterCacheFile = (featureCacheDirectory, scenarioID) => {
        return path.join(featureCacheDirectory, scenarioID) + '_raster.asc';
    };

    // test/cache/{feature_path}/{feature_hash}/{scenario}_speeds.csv
    this.getSpeedsCacheFile = (featureCacheDirectory, scenarioID) => {
        return path.join(featureCacheDirectory, scenarioID) + '_speeds.csv';
    };

    // test/cache/{feature_path}/{feature_hash}/{scenario}_penalties.csv
    this.getPenaltiesCacheFile = (featureCacheDirectory, scenarioID) => {
        return path.join(featureCacheDirectory, scenarioID) + '_penalties.csv';
    };

    // test/cache/{feature_path}/{feature_hash}/{scenario}_profile.lua
    this.getProfileCacheFile = (featureCacheDirectory, scenarioID) => {
        return path.join(featureCacheDirectory, scenarioID) + '_profile.lua';
    };

    // test/cache/{feature_path}/{feature_hash}/{scenario}.osm
    this.getScenarioCacheFile = (featureCacheDirectory, scenarioID) => {
        return path.join(featureCacheDirectory, scenarioID) + '.osm';
    };

    // test/cache/{feature_path}/{feature_hash}/{osrm_hash}/
    this.getFeatureProcessedCacheDirectory = (featureCacheDirectory, osrmHash) => {
        return path.join(featureCacheDirectory, osrmHash);
    };

    // test/cache/{feature_path}/{feature_hash}/{osrm_hash}/{scenario}.osrm
    this.getProcessedCacheFile = (featureProcessedCacheDirectory, scenarioID) => {
        return path.join(featureProcessedCacheDirectory, scenarioID) + '.osrm';
    };

    // test/cache/{feature_path}/{feature_hash}/{osrm_hash}/{scenario}.osm
    this.getInputCacheFile = (featureProcessedCacheDirectory, scenarioID) => {
        return path.join(featureProcessedCacheDirectory, scenarioID) + '.osm';
    };


    return this;
};