Fix publishing node binaries

2017-03-22 00:51:07 +00:00 · 2017-03-22 00:51:07 +00:00 · 927dea37bb
commit 927dea37bb
parent de98ae57b7
6 changed files with 161 additions and 123 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -26,6 +26,8 @@ cache:

 env:
  global:
+   - secure: "hk+32aXXF5t1ApaM2Wjqooz3dx1si907L87WRMkO47WlpJmUUU/Ye+MJk9sViH8MdhOcceocVAmdYl5/WFWOIbDWNlBya9QvXDZyIu2KIre/0QyOCTZbrsif8paBXKIO5O/R4OTvIZ8rvWZsadBdmAT9GSbDhih6FzqXAEgeIYQ="
+   - secure: "VE+cFkseFwW4jK6XwkP0yW3h4DixPJ8+Eb3yKcchGZ5iIJxlZ/8i1vKHYxadgPRwSYwPSB14tF70xj2OmiT2keGzZUfphmPXinBaLEhYk+Bde+GZZkoSl5ND109I/LcyNr0nG9dDgtV6pkvFchgchpyP9JnVOOS0+crEZlAz0RE="
   - CCACHE_TEMPDIR=/tmp/.ccache-temp
   - CCACHE_COMPRESS=1
   - CASHER_TIME_OUT=599 # one second less than 10m to avoid 10m timeout error: https://github.com/Project-OSRM/osrm-backend/issues/2742
@ -228,6 +230,5 @@ after_success:

  - |
    if [ -n "${ENABLE_NODE_BINDINGS}" ]; then
-      source ./scripts/travis/build.sh
      ./scripts/travis/publish.sh
    fi
--- a/cloudformation/ci.template
+++ b/cloudformation/ci.template
@ -0,0 +1,75 @@
+{
+    "AWSTemplateFormatVersion": "2010-09-09",
+    "Description": "user for publishing to s3://mapbox-node-binary/osrm",
+    "Resources": {
+        "User": {
+            "Type": "AWS::IAM::User",
+            "Properties": {
+                "Policies": [
+                    {
+                        "PolicyName": "list",
+                        "PolicyDocument": {
+                            "Statement": [
+                                {
+                                    "Action": [
+                                        "s3:ListBucket"
+                                    ],
+                                    "Effect": "Allow",
+                                    "Resource": "arn:aws:s3:::mapbox-node-binary",
+                                    "Condition": {
+                                        "StringLike": {
+                                            "s3:prefix": [
+                                                "osrm/*"
+                                            ]
+                                        }
+                                    }
+                                }
+                            ]
+                        }
+                    },
+                    {
+                        "PolicyName": "publish",
+                        "PolicyDocument": {
+                            "Statement": [
+                                {
+                                    "Action": [
+                                        "s3:DeleteObject",
+                                        "s3:GetObject",
+                                        "s3:GetObjectAcl",
+                                        "s3:PutObject",
+                                        "s3:PutObjectAcl"
+                                    ],
+                                    "Effect": "Allow",
+                                    "Resource": "arn:aws:s3:::mapbox-node-binary/osrm/*"
+                                }
+                            ]
+                        }
+                    }
+                ]
+            }
+        },
+        "AccessKey": {
+            "Type": "AWS::IAM::AccessKey",
+            "Properties": {
+                "UserName": {
+                    "Ref": "User"
+                }
+            }
+        }
+    },
+    "Outputs": {
+        "AccessKeyId": {
+            "Value": {
+                "Ref": "AccessKey"
+            }
+        },
+        "SecretAccessKey": {
+            "Value": {
+                "Fn::GetAtt": [
+                    "AccessKey",
+                    "SecretAccessKey"
+                ]
+            }
+        }
+    }
+}
--- a/cloudformation/ci.template.js
+++ b/cloudformation/ci.template.js
@ -0,0 +1,59 @@
+var cf = require('@mapbox/cloudfriend');
+var package_json = require('../package.json')
+
+module.exports = {
+  AWSTemplateFormatVersion: '2010-09-09',
+  Description: 'user for publishing to s3://mapbox-node-binary/' + package_json.name,
+  Resources: {
+    User: {
+      Type: 'AWS::IAM::User',
+      Properties: {
+        Policies: [
+          {
+            PolicyName: 'list',
+            PolicyDocument: {
+              Statement: [
+                {
+                  Action: ['s3:ListBucket'],
+                  Effect: 'Allow',
+                  Resource: 'arn:aws:s3:::mapbox-node-binary',
+                  Condition : {
+                    StringLike : {
+                      "s3:prefix": [ package_json.name + "/*"]
+                    }
+                  }
+                }
+              ]
+            }
+          },
+          {
+            PolicyName: 'publish',
+            PolicyDocument: {
+              Statement: [
+                {
+                  Action: ['s3:DeleteObject', 's3:GetObject', 's3:GetObjectAcl', 's3:PutObject', 's3:PutObjectAcl'],
+                  Effect: 'Allow',
+                  Resource: 'arn:aws:s3:::mapbox-node-binary/' + package_json.name + '/*'
+                }
+              ]
+            }
+          }
+        ]
+      }
+    },
+    AccessKey: {
+      Type: 'AWS::IAM::AccessKey',
+      Properties: {
+        UserName: cf.ref('User')
+      }
+    }
+  },
+  Outputs: {
+    AccessKeyId: {
+      Value: cf.ref('AccessKey')
+    },
+    SecretAccessKey: {
+      Value: cf.getAtt('AccessKey', 'SecretAccessKey')
+    }
+  }
+};
--- a/package.json
+++ b/package.json
@ -1,24 +1,12 @@
 {
  "name": "osrm",
-  "version": "5.7.0-latest",
-  "private": true,
+  "version": "5.7.0-alpha.2",
+  "private": false,
  "description": "The Open Source Routing Machine is a high performance routing engine written in C++14 designed to run on OpenStreetMap data.",
  "dependencies": {
-    "chalk": "^1.1.3",
-    "cucumber": "^1.2.1",
-    "d3-queue": "^2.0.3",
-    "mkdirp": "^0.5.1",
    "nan": "^2.1.0",
    "node-cmake": "^1.2.1",
-    "node-pre-gyp": "^0.6.34",
-    "node-timeout": "0.0.4",
-    "polyline": "^0.2.0",
-    "request": "^2.69.0",
-    "rimraf": "^2.5.4",
-    "xmlbuilder": "^4.2.1"
-  },
-  "bin": {
-    "cucumber": "./node_modules/cucumber/bin/cucumber.js"
+    "node-pre-gyp": "^0.6.34"
  },
  "browserify": {
    "transform": [
@ -29,8 +17,7 @@
  "scripts": {
    "lint": "eslint -c ./.eslintrc features/step_definitions/ features/support/",
    "test": "npm run lint && node ./node_modules/cucumber/bin/cucumber.js features/ -p verify && node ./node_modules/cucumber/bin/cucumber.js features/ -p mld",
-    "clean-test": "rm -rf test/cache",
-    "cucumber": "./node_modules/cucumber/bin/cucumber.js",
+    "clean": "rm -rf test/cache",
    "docs": "./scripts/build_api_docs.sh",
    "install": "node-pre-gyp install --fallback-to-build=false"
  },
@ -51,9 +38,17 @@
    "docbox": "^1.0.5",
    "documentation": "^4.0.0-beta.18",
    "eslint": "^2.4.0",
-
+    "chalk": "^1.1.3",
+    "cucumber": "^1.2.1",
+    "d3-queue": "^2.0.3",
+    "mkdirp": "^0.5.1",
    "aws-sdk": "~2.0.31",
-    "tape": "^4.2.2"
+    "tape": "^4.2.2",
+    "node-timeout": "0.0.4",
+    "polyline": "^0.2.0",
+    "request": "^2.69.0",
+    "rimraf": "^2.5.4",
+    "xmlbuilder": "^4.2.1"
  },
  "bundleDependencies": [
    "node-pre-gyp"
--- a/scripts/travis/build.sh
+++ b/scripts/travis/build.sh
@ -1,99 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-set -o pipefail
-
-# defaults
-export ENABLE_COVERAGE=${ENABLE_COVERAGE:-"Off"}
-export BUILD_TYPE=${BUILD_TYPE:-"Release"}
-export NODE=${NODE:-4}
-
-export CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-export DEPS_DIR="$(pwd)/deps"
-export PATH=${DEPS_DIR}/bin:${PATH}
-mkdir -p ${DEPS_DIR}
-
-export CLANG_VERSION="${CLANG_VERSION:-4.0.0}"
-export CCACHE_VERSION=3.3.1
-export CMAKE_VERSION=3.7.2
-
-source ${CURRENT_DIR}/travis_helper.sh
-
-# ensure we start inside the root directory (two level up)
-cd ${CURRENT_DIR}/../../
-
-if [[ ! $(which wget) ]]; then
-    echo "echo wget must be installed";
-    exit 1;
-fi;
-
-SYSTEM_NAME=$(uname -s)
-if [[ "${SYSTEM_NAME}" == "Darwin" ]]; then
-    OS_NAME="osx"
-elif [[ "${SYSTEM_NAME}" == "Linux" ]]; then
-    OS_NAME="linux"
-fi
-
-# FIXME This should be replaced by proper calls to mason but we currently have a chicken-egg problem
-# since we rely on osrm-backend to ship mason for us. Once we merged this into osrm-backend this will not be needed.
-CMAKE_URL="https://s3.amazonaws.com/mason-binaries/${OS_NAME}-x86_64/cmake/${CMAKE_VERSION}.tar.gz"
-echo "Downloading cmake from ${CMAKE_URL} ..."
-wget --quiet -O - ${CMAKE_URL} | tar --strip-components=1 -xz -C ${DEPS_DIR} || exit 1
-CCACHE_URL="https://s3.amazonaws.com/mason-binaries/${OS_NAME}-x86_64/ccache/${CCACHE_VERSION}.tar.gz"
-echo "Downloading ccache from ${CCACHE_URL} ..."
-wget --quiet -O - ${CCACHE_URL} | tar --strip-components=1 -xz -C ${DEPS_DIR} || exit 1
-# install clang for linux but use the xcode version on OSX
-if [[ "${OS_NAME}" != "osx" ]]; then
-    CLANG_URL="https://s3.amazonaws.com/mason-binaries/${OS_NAME}-x86_64/clang++/${CLANG_VERSION}.tar.gz"
-    echo "Downloading clang from ${CLANG_URL} ..."
-    wget --quiet -O - ${CLANG_URL} | tar --strip-components=1 -xz -C ${DEPS_DIR} || exit 1
-    export CCOMPILER='clang'
-    export CXXCOMPILER='clang++'
-    export CC='clang'
-    export CXX='clang++'
-fi
-
-if [[ "${OS_NAME}" == "osx" ]]; then
-    if [[ -f /etc/sysctl.conf ]] && [[ $(grep shmmax /etc/sysctl.conf) ]]; then
-        echo "Note: found shmmax setting in /etc/sysctl.conf, not modifying"
-    else
-        echo "WARNING: Did not find shmmax setting in /etc/sysctl.conf, adding now (requires sudo and restarting)..."
-        sudo sysctl -w kern.sysv.shmmax=4294967296
-        sudo sysctl -w kern.sysv.shmall=1048576
-        sudo sysctl -w kern.sysv.shmseg=128
-    fi
-fi
-
-
-echo "Now build node-osrm and dependencies"
-export VERBOSE=1
-if [[ "${ENABLE_COVERAGE}" == "On" ]]; then
-    mapbox_time "make" make -j4 coverage
-else
-    mkdir -p build
-    pushd build
-    cmake .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DENABLE_NODE_BINDINGS=On -DENABLE_MASON=On
-    mapbox_time "make" make -j4
-    popd
-fi
-
-## run tests, with backtrace support
-#if [[ "${OS_NAME}" == "linux" ]]; then
-#    ulimit -c unlimited -S
-#    RESULT=0
-#    mapbox_time "make-test" make tests || RESULT=$?
-#    for i in $(find ./ -maxdepth 1 -name 'core*' -print);
-#      do gdb $(which node) $i -ex "thread apply all bt" -ex "set pagination 0" -batch;
-#    done;
-#    if [[ ${RESULT} != 0 ]]; then exit $RESULT; fi
-#else
-#    # todo: coredump support on OS X
-#    RESULT=0
-#    mapbox_time "make-test" make tests || RESULT=$?
-#    if [[ ${RESULT} != 0 ]]; then exit $RESULT; fi
-#fi
-
-
-set +eu
-set +o pipefail
--- a/scripts/travis/publish.sh
+++ b/scripts/travis/publish.sh
@ -29,17 +29,24 @@ fi

 echo "determining publishing status..."

+export COMMIT_MESSAGE=$(git log --format=%B --no-merges | head -n 1 | tr -d '\n')
+echo "Commit message: ${COMMIT_MESSAGE}"
+
 if [[ $(./scripts/travis/is_pr_merge.sh) ]]; then
-    echo "Skipping publishing because this is a PR merge commit"
+    if [[ ${COMMIT_MESSAGE} =~ "[force publish binary]" ]]; then
+        echo "Publishing because it's forced"
+        ./node_modules/.bin/node-pre-gyp package ${NPM_FLAGS}
+        ./node_modules/.bin/node-pre-gyp publish ${NPM_FLAGS}
+    else
+        echo "Skipping publishing because this is a PR merge commit"
+    fi
 else
    echo "This is a push commit, continuing to package..."
    ./node_modules/.bin/node-pre-gyp package ${NPM_FLAGS}

-    export COMMIT_MESSAGE=$(git log --format=%B --no-merges | head -n 1 | tr -d '\n')
-    echo "Commit message: ${COMMIT_MESSAGE}"
-
    if [[ ${COMMIT_MESSAGE} =~ "[publish binary]" ]]; then
        echo "Publishing"
+        ./node_modules/.bin/node-pre-gyp package ${NPM_FLAGS}
        ./node_modules/.bin/node-pre-gyp publish ${NPM_FLAGS}
    elif [[ ${COMMIT_MESSAGE} =~ "[republish binary]" ]]; then
        echo "*** Error: Republishing is disallowed for this repository"