feat: Add ELK stack for OSRM.

issue: https://github.com/Telenav/osrm-backend/issues/65

docker-orchestration/elk-compose/docker-elk/.env

@@ -0,0 +1 @@
+ELK_VERSION=7.3.1

docker-orchestration/elk-compose/docker-elk/.gitattributes

@@ -0,0 +1,2 @@
+# Declare files that will always have LF line endings on checkout.
+*.sh text eol=lf

docker-orchestration/elk-compose/docker-elk/README.md

@@ -0,0 +1 @@
+# ELK

docker-orchestration/elk-compose/docker-elk/docker-compose.yml

@@ -0,0 +1,73 @@
+version: '3.2'
+
+services:
+  elasticsearch:
+    build:
+      context: elasticsearch/
+      args:
+        ELK_VERSION: $ELK_VERSION
+    volumes:
+      - type: bind
+        source: ./elasticsearch/config/elasticsearch.yml
+        target: /usr/share/elasticsearch/config/elasticsearch.yml
+        read_only: true
+      - type: volume
+        source: elasticsearch
+        target: /usr/share/elasticsearch/data
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+    environment:
+      ES_JAVA_OPTS: "-Xmx256m -Xms256m"
+      ELASTIC_PASSWORD: changeme
+    networks:
+      - elk
+
+  logstash:
+    build:
+      context: logstash/
+      args:
+        ELK_VERSION: $ELK_VERSION
+    volumes:
+      - /Users/xunliu/Desktop/git/elastic-example/data/:/data/
+      - type: bind
+        source: ./logstash/config/logstash.yml
+        target: /usr/share/logstash/config/logstash.yml
+        read_only: true
+      - type: bind
+        source: ./logstash/pipeline
+        target: /usr/share/logstash/pipeline
+        read_only: true
+    ports:
+      - "5000:5000"
+      - "9600:9600"
+    environment:
+      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+
+  kibana:
+    build:
+      context: kibana/
+      args:
+        ELK_VERSION: $ELK_VERSION
+    volumes:
+      - type: bind
+        source: ./kibana/config/kibana.yml
+        target: /usr/share/kibana/config/kibana.yml
+        read_only: true
+    ports:
+      - "5601:5601"
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+
+networks:
+  elk:
+    driver: bridge
+
+volumes:
+  elasticsearch:

docker-orchestration/elk-compose/docker-elk/elasticsearch/Dockerfile

@@ -0,0 +1,7 @@
+ARG ELK_VERSION
+
+# https://github.com/elastic/elasticsearch-docker
+FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
+
+# Add your elasticsearch plugins setup here
+# Example: RUN elasticsearch-plugin install analysis-icu

docker-orchestration/elk-compose/docker-elk/elasticsearch/config/elasticsearch.yml

@@ -0,0 +1,18 @@
+---
+## Default Elasticsearch configuration from Elasticsearch base image.
+## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml
+#
+cluster.name: "docker-cluster"
+network.host: 0.0.0.0
+
+## Use single node discovery in order to disable production mode and avoid bootstrap checks
+## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
+#
+discovery.type: single-node
+
+## X-Pack settings
+## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html
+#
+xpack.license.self_generated.type: basic
+xpack.security.enabled: true
+xpack.monitoring.collection.enabled: true

docker-orchestration/elk-compose/docker-elk/kibana/Dockerfile

@@ -0,0 +1,7 @@
+ARG ELK_VERSION
+
+# https://github.com/elastic/kibana-docker
+FROM docker.elastic.co/kibana/kibana:${ELK_VERSION}
+
+# Add your kibana plugins setup here
+# Example: RUN kibana-plugin install <name|url>

docker-orchestration/elk-compose/docker-elk/kibana/config/kibana.yml

@@ -0,0 +1,13 @@
+---
+## Default Kibana configuration from Kibana base image.
+## https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.js
+#
+server.name: kibana
+server.host: "0"
+elasticsearch.hosts: [ "http://elasticsearch:9200" ]
+xpack.monitoring.ui.container.elasticsearch.enabled: true
+
+## X-Pack security credentials
+#
+elasticsearch.username: elastic
+elasticsearch.password: changeme

docker-orchestration/elk-compose/docker-elk/logstash/Dockerfile

@@ -0,0 +1,7 @@
+ARG ELK_VERSION
+
+# https://github.com/elastic/logstash-docker
+FROM docker.elastic.co/logstash/logstash:${ELK_VERSION}
+
+# Add your logstash plugins setup here
+# Example: RUN logstash-plugin install logstash-filter-json

docker-orchestration/elk-compose/docker-elk/logstash/config/logstash.yml

@@ -0,0 +1,12 @@
+---
+## Default Logstash configuration from Logstash base image.
+## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml
+#
+http.host: "0.0.0.0"
+xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]
+
+## X-Pack security credentials
+#
+xpack.monitoring.enabled: true
+xpack.monitoring.elasticsearch.username: elastic
+xpack.monitoring.elasticsearch.password: changeme

docker-orchestration/elk-compose/docker-elk/logstash/pipeline/logstash.conf

@@ -0,0 +1,32 @@
+input {
+    file {
+        path => "/data/*.log"
+        start_position => "beginning"
+    }
+}
+
+filter {
+
+    grok {
+        match => { "message" => "\[%{TIMESTAMP_ISO8601:timestamp} UTC\] \[%{LOGLEVEL:log-level}\] Used %{NUMBER:lua-speed-items} speeds from LUA profile or input map" }
+        add_field => { "subType" => "traffic-speed-update" }
+    }
+
+	if "_grokparsefailure" in [tags] {
+    grok {
+        match => { "message" => "\[%{TIMESTAMP_ISO8601:timestamp} UTC\]\s\[%{LOGLEVEL:log-level}\]\s%{DATE_EU:request-date}\s%{TIME:request-time}\s%{DATA:response_duration}\s%{IP:request-ip}\s\W\s(?<user-agent>.+?(?=\s\d{3}\s))\s(?<response-code>\d{3})\s%{URIPATH:uri}" }
+        add_tag => ["request"]
+    }
+  }
+}
+
+## Add your filters / logstash plugins configuration here
+
+output {
+	elasticsearch {
+		hosts => "elasticsearch:9200"
+		user => "elastic"
+		password => "changeme"
+		index => "osrm"
+	}
+}