From 4dce36fcb39dd89cefb39d78ba1d68bbdf2b0865 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 15 Oct 2019 18:09:45 -0700 Subject: [PATCH] Add ELK stack for OSRM. --- .../elk-compose/docker-elk/.env | 1 + .../elk-compose/docker-elk/.gitattributes | 2 + .../elk-compose/docker-elk/README.md | 1 + .../elk-compose/docker-elk/docker-compose.yml | 73 +++++++++++++++++++ .../docker-elk/elasticsearch/Dockerfile | 7 ++ .../elasticsearch/config/elasticsearch.yml | 18 +++++ .../elk-compose/docker-elk/kibana/Dockerfile | 7 ++ .../docker-elk/kibana/config/kibana.yml | 13 ++++ .../docker-elk/logstash/Dockerfile | 7 ++ .../docker-elk/logstash/config/logstash.yml | 12 +++ .../logstash/pipeline/logstash.conf | 32 ++++++++ 11 files changed, 173 insertions(+) create mode 100644 docker-orchestration/elk-compose/docker-elk/.env create mode 100644 docker-orchestration/elk-compose/docker-elk/.gitattributes create mode 100644 docker-orchestration/elk-compose/docker-elk/README.md create mode 100644 docker-orchestration/elk-compose/docker-elk/docker-compose.yml create mode 100644 docker-orchestration/elk-compose/docker-elk/elasticsearch/Dockerfile create mode 100644 docker-orchestration/elk-compose/docker-elk/elasticsearch/config/elasticsearch.yml create mode 100644 docker-orchestration/elk-compose/docker-elk/kibana/Dockerfile create mode 100644 docker-orchestration/elk-compose/docker-elk/kibana/config/kibana.yml create mode 100644 docker-orchestration/elk-compose/docker-elk/logstash/Dockerfile create mode 100644 docker-orchestration/elk-compose/docker-elk/logstash/config/logstash.yml create mode 100644 docker-orchestration/elk-compose/docker-elk/logstash/pipeline/logstash.conf diff --git a/docker-orchestration/elk-compose/docker-elk/.env b/docker-orchestration/elk-compose/docker-elk/.env new file mode 100644 index 000000000..311b6cb68 --- /dev/null +++ b/docker-orchestration/elk-compose/docker-elk/.env @@ -0,0 +1 @@ +ELK_VERSION=7.3.1 diff --git a/docker-orchestration/elk-compose/docker-elk/.gitattributes b/docker-orchestration/elk-compose/docker-elk/.gitattributes new file mode 100644 index 000000000..2858dda11 --- /dev/null +++ b/docker-orchestration/elk-compose/docker-elk/.gitattributes @@ -0,0 +1,2 @@ +# Declare files that will always have LF line endings on checkout. +*.sh text eol=lf \ No newline at end of file diff --git a/docker-orchestration/elk-compose/docker-elk/README.md b/docker-orchestration/elk-compose/docker-elk/README.md new file mode 100644 index 000000000..27a4b0fc4 --- /dev/null +++ b/docker-orchestration/elk-compose/docker-elk/README.md @@ -0,0 +1 @@ +# ELK diff --git a/docker-orchestration/elk-compose/docker-elk/docker-compose.yml b/docker-orchestration/elk-compose/docker-elk/docker-compose.yml new file mode 100644 index 000000000..b99cda57a --- /dev/null +++ b/docker-orchestration/elk-compose/docker-elk/docker-compose.yml @@ -0,0 +1,73 @@ +version: '3.2' + +services: + elasticsearch: + build: + context: elasticsearch/ + args: + ELK_VERSION: $ELK_VERSION + volumes: + - type: bind + source: ./elasticsearch/config/elasticsearch.yml + target: /usr/share/elasticsearch/config/elasticsearch.yml + read_only: true + - type: volume + source: elasticsearch + target: /usr/share/elasticsearch/data + ports: + - "9200:9200" + - "9300:9300" + environment: + ES_JAVA_OPTS: "-Xmx256m -Xms256m" + ELASTIC_PASSWORD: changeme + networks: + - elk + + logstash: + build: + context: logstash/ + args: + ELK_VERSION: $ELK_VERSION + volumes: + - /Users/xunliu/Desktop/git/elastic-example/data/:/data/ + - type: bind + source: ./logstash/config/logstash.yml + target: /usr/share/logstash/config/logstash.yml + read_only: true + - type: bind + source: ./logstash/pipeline + target: /usr/share/logstash/pipeline + read_only: true + ports: + - "5000:5000" + - "9600:9600" + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + depends_on: + - elasticsearch + + kibana: + build: + context: kibana/ + args: + ELK_VERSION: $ELK_VERSION + volumes: + - type: bind + source: ./kibana/config/kibana.yml + target: /usr/share/kibana/config/kibana.yml + read_only: true + ports: + - "5601:5601" + networks: + - elk + depends_on: + - elasticsearch + +networks: + elk: + driver: bridge + +volumes: + elasticsearch: diff --git a/docker-orchestration/elk-compose/docker-elk/elasticsearch/Dockerfile b/docker-orchestration/elk-compose/docker-elk/elasticsearch/Dockerfile new file mode 100644 index 000000000..24278f89d --- /dev/null +++ b/docker-orchestration/elk-compose/docker-elk/elasticsearch/Dockerfile @@ -0,0 +1,7 @@ +ARG ELK_VERSION + +# https://github.com/elastic/elasticsearch-docker +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION} + +# Add your elasticsearch plugins setup here +# Example: RUN elasticsearch-plugin install analysis-icu diff --git a/docker-orchestration/elk-compose/docker-elk/elasticsearch/config/elasticsearch.yml b/docker-orchestration/elk-compose/docker-elk/elasticsearch/config/elasticsearch.yml new file mode 100644 index 000000000..cbe96ab47 --- /dev/null +++ b/docker-orchestration/elk-compose/docker-elk/elasticsearch/config/elasticsearch.yml @@ -0,0 +1,18 @@ +--- +## Default Elasticsearch configuration from Elasticsearch base image. +## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml +# +cluster.name: "docker-cluster" +network.host: 0.0.0.0 + +## Use single node discovery in order to disable production mode and avoid bootstrap checks +## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html +# +discovery.type: single-node + +## X-Pack settings +## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html +# +xpack.license.self_generated.type: basic +xpack.security.enabled: true +xpack.monitoring.collection.enabled: true diff --git a/docker-orchestration/elk-compose/docker-elk/kibana/Dockerfile b/docker-orchestration/elk-compose/docker-elk/kibana/Dockerfile new file mode 100644 index 000000000..202855cc9 --- /dev/null +++ b/docker-orchestration/elk-compose/docker-elk/kibana/Dockerfile @@ -0,0 +1,7 @@ +ARG ELK_VERSION + +# https://github.com/elastic/kibana-docker +FROM docker.elastic.co/kibana/kibana:${ELK_VERSION} + +# Add your kibana plugins setup here +# Example: RUN kibana-plugin install diff --git a/docker-orchestration/elk-compose/docker-elk/kibana/config/kibana.yml b/docker-orchestration/elk-compose/docker-elk/kibana/config/kibana.yml new file mode 100644 index 000000000..f9844cdce --- /dev/null +++ b/docker-orchestration/elk-compose/docker-elk/kibana/config/kibana.yml @@ -0,0 +1,13 @@ +--- +## Default Kibana configuration from Kibana base image. +## https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.js +# +server.name: kibana +server.host: "0" +elasticsearch.hosts: [ "http://elasticsearch:9200" ] +xpack.monitoring.ui.container.elasticsearch.enabled: true + +## X-Pack security credentials +# +elasticsearch.username: elastic +elasticsearch.password: changeme diff --git a/docker-orchestration/elk-compose/docker-elk/logstash/Dockerfile b/docker-orchestration/elk-compose/docker-elk/logstash/Dockerfile new file mode 100644 index 000000000..6cc863745 --- /dev/null +++ b/docker-orchestration/elk-compose/docker-elk/logstash/Dockerfile @@ -0,0 +1,7 @@ +ARG ELK_VERSION + +# https://github.com/elastic/logstash-docker +FROM docker.elastic.co/logstash/logstash:${ELK_VERSION} + +# Add your logstash plugins setup here +# Example: RUN logstash-plugin install logstash-filter-json diff --git a/docker-orchestration/elk-compose/docker-elk/logstash/config/logstash.yml b/docker-orchestration/elk-compose/docker-elk/logstash/config/logstash.yml new file mode 100644 index 000000000..a48c35ff5 --- /dev/null +++ b/docker-orchestration/elk-compose/docker-elk/logstash/config/logstash.yml @@ -0,0 +1,12 @@ +--- +## Default Logstash configuration from Logstash base image. +## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml +# +http.host: "0.0.0.0" +xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ] + +## X-Pack security credentials +# +xpack.monitoring.enabled: true +xpack.monitoring.elasticsearch.username: elastic +xpack.monitoring.elasticsearch.password: changeme diff --git a/docker-orchestration/elk-compose/docker-elk/logstash/pipeline/logstash.conf b/docker-orchestration/elk-compose/docker-elk/logstash/pipeline/logstash.conf new file mode 100644 index 000000000..dc129c43d --- /dev/null +++ b/docker-orchestration/elk-compose/docker-elk/logstash/pipeline/logstash.conf @@ -0,0 +1,32 @@ +input { + file { + path => "/data/*.log" + start_position => "beginning" + } +} + +filter { + + grok { + match => { "message" => "\[%{TIMESTAMP_ISO8601:timestamp} UTC\] \[%{LOGLEVEL:log-level}\] Used %{NUMBER:lua-speed-items} speeds from LUA profile or input map" } + add_field => { "subType" => "traffic-speed-update" } + } + + if "_grokparsefailure" in [tags] { + grok { + match => { "message" => "\[%{TIMESTAMP_ISO8601:timestamp} UTC\]\s\[%{LOGLEVEL:log-level}\]\s%{DATE_EU:request-date}\s%{TIME:request-time}\s%{DATA:response_duration}\s%{IP:request-ip}\s\W\s(?.+?(?=\s\d{3}\s))\s(?\d{3})\s%{URIPATH:uri}" } + add_tag => ["request"] + } + } +} + +## Add your filters / logstash plugins configuration here + +output { + elasticsearch { + hosts => "elasticsearch:9200" + user => "elastic" + password => "changeme" + index => "osrm" + } +}