Add elk stack image build instructions

README.md

@@ -1,12 +1,3 @@
 ## Instructions
 
-1. Pull the filebeat image
+Note: cd to the exact directory and build the images.
-```bash
-docker pull docker.elastic.co/beats/filebeat:8.2.0
-```
-Note: Change the filebeat image name or tag in Dockerfile related to the pulled version.
-
-3. Build the Docker image
-```bash
-docker build -t filebeat:8.2.0-entgra-v1 .
-```

entgra-elasticsearch/Dockerfile

@@ -0,0 +1,5 @@
+FROM docker.elastic.co/elasticsearch/elasticsearch:8.2.0
+
+RUN rm -Rf /usr/share/elasticsearch/config/elasticsearch.yml
+
+COPY elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml

entgra-elasticsearch/README.md

@@ -0,0 +1,12 @@
+## Instructions
+
+1. Pull the elasticsearch image
+```bash
+docker pull docker.elastic.co/elasticsearch/elasticsearch:8.2.0
+```
+Note: Change the required image version name or tag in the Dockerfile.
+
+3. Build the Docker image
+```bash
+docker build -t elasticsearch:8.2.0-entgra-v1 .
+```

entgra-elasticsearch/docker-compose.yml

@@ -0,0 +1,26 @@
+version: "3.8"
+
+services:
+  elasticsearch:
+    image: elasticsearch:8.2.0-entgra-v2
+    container_name: elasticsearch
+    restart: unless-stopped
+    environment:
+      - TZ=Asia/Colombo
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+    volumes:
+      - ./files/data:/usr/share/elasticsearch/data
+      - ./files/backup:/usr/share/elasticsearch/backup
+#      - ./files/configs:/usr/share/elasticsearch/config
+      - ./files/logs:/usr/share/elasticsearch/logs
+    ports:
+      - "9200:9200"
+    healthcheck:
+      test: curl --fail http://localhost:9200/ || exit 1
+      interval: 60s
+      retries: 10
+      start_period: 30s
+      timeout: 10s

entgra-elasticsearch/elasticsearch.yml

@@ -0,0 +1,8 @@
+cluster.name: "docker-cluster"
+network.host: 0.0.0.0
+
+# minimum_master_nodes need to be explicitly set when bound on a public IP
+# # set to 1 to allow single node clusters
+# # Details: https://github.com/elastic/elasticsearch/pull/17288
+# discovery.zen.minimum_master_nodes: 1
+path.repo: ["/usr/share/elasticsearch/backup"]

entgra-filebeat/README.md

@@ -0,0 +1,12 @@
+## Instructions
+
+1. Pull the filebeat image
+```bash
+docker pull docker.elastic.co/beats/filebeat:8.2.0
+```
+Note: Change the required image version name or tag in the Dockerfile.
+
+3. Build the Docker image
+```bash
+docker build -t filebeat:8.2.0-entgra-v1 .
+```

entgra-filebeat/docker-compose.yml

@@ -9,7 +9,7 @@ services:
       - "host.docker.internal:host-gateway"
     volumes:
       - ./files/iots/logs/:/opt/logs
-      - ./files/filebeat/opt/filebeat-configs:/opt/filebeat-configs
+      - ./files/filebeat/:/opt/filebeat-configs
     command: ["--strict.perms=false"]
     environment:
       - TZ=Asia/Colombo

entgra-kibana/Dockerfile

@@ -0,0 +1,3 @@
+FROM docker.elastic.co/kibana/kibana:8.2.0
+
+COPY kibana.yml /usr/share/kibana/kibana.yml

entgra-kibana/README.md

@@ -0,0 +1,12 @@
+## Instructions
+
+1. Pull the kibana image
+```bash
+docker pull docker.elastic.co/kibana/kibana:8.2.0
+```
+Note: Change the required image version name or tag in the Dockerfile.
+
+3. Build the Docker image
+```bash
+docker build -t kibana:8.2.0-entgra-v1 .
+```

entgra-kibana/kibana.yml

@@ -0,0 +1,15 @@
+security.showInsecureClusterWarning: false
+monitoring.ui.container.elasticsearch.enabled: true
+logging.quiet: true
+logging.verbose: false
+
+logging:
+  appenders:
+    rolling-file:
+      type: rolling-file
+      fileName: /var/logs/kibana/kibana.log
+      policy:
+        type: size-limit
+        size: 10mb
+      layout:
+        type: pattern

entgra-logstash/Dockerfile

@@ -0,0 +1,3 @@
+FROM docker.elastic.co/logstash/logstash:8.2.0
+
+COPY logstash.conf /usr/share/logstash/pipeline/logstash.conf

entgra-logstash/README.md

@@ -0,0 +1,12 @@
+## Instructions
+
+1. Pull the logstash image
+```bash
+docker pull docker.elastic.co/logstash/logstash:8.2.0
+```
+Note: Change the required image version name or tag in the Dockerfile.
+
+3. Build the Docker image
+```bash
+docker build -t logstash:8.2.0-entgra-v1 .
+```

entgra-logstash/logstash.conf

@@ -0,0 +1,39 @@
+input {
+        beats {
+                port => 5044
+        }
+}
+filter {
+
+        grok {
+             match => {
+                  "message" => "\[%{GREEDYDATA:LogLevel}]\ \[%{TIMESTAMP_ISO8601:logdate}\] \[%{DATA:EventCategory}\] \[%{DATA:InitiatedBy}\] \[%{DATA:SgId}\] \[%{DATA:CbSerial}\] \[%{DATA:OcSerial}\] \[%{DATA:IpAddress}\] \[%{DATA:State}\] - %{GREEDYDATA:Description}"
+             }
+        }
+
+        if [log][file][path] =~ "/var/log/auth.log" {
+                drop { }
+        }
+        if [fileset][name] =~ "syslog" {
+                drop { }
+        }
+        date {
+                match => [ "logdate", "MMM D, YYYY @ HH:mm:ss.SSS", "ISO8601"]
+                target => "@timestamp"
+        }
+}
+output {
+
+        if "SwitchGearLogs" in [tags] {
+                elasticsearch {
+                        hosts => ["elasticsearch:9200"]
+                        index => "switchgearlogs-%{+YYYY.MM.dd}"
+                }
+        }
+
+#       elasticsearch{
+#               hosts => ["elasticsearch:9200"]
+#               index => "lecologs-%{+YYYY.MM.dd}"
+#       }
+
+}