Merge pull request 'Add elk stack image build instructions' (#3) from pasindu/entgra-filebeat:RemoveJQUsage into master

Reviewed-on: #3
2 months ago · b7a2e581a8
parent 7734be4321 4bfb7fabd8
commit b7a2e581a8
18 changed files with 389 additions and 58 deletions
--- a/12
+++ b/12
@ -1,12 +0,0 @@
-FROM docker.elastic.co/beats/filebeat:8.2.0
-
-# Switch to root user
-USER root
-
-#install jq to the container
-RUN apt-get update && apt-get install -y jq
-
-COPY entrypoint.sh /usr/local/bin/entrypoint.sh
-RUN chmod +x /usr/local/bin/entrypoint.sh
-
-ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
--- a/README.md
+++ b/README.md
@ -1,12 +1,3 @@
 ## Instructions

-1. Pull the filebeat image
-```bash
-docker pull docker.elastic.co/beats/filebeat:8.2.0
-```
-Note: Change the filebeat image name or tag in Dockerfile related to the pulled version.
-
-3. Build the Docker image
-```bash
-docker build -t filebeat:8.2.0-entgra-v1 .
-```
+Note: cd to the exact directory and build the images.
--- a/entgra-elasticsearch/Dockerfile
+++ b/entgra-elasticsearch/Dockerfile
@ -0,0 +1,21 @@
+# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
+#
+# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM docker.elastic.co/elasticsearch/elasticsearch:8.2.0
+
+RUN rm -Rf /usr/share/elasticsearch/config/elasticsearch.yml
+
+COPY elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml
--- a/entgra-elasticsearch/README.md
+++ b/entgra-elasticsearch/README.md
@ -0,0 +1,10 @@
+## Instructions
+
+1. Build the Docker image
+```bash
+docker build -t registry.entgra.io/entgra-elasticsearch:8.2.0-entgra-v2 -f Dockerfile .
+```
+
+2. docker-compose up -d 
+3. docker exec -it $(docker ps -aq -n 1) /bin/bash 
+4. docker-compose down
--- a/entgra-elasticsearch/docker-compose.yml
+++ b/entgra-elasticsearch/docker-compose.yml
@ -0,0 +1,42 @@
+# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
+#
+# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version: "3.8"
+
+services:
+  elasticsearch:
+    image: elasticsearch:8.2.0-entgra-v2
+    container_name: elasticsearch
+    restart: unless-stopped
+    environment:
+      - TZ=Asia/Colombo
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+    volumes:
+      - ./files/data:/usr/share/elasticsearch/data
+      - ./files/backup:/usr/share/elasticsearch/backup
+#      - ./files/configs:/usr/share/elasticsearch/config
+      - ./files/logs:/usr/share/elasticsearch/logs
+    ports:
+      - "9200:9200"
+    healthcheck:
+      test: curl --fail http://localhost:9200/ || exit 1
+      interval: 60s
+      retries: 10
+      start_period: 30s
+      timeout: 10s
--- a/entgra-elasticsearch/elasticsearch.yml
+++ b/entgra-elasticsearch/elasticsearch.yml
@ -0,0 +1,24 @@
+# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
+#
+# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cluster.name: "docker-cluster"
+network.host: 0.0.0.0
+
+# minimum_master_nodes need to be explicitly set when bound on a public IP
+# # set to 1 to allow single node clusters
+# # Details: https://github.com/elastic/elasticsearch/pull/17288
+# discovery.zen.minimum_master_nodes: 1
+path.repo: ["/usr/share/elasticsearch/backup"]
--- a/entgra-filebeat/Dockerfile
+++ b/entgra-filebeat/Dockerfile
@ -0,0 +1,29 @@
+# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
+#
+# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM docker.elastic.co/beats/filebeat:8.2.0
+
+# Switch to root user
+USER root
+
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+# Set the entry point to your custom script using tini
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/entrypoint.sh"]
+
+# Set the CMD to pass the environment argument to Filebeat
+CMD ["-environment", "container"]
--- a/entgra-filebeat/README.md
+++ b/entgra-filebeat/README.md
@ -0,0 +1,11 @@
+## Instructions
+
+1. Build the Docker image
+```bash
+docker build -t registry.entgra.io/entgra-filebeat:8.2.0-entgra-v2 -f Dockerfile .
+```
+
+2. docker-compose up -d
+3. docker exec -it $(docker ps -aq -n 1) /bin/bash
+4. docker-compose down
+
--- a/entgra-filebeat/docker-compose.yml
+++ b/entgra-filebeat/docker-compose.yml
@ -0,0 +1,33 @@
+# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
+#
+# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version: "3.8"
+
+services:
+  entgra-filebeat:
+    image: filebeat:8.2.0-entgra-v2
+    container_name: entgra-filebeat
+    restart: unless-stopped
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - ./files/iots/logs/:/opt/logs
+      - ./files/filebeat/:/opt/filebeat-configs
+    command: ["--strict.perms=false"]
+    environment:
+      - TZ=Asia/Colombo
+      - LOGSTASH_HOST=logstash
+      - LOGSTASH_PORT=5044
--- a/entgra-filebeat/entrypoint.sh
+++ b/entgra-filebeat/entrypoint.sh
@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
+#
+# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Check if ECS_CONTAINER_METADATA_FILE is set
+if [ -z "$ECS_CONTAINER_METADATA_FILE" ]; then
+  echo "ECS_CONTAINER_METADATA_FILE is not set or the server is running on local deployment environment"
+fi
+
+# Extract HostPrivateIPv4Address from the metadata file
+HOST_PRIVATE_IP=$(cat $ECS_CONTAINER_METADATA_FILE | grep -oP '(?<=HostPrivateIPv4Address": ")[^"]+')
+
+if [ -n "$HOST_PRIVATE_IP" ]; then
+    echo "Host Private IP: $HOST_PRIVATE_IP"
+
+    # Define the input template file and output file
+    TEMPLATE_FILE="filebeat.template.yml"
+    OUTPUT_FILE="filebeat.yml"
+
+    mkdir -p /opt/filebeat-configs/${HOST_PRIVATE_IP}
+    cp /opt/filebeat-configs/$TEMPLATE_FILE /opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE
+
+    # Replace the placeholder with the actual node name and write to the output file
+    sed -i 's/${HOST_PRIVATE_IP}/'$HOST_PRIVATE_IP'/g' /opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE
+    sed -i 's/${LOGSTASH_HOST}/'$LOGSTASH_HOST'/g' /opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE
+    sed -i 's/${LOGSTASH_PORT}/'$LOGSTASH_PORT'/g' /opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE
+
+    echo "Filebeat configuration file created: $OUTPUT_FILE"
+
+    # Create symlinks
+    rm -rf /usr/share/filebeat/elk
+    ln -sf "/opt/logs/${HOST_PRIVATE_IP}/elk" /usr/share/filebeat/elk
+    ln -sf "/opt/filebeat-configs/${HOST_PRIVATE_IP}/filebeat.yml" /usr/share/filebeat/filebeat.yml
+else
+    echo "HostPrivateIPv4Address not found"
+fi
+
+# Execute the original Filebeat entry point with passed arguments
+exec /usr/local/bin/docker-entrypoint "$@"
--- a/entgra-filebeat/files/filebeat/filebeat.template.yml
+++ b/entgra-filebeat/files/filebeat/filebeat.template.yml
@ -0,0 +1,35 @@
+# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
+#
+# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+filebeat.inputs:
+  - type: log
+    enabled: true
+    tags: ["SwitchGearLogs"]
+    paths:
+      - /usr/share/filebeat/elk/switchgear.log
+    exclude_files: [".gz$"]
+    exclude_lines: ["^.*TRACE.*$"]
+
+output.logstash:
+  hosts: ["${LOGSTASH_HOST}:${LOGSTASH_PORT}"]
+
+processors:
+  - add_docker_metadata:
+      host: "unix:///host_docker/docker.sock"
+  - add_fields:
+      target: ""
+      fields:
+        node.name: "${HOST_PRIVATE_IP}" #custom node name
--- a/entgra-kibana/Dockerfile
+++ b/entgra-kibana/Dockerfile
@ -0,0 +1,19 @@
+# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
+#
+# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM docker.elastic.co/kibana/kibana:8.2.0
+
+COPY kibana.yml /usr/share/kibana/kibana.yml
--- a/entgra-kibana/README.md
+++ b/entgra-kibana/README.md
@ -0,0 +1,11 @@
+## Instructions
+
+1. Build the Docker image
+```bash
+docker build -t registry.entgra.io/entgra-kibana:8.2.0-entgra-v2 -f Dockerfile .
+```
+
+2. docker-compose up -d
+3. docker exec -it $(docker ps -aq -n 1) /bin/bash
+4. docker-compose down
+
--- a/entgra-kibana/kibana.yml
+++ b/entgra-kibana/kibana.yml
@ -0,0 +1,31 @@
+# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
+#
+# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+security.showInsecureClusterWarning: false
+monitoring.ui.container.elasticsearch.enabled: true
+logging.quiet: true
+logging.verbose: false
+
+logging:
+  appenders:
+    rolling-file:
+      type: rolling-file
+      fileName: /var/logs/kibana/kibana.log
+      policy:
+        type: size-limit
+        size: 10mb
+      layout:
+        type: pattern
--- a/entgra-logstash/Dockerfile
+++ b/entgra-logstash/Dockerfile
@ -0,0 +1,19 @@
+# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
+#
+# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM docker.elastic.co/logstash/logstash:8.2.0
+
+COPY logstash.conf /usr/share/logstash/pipeline/logstash.conf
--- a/entgra-logstash/README.md
+++ b/entgra-logstash/README.md
@ -0,0 +1,11 @@
+## Instructions
+
+1. Build the Docker image
+```bash
+docker build -t registry.entgra.io/entgra-logstash:8.2.0-entgra-v2 -f Dockerfile .
+```
+
+2. docker-compose up -d
+3. docker exec -it $(docker ps -aq -n 1) /bin/bash
+4. docker-compose down
+
--- a/entgra-logstash/logstash.conf
+++ b/entgra-logstash/logstash.conf
@ -0,0 +1,39 @@
+input {
+        beats {
+                port => 5044
+        }
+}
+filter {
+
+        grok {
+             match => {
+                  "message" => "\[%{GREEDYDATA:LogLevel}]\ \[%{TIMESTAMP_ISO8601:logdate}\] \[%{DATA:EventCategory}\] \[%{DATA:InitiatedBy}\] \[%{DATA:SgId}\] \[%{DATA:CbSerial}\] \[%{DATA:OcSerial}\] \[%{DATA:IpAddress}\] \[%{DATA:State}\] - %{GREEDYDATA:Description}"
+             }
+        }
+
+        if [log][file][path] =~ "/var/log/auth.log" {
+                drop { }
+        }
+        if [fileset][name] =~ "syslog" {
+                drop { }
+        }
+        date {
+                match => [ "logdate", "MMM D, YYYY @ HH:mm:ss.SSS", "ISO8601"]
+                target => "@timestamp"
+        }
+}
+output {
+
+        if "SwitchGearLogs" in [tags] {
+                elasticsearch {
+                        hosts => ["elasticsearch:9200"]
+                        index => "switchgearlogs-%{+YYYY.MM.dd}"
+                }
+        }
+
+#       elasticsearch{
+#               hosts => ["elasticsearch:9200"]
+#               index => "lecologs-%{+YYYY.MM.dd}"
+#       }
+
+}
--- a/entrypoint.sh
+++ b/entrypoint.sh
@ -1,36 +0,0 @@
-#!/bin/bash
-set -e
-
-# Path to the ECS metadata file
-METADATA_FILE=$(cat "$ECS_CONTAINER_METADATA_FILE")
-
-if [ -f "$METADATA_FILE" ]; then
-    # Extract HostPrivateIPv4Address from the metadata file
-    HOST_PRIVATE_IP=$(jq -r '.HostPrivateIPv4Address' < "$METADATA_FILE")
-    if [ "$HOST_PRIVATE_IP" != "null" ]; then
-        echo "Host Private IP: $HOST_PRIVATE_IP"
-        export HOST_PRIVATE_IP
-
-        # Define the input template file and output file
-        TEMPLATE_FILE="filebeat.template.yml"
-        OUTPUT_FILE="filebeat.yml"
-
-        mkdir -p /opt/filebeat-configs/${HOST_PRIVATE_IP}
-
-        # Replace the placeholder with the actual node name and write to the output file
-        sed "s/{{HOST_PRIVATE_IP}}/${HOST_PRIVATE_IP}/g" "/opt/filebeat-configs/$TEMPLATE_FILE" > "/opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE"
-        echo "Filebeat configuration file created: $OUTPUT_FILE"
-
-        # Create symlinks
-        rm -rf /usr/share/filebeat/elk
-        ln -sf "/opt/logs/${HOST_PRIVATE_IP}/elk" /usr/share/filebeat/elk
-        ln -sf "/opt/filebeat-configs/${HOST_PRIVATE_IP}/filebeat.yml" /usr/share/filebeat/filebeat.yml
-    else
-        echo "HostPrivateIPv4Address not found in metadata file"
-    fi
-else
-    echo "Metadata file does not exist"
-fi
-
-# Run the original filebeat entrypoint
-exec /usr/bin/tini -- /usr/local/bin/docker-entrypoint "$@"