diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index ee195c6..0000000 --- a/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM docker.elastic.co/beats/filebeat:8.2.0 - -# Switch to root user -USER root - -#install jq to the container -RUN apt-get update && apt-get install -y jq - -COPY entrypoint.sh /usr/local/bin/entrypoint.sh -RUN chmod +x /usr/local/bin/entrypoint.sh - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/README.md b/README.md index bb953b3..121a767 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,3 @@ ## Instructions -1. Pull the filebeat image -```bash -docker pull docker.elastic.co/beats/filebeat:8.2.0 -``` -Note: Change the filebeat image name or tag in Dockerfile related to the pulled version. - -3. Build the Docker image -```bash -docker build -t filebeat:8.2.0-entgra-v1 . -``` +Note: cd to the exact directory and build the images. diff --git a/entgra-elasticsearch/Dockerfile b/entgra-elasticsearch/Dockerfile new file mode 100644 index 0000000..e6ce51e --- /dev/null +++ b/entgra-elasticsearch/Dockerfile @@ -0,0 +1,21 @@ +# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved. +# +# Entgra (Pvt) Ltd. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM docker.elastic.co/elasticsearch/elasticsearch:8.2.0 + +RUN rm -Rf /usr/share/elasticsearch/config/elasticsearch.yml + +COPY elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml diff --git a/entgra-elasticsearch/README.md b/entgra-elasticsearch/README.md new file mode 100644 index 0000000..9250ac3 --- /dev/null +++ b/entgra-elasticsearch/README.md @@ -0,0 +1,10 @@ +## Instructions + +1. Build the Docker image +```bash +docker build -t registry.entgra.io/entgra-elasticsearch:8.2.0-entgra-v2 -f Dockerfile . +``` + +2. docker-compose up -d +3. docker exec -it $(docker ps -aq -n 1) /bin/bash +4. docker-compose down diff --git a/entgra-elasticsearch/docker-compose.yml b/entgra-elasticsearch/docker-compose.yml new file mode 100644 index 0000000..797a575 --- /dev/null +++ b/entgra-elasticsearch/docker-compose.yml @@ -0,0 +1,42 @@ +# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved. +# +# Entgra (Pvt) Ltd. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: "3.8" + +services: + elasticsearch: + image: elasticsearch:8.2.0-entgra-v2 + container_name: elasticsearch + restart: unless-stopped + environment: + - TZ=Asia/Colombo + - discovery.type=single-node + - xpack.security.enabled=false + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + volumes: + - ./files/data:/usr/share/elasticsearch/data + - ./files/backup:/usr/share/elasticsearch/backup +# - ./files/configs:/usr/share/elasticsearch/config + - ./files/logs:/usr/share/elasticsearch/logs + ports: + - "9200:9200" + healthcheck: + test: curl --fail http://localhost:9200/ || exit 1 + interval: 60s + retries: 10 + start_period: 30s + timeout: 10s diff --git a/entgra-elasticsearch/elasticsearch.yml b/entgra-elasticsearch/elasticsearch.yml new file mode 100644 index 0000000..a3df086 --- /dev/null +++ b/entgra-elasticsearch/elasticsearch.yml @@ -0,0 +1,24 @@ +# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved. +# +# Entgra (Pvt) Ltd. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cluster.name: "docker-cluster" +network.host: 0.0.0.0 + +# minimum_master_nodes need to be explicitly set when bound on a public IP +# # set to 1 to allow single node clusters +# # Details: https://github.com/elastic/elasticsearch/pull/17288 +# discovery.zen.minimum_master_nodes: 1 +path.repo: ["/usr/share/elasticsearch/backup"] diff --git a/entgra-filebeat/Dockerfile b/entgra-filebeat/Dockerfile new file mode 100644 index 0000000..20ec312 --- /dev/null +++ b/entgra-filebeat/Dockerfile @@ -0,0 +1,29 @@ +# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved. +# +# Entgra (Pvt) Ltd. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM docker.elastic.co/beats/filebeat:8.2.0 + +# Switch to root user +USER root + +COPY entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +# Set the entry point to your custom script using tini +ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/entrypoint.sh"] + +# Set the CMD to pass the environment argument to Filebeat +CMD ["-environment", "container"] diff --git a/entgra-filebeat/README.md b/entgra-filebeat/README.md new file mode 100644 index 0000000..39d9223 --- /dev/null +++ b/entgra-filebeat/README.md @@ -0,0 +1,11 @@ +## Instructions + +1. Build the Docker image +```bash +docker build -t registry.entgra.io/entgra-filebeat:8.2.0-entgra-v2 -f Dockerfile . +``` + +2. docker-compose up -d +3. docker exec -it $(docker ps -aq -n 1) /bin/bash +4. docker-compose down + diff --git a/entgra-filebeat/docker-compose.yml b/entgra-filebeat/docker-compose.yml new file mode 100644 index 0000000..189af13 --- /dev/null +++ b/entgra-filebeat/docker-compose.yml @@ -0,0 +1,33 @@ +# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved. +# +# Entgra (Pvt) Ltd. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: "3.8" + +services: + entgra-filebeat: + image: filebeat:8.2.0-entgra-v2 + container_name: entgra-filebeat + restart: unless-stopped + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - ./files/iots/logs/:/opt/logs + - ./files/filebeat/:/opt/filebeat-configs + command: ["--strict.perms=false"] + environment: + - TZ=Asia/Colombo + - LOGSTASH_HOST=logstash + - LOGSTASH_PORT=5044 diff --git a/entgra-filebeat/entrypoint.sh b/entgra-filebeat/entrypoint.sh new file mode 100644 index 0000000..80126c3 --- /dev/null +++ b/entgra-filebeat/entrypoint.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved. +# +# Entgra (Pvt) Ltd. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Check if ECS_CONTAINER_METADATA_FILE is set +if [ -z "$ECS_CONTAINER_METADATA_FILE" ]; then + echo "ECS_CONTAINER_METADATA_FILE is not set or the server is running on local deployment environment" +fi + +# Extract HostPrivateIPv4Address from the metadata file +HOST_PRIVATE_IP=$(cat $ECS_CONTAINER_METADATA_FILE | grep -oP '(?<=HostPrivateIPv4Address": ")[^"]+') + +if [ -n "$HOST_PRIVATE_IP" ]; then + echo "Host Private IP: $HOST_PRIVATE_IP" + + # Define the input template file and output file + TEMPLATE_FILE="filebeat.template.yml" + OUTPUT_FILE="filebeat.yml" + + mkdir -p /opt/filebeat-configs/${HOST_PRIVATE_IP} + cp /opt/filebeat-configs/$TEMPLATE_FILE /opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE + + # Replace the placeholder with the actual node name and write to the output file + sed -i 's/${HOST_PRIVATE_IP}/'$HOST_PRIVATE_IP'/g' /opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE + sed -i 's/${LOGSTASH_HOST}/'$LOGSTASH_HOST'/g' /opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE + sed -i 's/${LOGSTASH_PORT}/'$LOGSTASH_PORT'/g' /opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE + + echo "Filebeat configuration file created: $OUTPUT_FILE" + + # Create symlinks + rm -rf /usr/share/filebeat/elk + ln -sf "/opt/logs/${HOST_PRIVATE_IP}/elk" /usr/share/filebeat/elk + ln -sf "/opt/filebeat-configs/${HOST_PRIVATE_IP}/filebeat.yml" /usr/share/filebeat/filebeat.yml +else + echo "HostPrivateIPv4Address not found" +fi + +# Execute the original Filebeat entry point with passed arguments +exec /usr/local/bin/docker-entrypoint "$@" diff --git a/entgra-filebeat/files/filebeat/filebeat.template.yml b/entgra-filebeat/files/filebeat/filebeat.template.yml new file mode 100644 index 0000000..f5f2ea5 --- /dev/null +++ b/entgra-filebeat/files/filebeat/filebeat.template.yml @@ -0,0 +1,35 @@ +# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved. +# +# Entgra (Pvt) Ltd. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +filebeat.inputs: + - type: log + enabled: true + tags: ["SwitchGearLogs"] + paths: + - /usr/share/filebeat/elk/switchgear.log + exclude_files: [".gz$"] + exclude_lines: ["^.*TRACE.*$"] + +output.logstash: + hosts: ["${LOGSTASH_HOST}:${LOGSTASH_PORT}"] + +processors: + - add_docker_metadata: + host: "unix:///host_docker/docker.sock" + - add_fields: + target: "" + fields: + node.name: "${HOST_PRIVATE_IP}" #custom node name diff --git a/entgra-kibana/Dockerfile b/entgra-kibana/Dockerfile new file mode 100644 index 0000000..9140c27 --- /dev/null +++ b/entgra-kibana/Dockerfile @@ -0,0 +1,19 @@ +# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved. +# +# Entgra (Pvt) Ltd. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM docker.elastic.co/kibana/kibana:8.2.0 + +COPY kibana.yml /usr/share/kibana/kibana.yml diff --git a/entgra-kibana/README.md b/entgra-kibana/README.md new file mode 100644 index 0000000..52fba2e --- /dev/null +++ b/entgra-kibana/README.md @@ -0,0 +1,11 @@ +## Instructions + +1. Build the Docker image +```bash +docker build -t registry.entgra.io/entgra-kibana:8.2.0-entgra-v2 -f Dockerfile . +``` + +2. docker-compose up -d +3. docker exec -it $(docker ps -aq -n 1) /bin/bash +4. docker-compose down + diff --git a/entgra-kibana/kibana.yml b/entgra-kibana/kibana.yml new file mode 100644 index 0000000..20ad89b --- /dev/null +++ b/entgra-kibana/kibana.yml @@ -0,0 +1,31 @@ +# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved. +# +# Entgra (Pvt) Ltd. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +security.showInsecureClusterWarning: false +monitoring.ui.container.elasticsearch.enabled: true +logging.quiet: true +logging.verbose: false + +logging: + appenders: + rolling-file: + type: rolling-file + fileName: /var/logs/kibana/kibana.log + policy: + type: size-limit + size: 10mb + layout: + type: pattern diff --git a/entgra-logstash/Dockerfile b/entgra-logstash/Dockerfile new file mode 100644 index 0000000..728caa0 --- /dev/null +++ b/entgra-logstash/Dockerfile @@ -0,0 +1,19 @@ +# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved. +# +# Entgra (Pvt) Ltd. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM docker.elastic.co/logstash/logstash:8.2.0 + +COPY logstash.conf /usr/share/logstash/pipeline/logstash.conf diff --git a/entgra-logstash/README.md b/entgra-logstash/README.md new file mode 100644 index 0000000..1de98c1 --- /dev/null +++ b/entgra-logstash/README.md @@ -0,0 +1,11 @@ +## Instructions + +1. Build the Docker image +```bash +docker build -t registry.entgra.io/entgra-logstash:8.2.0-entgra-v2 -f Dockerfile . +``` + +2. docker-compose up -d +3. docker exec -it $(docker ps -aq -n 1) /bin/bash +4. docker-compose down + diff --git a/entgra-logstash/logstash.conf b/entgra-logstash/logstash.conf new file mode 100644 index 0000000..05535ff --- /dev/null +++ b/entgra-logstash/logstash.conf @@ -0,0 +1,39 @@ +input { + beats { + port => 5044 + } +} +filter { + + grok { + match => { + "message" => "\[%{GREEDYDATA:LogLevel}]\ \[%{TIMESTAMP_ISO8601:logdate}\] \[%{DATA:EventCategory}\] \[%{DATA:InitiatedBy}\] \[%{DATA:SgId}\] \[%{DATA:CbSerial}\] \[%{DATA:OcSerial}\] \[%{DATA:IpAddress}\] \[%{DATA:State}\] - %{GREEDYDATA:Description}" + } + } + + if [log][file][path] =~ "/var/log/auth.log" { + drop { } + } + if [fileset][name] =~ "syslog" { + drop { } + } + date { + match => [ "logdate", "MMM D, YYYY @ HH:mm:ss.SSS", "ISO8601"] + target => "@timestamp" + } +} +output { + + if "SwitchGearLogs" in [tags] { + elasticsearch { + hosts => ["elasticsearch:9200"] + index => "switchgearlogs-%{+YYYY.MM.dd}" + } + } + +# elasticsearch{ +# hosts => ["elasticsearch:9200"] +# index => "lecologs-%{+YYYY.MM.dd}" +# } + +} \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100644 index a13a7ae..0000000 --- a/entrypoint.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -set -e - -# Path to the ECS metadata file -METADATA_FILE=$(cat "$ECS_CONTAINER_METADATA_FILE") - -if [ -f "$METADATA_FILE" ]; then - # Extract HostPrivateIPv4Address from the metadata file - HOST_PRIVATE_IP=$(jq -r '.HostPrivateIPv4Address' < "$METADATA_FILE") - if [ "$HOST_PRIVATE_IP" != "null" ]; then - echo "Host Private IP: $HOST_PRIVATE_IP" - export HOST_PRIVATE_IP - - # Define the input template file and output file - TEMPLATE_FILE="filebeat.template.yml" - OUTPUT_FILE="filebeat.yml" - - mkdir -p /opt/filebeat-configs/${HOST_PRIVATE_IP} - - # Replace the placeholder with the actual node name and write to the output file - sed "s/{{HOST_PRIVATE_IP}}/${HOST_PRIVATE_IP}/g" "/opt/filebeat-configs/$TEMPLATE_FILE" > "/opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE" - echo "Filebeat configuration file created: $OUTPUT_FILE" - - # Create symlinks - rm -rf /usr/share/filebeat/elk - ln -sf "/opt/logs/${HOST_PRIVATE_IP}/elk" /usr/share/filebeat/elk - ln -sf "/opt/filebeat-configs/${HOST_PRIVATE_IP}/filebeat.yml" /usr/share/filebeat/filebeat.yml - else - echo "HostPrivateIPv4Address not found in metadata file" - fi -else - echo "Metadata file does not exist" -fi - -# Run the original filebeat entrypoint -exec /usr/bin/tini -- /usr/local/bin/docker-entrypoint "$@"