From 69e411f70a889a1efecb1efafd3f84b615b7867d Mon Sep 17 00:00:00 2001 From: pasindu Date: Thu, 11 Jul 2024 06:22:52 +0530 Subject: [PATCH] Add elk stack image build instructions --- README.md | 11 +----- entgra-elasticsearch/Dockerfile | 5 +++ entgra-elasticsearch/README.md | 12 ++++++ entgra-elasticsearch/docker-compose.yml | 26 +++++++++++++ entgra-elasticsearch/elasticsearch.yml | 8 ++++ Dockerfile => entgra-filebeat/Dockerfile | 0 entgra-filebeat/README.md | 12 ++++++ .../docker-compose.yml | 2 +- .../entrypoint.sh | 0 .../files/filebeat}/filebeat.template.yml | 0 entgra-kibana/Dockerfile | 3 ++ entgra-kibana/README.md | 12 ++++++ entgra-kibana/kibana.yml | 15 +++++++ entgra-logstash/Dockerfile | 3 ++ entgra-logstash/README.md | 12 ++++++ entgra-logstash/logstash.conf | 39 +++++++++++++++++++ 16 files changed, 149 insertions(+), 11 deletions(-) create mode 100644 entgra-elasticsearch/Dockerfile create mode 100644 entgra-elasticsearch/README.md create mode 100644 entgra-elasticsearch/docker-compose.yml create mode 100644 entgra-elasticsearch/elasticsearch.yml rename Dockerfile => entgra-filebeat/Dockerfile (100%) create mode 100644 entgra-filebeat/README.md rename docker-compose.yml => entgra-filebeat/docker-compose.yml (79%) rename entrypoint.sh => entgra-filebeat/entrypoint.sh (100%) rename {files/filebeat/opt/filebeat-configs => entgra-filebeat/files/filebeat}/filebeat.template.yml (100%) create mode 100644 entgra-kibana/Dockerfile create mode 100644 entgra-kibana/README.md create mode 100644 entgra-kibana/kibana.yml create mode 100644 entgra-logstash/Dockerfile create mode 100644 entgra-logstash/README.md create mode 100644 entgra-logstash/logstash.conf diff --git a/README.md b/README.md index bb953b3..121a767 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,3 @@ ## Instructions -1. Pull the filebeat image -```bash -docker pull docker.elastic.co/beats/filebeat:8.2.0 -``` -Note: Change the filebeat image name or tag in Dockerfile related to the pulled version. - -3. Build the Docker image -```bash -docker build -t filebeat:8.2.0-entgra-v1 . -``` +Note: cd to the exact directory and build the images. diff --git a/entgra-elasticsearch/Dockerfile b/entgra-elasticsearch/Dockerfile new file mode 100644 index 0000000..0be8d4f --- /dev/null +++ b/entgra-elasticsearch/Dockerfile @@ -0,0 +1,5 @@ +FROM docker.elastic.co/elasticsearch/elasticsearch:8.2.0 + +RUN rm -Rf /usr/share/elasticsearch/config/elasticsearch.yml + +COPY elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml \ No newline at end of file diff --git a/entgra-elasticsearch/README.md b/entgra-elasticsearch/README.md new file mode 100644 index 0000000..c219b13 --- /dev/null +++ b/entgra-elasticsearch/README.md @@ -0,0 +1,12 @@ +## Instructions + +1. Pull the elasticsearch image +```bash +docker pull docker.elastic.co/elasticsearch/elasticsearch:8.2.0 +``` +Note: Change the required image version name or tag in the Dockerfile. + +3. Build the Docker image +```bash +docker build -t elasticsearch:8.2.0-entgra-v1 . +``` diff --git a/entgra-elasticsearch/docker-compose.yml b/entgra-elasticsearch/docker-compose.yml new file mode 100644 index 0000000..923873d --- /dev/null +++ b/entgra-elasticsearch/docker-compose.yml @@ -0,0 +1,26 @@ +version: "3.8" + +services: + elasticsearch: + image: elasticsearch:8.2.0-entgra-v2 + container_name: elasticsearch + restart: unless-stopped + environment: + - TZ=Asia/Colombo + - discovery.type=single-node + - xpack.security.enabled=false + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + volumes: + - ./files/data:/usr/share/elasticsearch/data + - ./files/backup:/usr/share/elasticsearch/backup +# - ./files/configs:/usr/share/elasticsearch/config + - ./files/logs:/usr/share/elasticsearch/logs + ports: + - "9200:9200" + healthcheck: + test: curl --fail http://localhost:9200/ || exit 1 + interval: 60s + retries: 10 + start_period: 30s + timeout: 10s \ No newline at end of file diff --git a/entgra-elasticsearch/elasticsearch.yml b/entgra-elasticsearch/elasticsearch.yml new file mode 100644 index 0000000..c660745 --- /dev/null +++ b/entgra-elasticsearch/elasticsearch.yml @@ -0,0 +1,8 @@ +cluster.name: "docker-cluster" +network.host: 0.0.0.0 + +# minimum_master_nodes need to be explicitly set when bound on a public IP +# # set to 1 to allow single node clusters +# # Details: https://github.com/elastic/elasticsearch/pull/17288 +# discovery.zen.minimum_master_nodes: 1 +path.repo: ["/usr/share/elasticsearch/backup"] diff --git a/Dockerfile b/entgra-filebeat/Dockerfile similarity index 100% rename from Dockerfile rename to entgra-filebeat/Dockerfile diff --git a/entgra-filebeat/README.md b/entgra-filebeat/README.md new file mode 100644 index 0000000..9b5c4c6 --- /dev/null +++ b/entgra-filebeat/README.md @@ -0,0 +1,12 @@ +## Instructions + +1. Pull the filebeat image +```bash +docker pull docker.elastic.co/beats/filebeat:8.2.0 +``` +Note: Change the required image version name or tag in the Dockerfile. + +3. Build the Docker image +```bash +docker build -t filebeat:8.2.0-entgra-v1 . +``` diff --git a/docker-compose.yml b/entgra-filebeat/docker-compose.yml similarity index 79% rename from docker-compose.yml rename to entgra-filebeat/docker-compose.yml index 50804ec..6a60339 100644 --- a/docker-compose.yml +++ b/entgra-filebeat/docker-compose.yml @@ -9,7 +9,7 @@ services: - "host.docker.internal:host-gateway" volumes: - ./files/iots/logs/:/opt/logs - - ./files/filebeat/opt/filebeat-configs:/opt/filebeat-configs + - ./files/filebeat/:/opt/filebeat-configs command: ["--strict.perms=false"] environment: - TZ=Asia/Colombo diff --git a/entrypoint.sh b/entgra-filebeat/entrypoint.sh similarity index 100% rename from entrypoint.sh rename to entgra-filebeat/entrypoint.sh diff --git a/files/filebeat/opt/filebeat-configs/filebeat.template.yml b/entgra-filebeat/files/filebeat/filebeat.template.yml similarity index 100% rename from files/filebeat/opt/filebeat-configs/filebeat.template.yml rename to entgra-filebeat/files/filebeat/filebeat.template.yml diff --git a/entgra-kibana/Dockerfile b/entgra-kibana/Dockerfile new file mode 100644 index 0000000..ccb1e4c --- /dev/null +++ b/entgra-kibana/Dockerfile @@ -0,0 +1,3 @@ +FROM docker.elastic.co/kibana/kibana:8.2.0 + +COPY kibana.yml /usr/share/kibana/kibana.yml \ No newline at end of file diff --git a/entgra-kibana/README.md b/entgra-kibana/README.md new file mode 100644 index 0000000..e9ac868 --- /dev/null +++ b/entgra-kibana/README.md @@ -0,0 +1,12 @@ +## Instructions + +1. Pull the kibana image +```bash +docker pull docker.elastic.co/kibana/kibana:8.2.0 +``` +Note: Change the required image version name or tag in the Dockerfile. + +3. Build the Docker image +```bash +docker build -t kibana:8.2.0-entgra-v1 . +``` diff --git a/entgra-kibana/kibana.yml b/entgra-kibana/kibana.yml new file mode 100644 index 0000000..2db5738 --- /dev/null +++ b/entgra-kibana/kibana.yml @@ -0,0 +1,15 @@ +security.showInsecureClusterWarning: false +monitoring.ui.container.elasticsearch.enabled: true +logging.quiet: true +logging.verbose: false + +logging: + appenders: + rolling-file: + type: rolling-file + fileName: /var/logs/kibana/kibana.log + policy: + type: size-limit + size: 10mb + layout: + type: pattern diff --git a/entgra-logstash/Dockerfile b/entgra-logstash/Dockerfile new file mode 100644 index 0000000..ba7a656 --- /dev/null +++ b/entgra-logstash/Dockerfile @@ -0,0 +1,3 @@ +FROM docker.elastic.co/logstash/logstash:8.2.0 + +COPY logstash.conf /usr/share/logstash/pipeline/logstash.conf \ No newline at end of file diff --git a/entgra-logstash/README.md b/entgra-logstash/README.md new file mode 100644 index 0000000..7e7ef5d --- /dev/null +++ b/entgra-logstash/README.md @@ -0,0 +1,12 @@ +## Instructions + +1. Pull the logstash image +```bash +docker pull docker.elastic.co/logstash/logstash:8.2.0 +``` +Note: Change the required image version name or tag in the Dockerfile. + +3. Build the Docker image +```bash +docker build -t logstash:8.2.0-entgra-v1 . +``` diff --git a/entgra-logstash/logstash.conf b/entgra-logstash/logstash.conf new file mode 100644 index 0000000..05535ff --- /dev/null +++ b/entgra-logstash/logstash.conf @@ -0,0 +1,39 @@ +input { + beats { + port => 5044 + } +} +filter { + + grok { + match => { + "message" => "\[%{GREEDYDATA:LogLevel}]\ \[%{TIMESTAMP_ISO8601:logdate}\] \[%{DATA:EventCategory}\] \[%{DATA:InitiatedBy}\] \[%{DATA:SgId}\] \[%{DATA:CbSerial}\] \[%{DATA:OcSerial}\] \[%{DATA:IpAddress}\] \[%{DATA:State}\] - %{GREEDYDATA:Description}" + } + } + + if [log][file][path] =~ "/var/log/auth.log" { + drop { } + } + if [fileset][name] =~ "syslog" { + drop { } + } + date { + match => [ "logdate", "MMM D, YYYY @ HH:mm:ss.SSS", "ISO8601"] + target => "@timestamp" + } +} +output { + + if "SwitchGearLogs" in [tags] { + elasticsearch { + hosts => ["elasticsearch:9200"] + index => "switchgearlogs-%{+YYYY.MM.dd}" + } + } + +# elasticsearch{ +# hosts => ["elasticsearch:9200"] +# index => "lecologs-%{+YYYY.MM.dd}" +# } + +} \ No newline at end of file