Add elk stack image build instructions #3
Merged
amalka.subasinghe
merged 4 commits from pasindu/entgra-filebeat:RemoveJQUsage into master 4 months ago
@ -1,12 +0,0 @@
|
||||
FROM docker.elastic.co/beats/filebeat:8.2.0
|
||||
|
||||
# Switch to root user
|
||||
USER root
|
||||
|
||||
#install jq to the container
|
||||
RUN apt-get update && apt-get install -y jq
|
||||
|
||||
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||
@ -1,12 +1,3 @@
|
||||
## Instructions
|
||||
|
||||
1. Pull the filebeat image
|
||||
```bash
|
||||
docker pull docker.elastic.co/beats/filebeat:8.2.0
|
||||
```
|
||||
Note: Change the filebeat image name or tag in Dockerfile related to the pulled version.
|
||||
|
||||
3. Build the Docker image
|
||||
```bash
|
||||
docker build -t filebeat:8.2.0-entgra-v1 .
|
||||
```
|
||||
Note: cd to the exact directory and build the images.
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
|
||||
#
|
||||
# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
|
||||
# Version 2.0 (the "License"); you may not use this file except
|
||||
# in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
FROM docker.elastic.co/elasticsearch/elasticsearch:8.2.0
|
||||
|
||||
RUN rm -Rf /usr/share/elasticsearch/config/elasticsearch.yml
|
||||
|
||||
COPY elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml
|
||||
@ -0,0 +1,10 @@
|
||||
## Instructions
|
||||
|
||||
1. Build the Docker image
|
||||
```bash
|
||||
docker build -t registry.entgra.io/entgra-elasticsearch:8.2.0-entgra-v2 -f Dockerfile .
|
||||
```
|
||||
|
||||
2. docker-compose up -d
|
||||
3. docker exec -it $(docker ps -aq -n 1) /bin/bash
|
||||
4. docker-compose down
|
||||
@ -0,0 +1,42 @@
|
||||
# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
|
||||
#
|
||||
# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
|
||||
# Version 2.0 (the "License"); you may not use this file except
|
||||
# in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
version: "3.8"
|
||||
|
||||
services:
|
||||
elasticsearch:
|
||||
image: elasticsearch:8.2.0-entgra-v2
|
||||
container_name: elasticsearch
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- TZ=Asia/Colombo
|
||||
- discovery.type=single-node
|
||||
- xpack.security.enabled=false
|
||||
- bootstrap.memory_lock=true
|
||||
- "ES_JAVA_OPTS=-Xms1g -Xmx1g"
|
||||
volumes:
|
||||
- ./files/data:/usr/share/elasticsearch/data
|
||||
- ./files/backup:/usr/share/elasticsearch/backup
|
||||
# - ./files/configs:/usr/share/elasticsearch/config
|
||||
- ./files/logs:/usr/share/elasticsearch/logs
|
||||
ports:
|
||||
- "9200:9200"
|
||||
healthcheck:
|
||||
test: curl --fail http://localhost:9200/ || exit 1
|
||||
interval: 60s
|
||||
retries: 10
|
||||
start_period: 30s
|
||||
timeout: 10s
|
||||
@ -0,0 +1,24 @@
|
||||
# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
|
||||
#
|
||||
# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
|
||||
# Version 2.0 (the "License"); you may not use this file except
|
||||
# in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
cluster.name: "docker-cluster"
|
||||
network.host: 0.0.0.0
|
||||
|
||||
# minimum_master_nodes need to be explicitly set when bound on a public IP
|
||||
# # set to 1 to allow single node clusters
|
||||
# # Details: https://github.com/elastic/elasticsearch/pull/17288
|
||||
# discovery.zen.minimum_master_nodes: 1
|
||||
path.repo: ["/usr/share/elasticsearch/backup"]
|
||||
@ -0,0 +1,29 @@
|
||||
# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
|
||||
#
|
||||
# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
|
||||
# Version 2.0 (the "License"); you may not use this file except
|
||||
# in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
FROM docker.elastic.co/beats/filebeat:8.2.0
|
||||
|
||||
# Switch to root user
|
||||
USER root
|
||||
|
||||
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/entrypoint.sh
|
||||
|
||||
# Set the entry point to your custom script using tini
|
||||
ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/entrypoint.sh"]
|
||||
|
||||
# Set the CMD to pass the environment argument to Filebeat
|
||||
CMD ["-environment", "container"]
|
||||
@ -0,0 +1,11 @@
|
||||
## Instructions
|
||||
|
||||
1. Build the Docker image
|
||||
```bash
|
||||
docker build -t registry.entgra.io/entgra-filebeat:8.2.0-entgra-v2 -f Dockerfile .
|
||||
```
|
||||
|
||||
2. docker-compose up -d
|
||||
3. docker exec -it $(docker ps -aq -n 1) /bin/bash
|
||||
4. docker-compose down
|
||||
|
||||
@ -0,0 +1,33 @@
|
||||
# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
|
||||
#
|
||||
# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
|
||||
# Version 2.0 (the "License"); you may not use this file except
|
||||
# in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
version: "3.8"
|
||||
|
||||
services:
|
||||
entgra-filebeat:
|
||||
image: filebeat:8.2.0-entgra-v2
|
||||
container_name: entgra-filebeat
|
||||
restart: unless-stopped
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
volumes:
|
||||
- ./files/iots/logs/:/opt/logs
|
||||
- ./files/filebeat/:/opt/filebeat-configs
|
||||
command: ["--strict.perms=false"]
|
||||
environment:
|
||||
- TZ=Asia/Colombo
|
||||
- LOGSTASH_HOST=logstash
|
||||
- LOGSTASH_PORT=5044
|
||||
@ -0,0 +1,53 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
|
||||
#
|
||||
# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
|
||||
# Version 2.0 (the "License"); you may not use this file except
|
||||
# in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Check if ECS_CONTAINER_METADATA_FILE is set
|
||||
if [ -z "$ECS_CONTAINER_METADATA_FILE" ]; then
|
||||
echo "ECS_CONTAINER_METADATA_FILE is not set or the server is running on local deployment environment"
|
||||
fi
|
||||
|
||||
# Extract HostPrivateIPv4Address from the metadata file
|
||||
HOST_PRIVATE_IP=$(cat $ECS_CONTAINER_METADATA_FILE | grep -oP '(?<=HostPrivateIPv4Address": ")[^"]+')
|
||||
|
||||
if [ -n "$HOST_PRIVATE_IP" ]; then
|
||||
echo "Host Private IP: $HOST_PRIVATE_IP"
|
||||
|
||||
# Define the input template file and output file
|
||||
TEMPLATE_FILE="filebeat.template.yml"
|
||||
OUTPUT_FILE="filebeat.yml"
|
||||
|
||||
mkdir -p /opt/filebeat-configs/${HOST_PRIVATE_IP}
|
||||
cp /opt/filebeat-configs/$TEMPLATE_FILE /opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE
|
||||
|
||||
# Replace the placeholder with the actual node name and write to the output file
|
||||
sed -i 's/${HOST_PRIVATE_IP}/'$HOST_PRIVATE_IP'/g' /opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE
|
||||
sed -i 's/${LOGSTASH_HOST}/'$LOGSTASH_HOST'/g' /opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE
|
||||
sed -i 's/${LOGSTASH_PORT}/'$LOGSTASH_PORT'/g' /opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE
|
||||
|
||||
echo "Filebeat configuration file created: $OUTPUT_FILE"
|
||||
|
||||
# Create symlinks
|
||||
rm -rf /usr/share/filebeat/elk
|
||||
ln -sf "/opt/logs/${HOST_PRIVATE_IP}/elk" /usr/share/filebeat/elk
|
||||
ln -sf "/opt/filebeat-configs/${HOST_PRIVATE_IP}/filebeat.yml" /usr/share/filebeat/filebeat.yml
|
||||
else
|
||||
echo "HostPrivateIPv4Address not found"
|
||||
fi
|
||||
|
||||
# Execute the original Filebeat entry point with passed arguments
|
||||
exec /usr/local/bin/docker-entrypoint "$@"
|
||||
@ -0,0 +1,35 @@
|
||||
# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
|
||||
#
|
||||
# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
|
||||
# Version 2.0 (the "License"); you may not use this file except
|
||||
# in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
filebeat.inputs:
|
||||
- type: log
|
||||
enabled: true
|
||||
tags: ["SwitchGearLogs"]
|
||||
paths:
|
||||
- /usr/share/filebeat/elk/switchgear.log
|
||||
exclude_files: [".gz$"]
|
||||
exclude_lines: ["^.*TRACE.*$"]
|
||||
|
||||
output.logstash:
|
||||
hosts: ["${LOGSTASH_HOST}:${LOGSTASH_PORT}"]
|
||||
|
||||
processors:
|
||||
- add_docker_metadata:
|
||||
host: "unix:///host_docker/docker.sock"
|
||||
- add_fields:
|
||||
target: ""
|
||||
fields:
|
||||
node.name: "${HOST_PRIVATE_IP}" #custom node name
|
||||
@ -0,0 +1,19 @@
|
||||
# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
|
||||
#
|
||||
# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
|
||||
# Version 2.0 (the "License"); you may not use this file except
|
||||
# in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
FROM docker.elastic.co/kibana/kibana:8.2.0
|
||||
|
||||
COPY kibana.yml /usr/share/kibana/kibana.yml
|
||||
@ -0,0 +1,11 @@
|
||||
## Instructions
|
||||
|
||||
1. Build the Docker image
|
||||
```bash
|
||||
docker build -t registry.entgra.io/entgra-kibana:8.2.0-entgra-v2 -f Dockerfile .
|
||||
```
|
||||
|
||||
2. docker-compose up -d
|
||||
3. docker exec -it $(docker ps -aq -n 1) /bin/bash
|
||||
4. docker-compose down
|
||||
|
||||
@ -0,0 +1,31 @@
|
||||
# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
|
||||
#
|
||||
# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
|
||||
# Version 2.0 (the "License"); you may not use this file except
|
||||
# in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
security.showInsecureClusterWarning: false
|
||||
monitoring.ui.container.elasticsearch.enabled: true
|
||||
logging.quiet: true
|
||||
logging.verbose: false
|
||||
|
||||
logging:
|
||||
appenders:
|
||||
rolling-file:
|
||||
type: rolling-file
|
||||
fileName: /var/logs/kibana/kibana.log
|
||||
policy:
|
||||
type: size-limit
|
||||
size: 10mb
|
||||
layout:
|
||||
type: pattern
|
||||
@ -0,0 +1,19 @@
|
||||
# Copyright (c) 2018 - 2024, Entgra (Pvt) Ltd. (http://www.entgra.io) All Rights Reserved.
|
||||
#
|
||||
# Entgra (Pvt) Ltd. licenses this file to you under the Apache License,
|
||||
# Version 2.0 (the "License"); you may not use this file except
|
||||
# in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
FROM docker.elastic.co/logstash/logstash:8.2.0
|
||||
|
||||
COPY logstash.conf /usr/share/logstash/pipeline/logstash.conf
|
||||
@ -0,0 +1,11 @@
|
||||
## Instructions
|
||||
|
||||
1. Build the Docker image
|
||||
```bash
|
||||
docker build -t registry.entgra.io/entgra-logstash:8.2.0-entgra-v2 -f Dockerfile .
|
||||
```
|
||||
|
||||
2. docker-compose up -d
|
||||
3. docker exec -it $(docker ps -aq -n 1) /bin/bash
|
||||
4. docker-compose down
|
||||
|
||||
@ -0,0 +1,39 @@
|
||||
input {
|
||||
beats {
|
||||
port => 5044
|
||||
}
|
||||
}
|
||||
filter {
|
||||
|
||||
grok {
|
||||
match => {
|
||||
"message" => "\[%{GREEDYDATA:LogLevel}]\ \[%{TIMESTAMP_ISO8601:logdate}\] \[%{DATA:EventCategory}\] \[%{DATA:InitiatedBy}\] \[%{DATA:SgId}\] \[%{DATA:CbSerial}\] \[%{DATA:OcSerial}\] \[%{DATA:IpAddress}\] \[%{DATA:State}\] - %{GREEDYDATA:Description}"
|
||||
}
|
||||
}
|
||||
|
||||
if [log][file][path] =~ "/var/log/auth.log" {
|
||||
drop { }
|
||||
}
|
||||
if [fileset][name] =~ "syslog" {
|
||||
drop { }
|
||||
}
|
||||
date {
|
||||
match => [ "logdate", "MMM D, YYYY @ HH:mm:ss.SSS", "ISO8601"]
|
||||
target => "@timestamp"
|
||||
}
|
||||
}
|
||||
output {
|
||||
|
||||
if "SwitchGearLogs" in [tags] {
|
||||
elasticsearch {
|
||||
hosts => ["elasticsearch:9200"]
|
||||
index => "switchgearlogs-%{+YYYY.MM.dd}"
|
||||
}
|
||||
}
|
||||
|
||||
# elasticsearch{
|
||||
# hosts => ["elasticsearch:9200"]
|
||||
# index => "lecologs-%{+YYYY.MM.dd}"
|
||||
# }
|
||||
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Path to the ECS metadata file
|
||||
METADATA_FILE=$(cat "$ECS_CONTAINER_METADATA_FILE")
|
||||
|
||||
if [ -f "$METADATA_FILE" ]; then
|
||||
# Extract HostPrivateIPv4Address from the metadata file
|
||||
HOST_PRIVATE_IP=$(jq -r '.HostPrivateIPv4Address' < "$METADATA_FILE")
|
||||
if [ "$HOST_PRIVATE_IP" != "null" ]; then
|
||||
echo "Host Private IP: $HOST_PRIVATE_IP"
|
||||
export HOST_PRIVATE_IP
|
||||
|
||||
# Define the input template file and output file
|
||||
TEMPLATE_FILE="filebeat.template.yml"
|
||||
OUTPUT_FILE="filebeat.yml"
|
||||
|
||||
mkdir -p /opt/filebeat-configs/${HOST_PRIVATE_IP}
|
||||
|
||||
# Replace the placeholder with the actual node name and write to the output file
|
||||
sed "s/{{HOST_PRIVATE_IP}}/${HOST_PRIVATE_IP}/g" "/opt/filebeat-configs/$TEMPLATE_FILE" > "/opt/filebeat-configs/${HOST_PRIVATE_IP}/$OUTPUT_FILE"
|
||||
echo "Filebeat configuration file created: $OUTPUT_FILE"
|
||||
|
||||
# Create symlinks
|
||||
rm -rf /usr/share/filebeat/elk
|
||||
ln -sf "/opt/logs/${HOST_PRIVATE_IP}/elk" /usr/share/filebeat/elk
|
||||
ln -sf "/opt/filebeat-configs/${HOST_PRIVATE_IP}/filebeat.yml" /usr/share/filebeat/filebeat.yml
|
||||
else
|
||||
echo "HostPrivateIPv4Address not found in metadata file"
|
||||
fi
|
||||
else
|
||||
echo "Metadata file does not exist"
|
||||
fi
|
||||
|
||||
# Run the original filebeat entrypoint
|
||||
exec /usr/bin/tini -- /usr/local/bin/docker-entrypoint "$@"
|
||||
Loading…
Reference in new issue