docker-elk
# docker-elk
docker-elk github (opens new window)
# 加速链接
git clone https://github.91chi.fun/https://github.com/brook-w/docker-elk.git
# 文件目录结构
.
│ .env
│ .gitattributes
│ docker-compose.yml
│ LICENSE
│ README.md
├─elasticsearch
│ │ .dockerignore
│ │ Dockerfile
│ │
│ └─config
│ elasticsearch.yml
│
├─kibana
│ │ .dockerignore
│ │ Dockerfile
│ │
│ └─config
│ kibana.yml
│
├─logstash
│ │ .dockerignore
│ │ Dockerfile
│ │
│ ├─config
│ │ logstash.yml
│ │
│ └─pipeline
│ logstash.conf
│
└─setup
│ .dockerignore
│ .gitignore
│ Dockerfile
│ entrypoint.sh
│ helpers.sh
│
└─roles
logstash_writer.json
# 根目录
# docker-compose.yml
version: '3.7'
services:
# The 'setup' service runs a one-off script which initializes the
# 'logstash_internal' and 'kibana_system' users inside Elasticsearch with the
# values of the passwords defined in the '.env' file.
#
# This task is only performed during the *initial* startup of the stack. On all
# subsequent runs, the service simply returns immediately, without performing
# any modification to existing users.
setup:
build:
context: setup/
args:
ELASTIC_VERSION: ${ELASTIC_VERSION}
init: true
volumes:
- setup:/state:Z
environment:
ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-}
KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}
networks:
- elk
depends_on:
- elasticsearch
elasticsearch:
build:
context: elasticsearch/
args:
ELASTIC_VERSION: ${ELASTIC_VERSION}
volumes:
- ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,z
- elasticsearch:/usr/share/elasticsearch/data:z
ports:
- "9200:9200"
- "9300:9300"
environment:
ES_JAVA_OPTS: -Xms512m -Xmx512m
# Bootstrap password.
# Used to initialize the keystore during the initial startup of
# Elasticsearch. Ignored on subsequent runs.
ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
# Use single node discovery in order to disable production mode and avoid bootstrap checks.
# see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
discovery.type: single-node
networks:
- elk
logstash:
build:
context: logstash/
args:
ELASTIC_VERSION: ${ELASTIC_VERSION}
volumes:
- ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,Z
- ./logstash/pipeline:/usr/share/logstash/pipeline:ro,Z
ports:
- "5044:5044"
- "50000:50000/tcp"
- "50000:50000/udp"
- "9600:9600"
environment:
LS_JAVA_OPTS: -Xms256m -Xmx256m
LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-}
networks:
- elk
depends_on:
- elasticsearch
kibana:
build:
context: kibana/
args:
ELASTIC_VERSION: ${ELASTIC_VERSION}
volumes:
- ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z
ports:
- "5601:5601"
environment:
KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}
# Fleet plugin
KIBANA_FLEET_SETUP: '1'
networks:
- elk
depends_on:
- elasticsearch
networks:
elk:
driver: bridge
volumes:
setup:
elasticsearch:
# .env
ELASTIC_VERSION=8.4.3
## Passwords for stack users
#
# User 'elastic' (built-in)
#
# Superuser role, full access to cluster management and data indices.
# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html
ELASTIC_PASSWORD='changeme'
# User 'logstash_internal' (custom)
#
# The user Logstash uses to connect and send data to Elasticsearch.
# https://www.elastic.co/guide/en/logstash/current/ls-security.html
LOGSTASH_INTERNAL_PASSWORD='changeme'
# User 'kibana_system' (built-in)
#
# The user Kibana uses to connect and communicate with Elasticsearch.
# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html
KIBANA_SYSTEM_PASSWORD='changeme'
# elasticsearch
# elasticsearch-Dockerfile
ARG ELASTIC_VERSION
# https://www.docker.elastic.co/
FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
# Add your elasticsearch plugins setup here
# Example: RUN elasticsearch-plugin install analysis-icu
# elasticsearch/config/elasticsearch.yml
---
## Default Elasticsearch configuration from Elasticsearch base image.
## https://github.com/elastic/elasticsearch/blob/main/distribution/docker/src/docker/config/elasticsearch.yml
#
cluster.name: "docker-cluster"
network.host: 0.0.0.0
## X-Pack settings
## see https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html
#
xpack.license.self_generated.type: trial
xpack.security.enabled: true
# kibana
# kibana-Dockerfile
ARG ELASTIC_VERSION
# https://www.docker.elastic.co/
FROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION}
# Add your kibana plugins setup here
# Example: RUN kibana-plugin install <name|url>
# kibana/config/kibana.yml
---
## Default Kibana configuration from Kibana base image.
## https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.ts
#
server.name: kibana
server.host: 0.0.0.0
elasticsearch.hosts: [ 'http://elasticsearch:9200' ]
monitoring.ui.container.elasticsearch.enabled: true
## X-Pack security credentials
#
elasticsearch.username: kibana_system
elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD}
## Fleet
## https://www.elastic.co/guide/en/kibana/current/fleet-settings-kb.html
#
xpack.fleet.agents.fleet_server.hosts: [ 'http://fleet-server:8220' ]
xpack.fleet.agents.elasticsearch.hosts: [ 'http://elasticsearch:9200' ]
xpack.fleet.packages:
- name: fleet_server
version: latest
- name: system
version: latest
- name: elastic_agent
version: latest
xpack.fleet.agentPolicies:
- name: Fleet Server policy
id: fleet-server-policy
description: Fleet Server policy
monitoring_enabled:
- logs
- metrics
package_policies:
- name: fleet_server-1
package:
name: fleet_server
- name: system-1
package:
name: system
- name: elastic_agent-1
package:
name: elastic_agent
# logstash
# logstash-Dockerfile
ARG ELASTIC_VERSION
# https://www.docker.elastic.co/
FROM docker.elastic.co/logstash/logstash:${ELASTIC_VERSION}
# Add your logstash plugins setup here
# Example: RUN logstash-plugin install logstash-filter-json
# logstash/config/logstash.yml
---
## Default Logstash configuration from Logstash base image.
## https://github.com/elastic/logstash/blob/main/docker/data/logstash/config/logstash-full.yml
#
http.host: "0.0.0.0"
# logstash/pipeline/logstash.conf
input {
beats {
port => 5044
}
tcp {
port => 50000
codec => json
}
}
filter {
json {
source => "message"
add_field => ["type", "%{dtype}"]
remove_field => [ "server", "server.fqdn", "timestamp" ]
}
}
## Add your filters / logstash plugins configuration here
output {
elasticsearch {
hosts => "elasticsearch:9200"
user => "logstash_internal"
password => "${LOGSTASH_INTERNAL_PASSWORD}"
}
}
# setup
# setup-Dockerfile
ARG ELASTIC_VERSION
# https://www.docker.elastic.co/
FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
USER root
COPY . /
RUN set -eux; \
mkdir /state; \
chown elasticsearch /state; \
chmod +x /entrypoint.sh
USER elasticsearch:root
ENTRYPOINT ["/entrypoint.sh"]
# entrypoint.sh
# entrypoint.sh
#!/usr/bin/env bash
set -eu
set -o pipefail
source "$(dirname "${BASH_SOURCE[0]}")/helpers.sh"
# --------------------------------------------------------
# Users declarations
declare -A users_passwords
users_passwords=(
[logstash_internal]="${LOGSTASH_INTERNAL_PASSWORD:-}"
[kibana_system]="${KIBANA_SYSTEM_PASSWORD:-}"
)
declare -A users_roles
users_roles=(
[logstash_internal]='logstash_writer'
)
# --------------------------------------------------------
# Roles declarations
declare -A roles_files
roles_files=(
[logstash_writer]='logstash_writer.json'
)
# --------------------------------------------------------
echo "-------- $(date) --------"
state_file="$(dirname "${BASH_SOURCE[0]}")/state/.done"
if [[ -e "$state_file" ]]; then
log "State file exists at '${state_file}', skipping setup"
exit 0
fi
log 'Waiting for availability of Elasticsearch. This can take several minutes.'
declare -i exit_code=0
wait_for_elasticsearch || exit_code=$?
if ((exit_code)); then
case $exit_code in
6)
suberr 'Could not resolve host. Is Elasticsearch running?'
;;
7)
suberr 'Failed to connect to host. Is Elasticsearch healthy?'
;;
28)
suberr 'Timeout connecting to host. Is Elasticsearch healthy?'
;;
*)
suberr "Connection to Elasticsearch failed. Exit code: ${exit_code}"
;;
esac
exit $exit_code
fi
sublog 'Elasticsearch is running'
for role in "${!roles_files[@]}"; do
log "Role '$role'"
declare body_file
body_file="$(dirname "${BASH_SOURCE[0]}")/roles/${roles_files[$role]:-}"
if [[ ! -f "${body_file:-}" ]]; then
sublog "No role body found at '${body_file}', skipping"
continue
fi
sublog 'Creating/updating'
ensure_role "$role" "$(<"${body_file}")"
done
for user in "${!users_passwords[@]}"; do
log "User '$user'"
if [[ -z "${users_passwords[$user]:-}" ]]; then
sublog 'No password defined, skipping'
continue
fi
declare -i user_exists=0
user_exists="$(check_user_exists "$user")"
if ((user_exists)); then
sublog 'User exists, setting password'
set_user_password "$user" "${users_passwords[$user]}"
else
if [[ -z "${users_roles[$user]:-}" ]]; then
err ' No role defined, skipping creation'
continue
fi
sublog 'User does not exist, creating'
create_user "$user" "${users_passwords[$user]}" "${users_roles[$user]}"
fi
done
mkdir -p "$(dirname "${state_file}")"
touch "$state_file"
# helpers.sh
#!/usr/bin/env bash
# Log a message.
function log {
echo "[+] $1"
}
# Log a message at a sub-level.
function sublog {
echo " ⠿ $1"
}
# Log an error.
function err {
echo "[x] $1" >&2
}
# Log an error at a sub-level.
function suberr {
echo " ⠍ $1" >&2
}
# Poll the 'elasticsearch' service until it responds with HTTP code 200.
function wait_for_elasticsearch {
local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' "http://${elasticsearch_host}:9200/" )
if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
fi
local -i result=1
local output
# retry for max 300s (60*5s)
for _ in $(seq 1 60); do
local -i exit_code=0
output="$(curl "${args[@]}")" || exit_code=$?
if ((exit_code)); then
result=$exit_code
fi
if [[ "${output: -3}" -eq 200 ]]; then
result=0
break
fi
sleep 5
done
if ((result)) && [[ "${output: -3}" -ne 000 ]]; then
echo -e "\n${output::-3}"
fi
return $result
}
# Verify that the given Elasticsearch user exists.
function check_user_exists {
local username=$1
local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
"http://${elasticsearch_host}:9200/_security/user/${username}"
)
if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
fi
local -i result=1
local -i exists=0
local output
output="$(curl "${args[@]}")"
if [[ "${output: -3}" -eq 200 || "${output: -3}" -eq 404 ]]; then
result=0
fi
if [[ "${output: -3}" -eq 200 ]]; then
exists=1
fi
if ((result)); then
echo -e "\n${output::-3}"
else
echo "$exists"
fi
return $result
}
# Set password of a given Elasticsearch user.
function set_user_password {
local username=$1
local password=$2
local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
"http://${elasticsearch_host}:9200/_security/user/${username}/_password"
'-X' 'POST'
'-H' 'Content-Type: application/json'
'-d' "{\"password\" : \"${password}\"}"
)
if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
fi
local -i result=1
local output
output="$(curl "${args[@]}")"
if [[ "${output: -3}" -eq 200 ]]; then
result=0
fi
if ((result)); then
echo -e "\n${output::-3}\n"
fi
return $result
}
# Create the given Elasticsearch user.
function create_user {
local username=$1
local password=$2
local role=$3
local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
"http://${elasticsearch_host}:9200/_security/user/${username}"
'-X' 'POST'
'-H' 'Content-Type: application/json'
'-d' "{\"password\":\"${password}\",\"roles\":[\"${role}\"]}"
)
if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
fi
local -i result=1
local output
output="$(curl "${args[@]}")"
if [[ "${output: -3}" -eq 200 ]]; then
result=0
fi
if ((result)); then
echo -e "\n${output::-3}\n"
fi
return $result
}
# Ensure that the given Elasticsearch role is up-to-date, create it if required.
function ensure_role {
local name=$1
local body=$2
local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
"http://${elasticsearch_host}:9200/_security/role/${name}"
'-X' 'POST'
'-H' 'Content-Type: application/json'
'-d' "$body"
)
if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
fi
local -i result=1
local output
output="$(curl "${args[@]}")"
if [[ "${output: -3}" -eq 200 ]]; then
result=0
fi
if ((result)); then
echo -e "\n${output::-3}\n"
fi
return $result
}
# setup/roles/logstash_writer.json
{
"cluster": [
"manage_index_templates",
"monitor",
"manage_ilm"
],
"indices": [
{
"names": [
"logs-generic-default",
"logstash-*",
"ecs-logstash-*"
],
"privileges": [
"write",
"create",
"create_index",
"manage",
"manage_ilm"
]
},
{
"names": [
"logstash",
"ecs-logstash"
],
"privileges": [
"write",
"manage"
]
}
]
}
上次更新: 2022/11/14, 11:24:11