From 38f5ac4406bc0a12e91a98c98e513bb42648a4f4 Mon Sep 17 00:00:00 2001 From: Florian Zirker Date: Sun, 28 Mar 2021 13:50:02 +0200 Subject: [PATCH] Monitoring with telegraf, influxdb grafana: - Influxdb (1.8) behind traefik proxy so that telegraf can run in host_mode. Secured with HTTP basic auth - Grafana (7.4) with Postgres database for beatifull dashboards - Telegraf (1.18) in docker host mode with local configuration file - Added docker.group to all other docker services to track them in monitoring --- gitea/docker-compose.yaml | 3 + monitoring/docker-compose.yaml | 107 +++++++++++++ monitoring/influxdb.conf | 12 ++ monitoring/telegraf.conf | 279 +++++++++++++++++++++++++++++++++ nextcloud/docker-compose.yaml | 6 + proxy/docker-compose.yaml | 2 + wallabag/docker-compose.yaml | 3 + www/docker-compose.yaml | 1 + 8 files changed, 413 insertions(+) create mode 100644 monitoring/docker-compose.yaml create mode 100644 monitoring/influxdb.conf create mode 100644 monitoring/telegraf.conf diff --git a/gitea/docker-compose.yaml b/gitea/docker-compose.yaml index 5fbcab2..e632a46 100644 --- a/gitea/docker-compose.yaml +++ b/gitea/docker-compose.yaml @@ -20,6 +20,7 @@ services: - "traefik.http.routers.gitea.tls.certresolver=netcup" - "traefik.http.routers.gitea.tls.options=intermediate@file" - "traefik.http.services.gitea.loadbalancer.server.port=3000" + - "docker.group=gitea" depends_on: - db restart: unless-stopped @@ -45,6 +46,8 @@ services: - gitea volumes: - ${VOLUMES_PATH}/gitea_db:/var/lib/postgresql/data + labels: + - "docker.group=gitea" networks: diff --git a/monitoring/docker-compose.yaml b/monitoring/docker-compose.yaml new file mode 100644 index 0000000..d320ff3 --- /dev/null +++ b/monitoring/docker-compose.yaml @@ -0,0 +1,107 @@ +version: "3.3" +services: + + grafana: + image: grafana/grafana:7.4.3 + restart: unless-stopped + networks: + - web + - monitoring + labels: + - "traefik.enable=true" + - "traefik.http.routers.grafana.rule=Host(`monitoring.${DOMAIN}`)" + - "traefik.http.routers.grafana.entrypoints=websecure" + - "traefik.http.routers.grafana.tls.certresolver=netcup" + - "traefik.http.routers.grafana.tls.options=intermediate@file" + - "traefik.http.services.grafana.loadbalancer.server.port=3000" + - "traefik.docker.network=web" + - "docker.group=monitoring" + environment: + - GF_DEFAULT_INSTANCE_NAME=monitoring.${DOMAIN} + - GF_SERVER_ROOT_URL=http://monitoring.${DOMAIN} + - GF_SERVER_DOMAIN=monitoring.${DOMAIN} + - GF_SERVER_SERVE_FROM_SUB_PATH=true + - GF_SECURITY_DISABLE_GRAVATAR=true + - GF_AUTH_ANONYMOUS_ENABLED=false + - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer + - GF_DATABASE_TYPE=postgres + - GF_DATABASE_HOST=grafanadb:5432 + - GF_DATABASE_SSL_MODE=disable + - GF_DATABASE_NAME=grafana + - GF_DATABASE_USER=${POSTGRES_USER} + - GF_DATABASE_PASSWORD=${POSTGRES_PASSWORD} + - GF_INSTALL_PLUGINS=flant-statusmap-panel + depends_on: + - influxdb + - grafanadb + + + grafanadb: + image: postgres + volumes: + - ${VOLUMES_PATH}/grafanadb:/var/lib/postgresql/data + networks: + - monitoring + restart: unless-stopped + environment: + - POSTGRES_DB=grafana + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + labels: + - "docker.group=monitoring" + + + influxdb: + image: influxdb:1.8 + restart: unless-stopped + networks: + - web + - monitoring + environment: + - INFLUXDB_MONITOR_STORE_ENABLED=false + volumes: + - ${VOLUMES_PATH}/influxdb/:/var/lib/influxdb + - ${PWD}/influxdb.conf:/etc/influxdb/influxdb.conf:ro + labels: + - "traefik.enable=true" + - "traefik.http.routers.influxdb.rule=Host(`influxdb.${DOMAIN}`)" + - "traefik.http.routers.influxdb.entrypoints=websecure" + - "traefik.http.routers.influxdb.tls.certresolver=netcup" + - "traefik.http.routers.influxdb.tls.options=intermediate@file" + - "traefik.http.services.influxdb.loadbalancer.server.port=8086" + - "traefik.http.routers.influxdb.middlewares=influxauth" + - "traefik.http.middlewares.influxauth.basicauth.users=${INFLUXDB_HTPASSWD}" + - "docker.group=monitoring" + + + + ################################################################## + # here starts data collection + + telegraf: + image: telegraf:1.18 + restart: unless-stopped + environment: + - HOST_MOUNT_PREFIX=/hostfs + - HOST_PROC=/hostfs/proc + - HOST_SYS=/hostfs/sys + - HOST_ETC=/hostfs/etc + - HOST_VAR=/hostfs/var + - HOST_RUN=/hostfs/run + env_file: + - ./.env # set environments into container + volumes: + - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/run/utmp:/var/run/utmp:ro + - /:/hostfs:ro + network_mode: "host" + labels: + - "docker.group=monitoring" + depends_on: + - influxdb + +networks: + monitoring: + web: + external: true diff --git a/monitoring/influxdb.conf b/monitoring/influxdb.conf new file mode 100644 index 0000000..9244c34 --- /dev/null +++ b/monitoring/influxdb.conf @@ -0,0 +1,12 @@ +[meta] + dir = "/var/lib/influxdb/meta" + +[data] + dir = "/var/lib/influxdb/data" + wal-dir = "/var/lib/influxdb/wal" + max-concurrent-compactions = 1 + +[monitor] + store-enabled = false + store-database = "_internal" + store-interval = "10s" diff --git a/monitoring/telegraf.conf b/monitoring/telegraf.conf new file mode 100644 index 0000000..ec5c6d4 --- /dev/null +++ b/monitoring/telegraf.conf @@ -0,0 +1,279 @@ +# Telegraf Configuration +# +# Telegraf is entirely plugin driven. All metrics are gathered from the +# declared inputs, and sent to the declared outputs. +# +# Plugins must be declared in here to be active. +# To deactivate a plugin, comment out the name and any variables. +# +# Use 'telegraf -config telegraf.conf -test' to see what metrics a config +# file would generate. +# +# Environment variables can be used anywhere in this config file, simply surround +# them with ${}. For strings the variable must be within quotes (ie, "${STR_VAR}"), +# for numbers and booleans they should be plain (ie, ${INT_VAR}, ${BOOL_VAR}) + +# Config Sample under https://github.com/influxdata/telegraf/blob/master/etc/telegraf.conf + +# Global tags can be specified here in key="value" format. +[global_tags] + # datacenter + dc="florianzirker.de" + +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "10s" + ## Rounds collection interval to 'interval' + ## ie, if interval="10s" then always collect on :00, :10, :20, etc. + round_interval = true + + ## Telegraf will send metrics to outputs in batches of at most + ## metric_batch_size metrics. + ## This controls the size of writes that Telegraf sends to output plugins. + metric_batch_size = 1000 + + ## Maximum number of unwritten metrics per output. Increasing this value + ## allows for longer periods of output downtime without dropping metrics at the + ## cost of higher maximum memory usage. + metric_buffer_limit = 10000 + + ## Collection jitter is used to jitter the collection by a random amount. + ## Each plugin will sleep for a random time within jitter before collecting. + ## This can be used to avoid many plugins querying things like sysfs at the + ## same time, which can have a measurable effect on the system. + collection_jitter = "0s" + + ## Default flushing interval for all outputs. Maximum flush_interval will be + ## flush_interval + flush_jitter + flush_interval = "10s" + ## Jitter the flush interval by a random amount. This is primarily to avoid + ## large write spikes for users running a large number of telegraf instances. + ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + flush_jitter = "0s" + + ## By default or when set to "0s", precision will be set to the same + ## timestamp order as the collection interval, with the maximum being 1s. + ## ie, when interval = "10s", precision will be "1s" + ## when interval = "250ms", precision will be "1ms" + ## Precision will NOT be used for service inputs. It is up to each individual + ## service input to set the timestamp at the appropriate precision. + ## Valid time units are "ns", "us" (or "µs"), "ms", "s". + precision = "" + + ## Override default hostname, if empty use os.Hostname() + hostname = "wong" + ## If set to true, do no set the "host" tag in the telegraf agent. + omit_hostname = false + + +############################################################################### +# OUTPUT PLUGINS # +############################################################################### + +# Configuration for sending metrics to InfluxDB +[[outputs.influxdb]] + ## The full HTTP or UDP URL for your InfluxDB instance. + ## + ## Multiple URLs can be specified for a single cluster, only ONE of the + ## urls will be written to each interval. + # urls = ["unix:///var/run/influxdb.sock"] + # urls = ["udp://127.0.0.1:8089"] + # urls = ["http://127.0.0.1:8086"] + + ## HTTP Basic Auth + username = "${INFLUXDB_HTTP_BASIC_AUTH_USER}" + password = "${INFLUXDB_HTTP_BASIC_AUTH_PASSWORD}" + urls = ["https://influxdb.florianzirker.de"] # required + + +############################################################################### +# INPUT PLUGINS # +############################################################################### + + +# Read metrics about cpu usage +[[inputs.cpu]] + ## Whether to report per-cpu stats or not + percpu = true + ## Whether to report total system cpu stats or not + totalcpu = true + ## If true, collect raw CPU time metrics. + collect_cpu_time = false + ## If true, compute and report the sum of all non-idle CPU states. + report_active = false + + +# Read metrics about disk usage by mount point +[[inputs.disk]] + ## By default stats will be gathered for all mount points. + ## Set mount_points will restrict the stats to only the specified mount points. + mount_points = ["/hostfs", "/hostfs/boot"] + + ## Ignore mount points by filesystem type. + ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] + + +# Read metrics about disk IO by device +[[inputs.diskio]] + ## By default, telegraf will gather stats for all devices including + ## disk partitions. + ## Setting devices will restrict the stats to the specified devices. + # devices = ["sda", "sdb", "vd*"] + ## Uncomment the following line if you need disk serial numbers. + # skip_serial_number = false + # + ## On systems which support it, device metadata can be added in the form of + ## tags. + ## Currently only Linux is supported via udev properties. You can view + ## available properties for a device by running: + ## 'udevadm info -q property -n /dev/sda' + ## Note: Most, but not all, udev properties can be accessed this way. Properties + ## that are currently inaccessible include DEVTYPE, DEVNAME, and DEVPATH. + # device_tags = ["ID_FS_TYPE", "ID_FS_USAGE"] + # + ## Using the same metadata source as device_tags, you can also customize the + ## name of the device via templates. + ## The 'name_templates' parameter is a list of templates to try and apply to + ## the device. The template may contain variables in the form of '$PROPERTY' or + ## '${PROPERTY}'. The first template which does not contain any variables not + ## present for the device is used as the device name tag. + ## The typical use case is for LVM volumes, to get the VG/LV name instead of + ## the near-meaningless DM-0 name. + # name_templates = ["$ID_FS_LABEL","$DM_VG_NAME/$DM_LV_NAME"] + + +# Get kernel statistics from /proc/stat +[[inputs.kernel]] + # no configuration + + +# Read metrics about memory usage +[[inputs.mem]] + # no configuration + + +# Get the number of processes and group them by status +[[inputs.processes]] + # no configuration + + +# Read metrics about swap memory usage +[[inputs.swap]] + # no configuration + + +# Read metrics about system load & uptime +[[inputs.system]] + ## Uncomment to remove deprecated metrics. + # fielddrop = ["uptime_format"] + + + +# Read metrics about docker containers +[[inputs.docker]] + ## Docker Endpoint + ## To use TCP, set endpoint = "tcp://[ip]:[port]" + ## To use environment variables (ie, docker-machine), set endpoint = "ENV" + endpoint = "unix:///var/run/docker.sock" + + ## Set to true to collect Swarm metrics(desired_replicas, running_replicas) + ## Note: configure this in one of the manager nodes in a Swarm cluster. + ## configuring in multiple Swarm managers results in duplication of metrics. + gather_services = false + + ## Only collect metrics for these containers. Values will be appended to + ## container_name_include. + ## Deprecated (1.4.0), use container_name_include + container_names = [] + + ## Set the source tag for the metrics to the container ID hostname, eg first 12 chars + source_tag = false + + ## Containers to include and exclude. Collect all if empty. Globs accepted. + container_name_include = [] + container_name_exclude = [] + + ## Container states to include and exclude. Globs accepted. + ## When empty only containers in the "running" state will be captured. + ## example: container_state_include = ["created", "restarting", "running", "removing", "paused", "exited", "dead"] + ## example: container_state_exclude = ["created", "restarting", "running", "removing", "paused", "exited", "dead"] + # container_state_include = [] + # container_state_exclude = [] + + ## Timeout for docker list, info, and stats commands + timeout = "5s" + + ## Whether to report for each container per-device blkio (8:0, 8:1...), + ## network (eth0, eth1, ...) and cpu (cpu0, cpu1, ...) stats or not. + ## Usage of this setting is discouraged since it will be deprecated in favor of 'perdevice_include'. + ## Default value is 'true' for backwards compatibility, please set it to 'false' so that 'perdevice_include' setting + ## is honored. + perdevice = false + + ## Specifies for which classes a per-device metric should be issued + ## Possible values are 'cpu' (cpu0, cpu1, ...), 'blkio' (8:0, 8:1, ...) and 'network' (eth0, eth1, ...) + ## Please note that this setting has no effect if 'perdevice' is set to 'true' + perdevice_include = ["cpu", "blkio", "network"] + + ## Whether to report for each container total blkio and network stats or not. + ## Usage of this setting is discouraged since it will be deprecated in favor of 'total_include'. + ## Default value is 'false' for backwards compatibility, please set it to 'true' so that 'total_include' setting + ## is honored. + total = true + + ## Specifies for which classes a total metric should be issued. Total is an aggregated of the 'perdevice' values. + ## Possible values are 'cpu', 'blkio' and 'network' + ## Total 'cpu' is reported directly by Docker daemon, and 'network' and 'blkio' totals are aggregated by this plugin. + ## Please note that this setting has no effect if 'total' is set to 'false' + total_include = ["cpu", "blkio", "network"] + + ## docker labels to include and exclude as tags. Globs accepted. + ## Note that an empty array for both will include all labels as tags + docker_label_include = [] + docker_label_exclude = [] + + ## Which environment variables should we use as a tag + tag_env = ["JAVA_HOME", "HEAP_SIZE"] + + ## Optional TLS Config + # tls_ca = "/etc/telegraf/ca.pem" + # tls_cert = "/etc/telegraf/cert.pem" + # tls_key = "/etc/telegraf/key.pem" + ## Use TLS but skip chain & host verification + # insecure_skip_verify = false + + +# Gather metrics about network interfaces +[[inputs.net]] + ## By default, telegraf gathers stats from any up interface (excluding loopback) + ## Setting interfaces will tell it to gather these explicit interfaces, + ## regardless of status. When specifying an interface, glob-style + ## patterns are also supported. + ## + interfaces = ["eth*"] + ## + ## On linux systems telegraf also collects protocol stats. + ## Setting ignore_protocol_stats to true will skip reporting of protocol metrics. + ## + # ignore_protocol_stats = false + ## + + +# # Read TCP metrics such as established, time wait and sockets counts. +[[inputs.netstat]] + # no configuration + + +# Collect kernel snmp counters and network interface statistics +[[inputs.nstat]] + ## file paths for proc files. If empty default paths will be used: + ## /proc/net/netstat, /proc/net/snmp, /proc/net/snmp6 + ## These can also be overridden with env variables, see README. + proc_net_netstat = "/proc/net/netstat" + proc_net_snmp = "/proc/net/snmp" + proc_net_snmp6 = "/proc/net/snmp6" + ## dump metrics with 0 values too + dump_zeros = true + + diff --git a/nextcloud/docker-compose.yaml b/nextcloud/docker-compose.yaml index 1a5acb6..af39c75 100644 --- a/nextcloud/docker-compose.yaml +++ b/nextcloud/docker-compose.yaml @@ -25,6 +25,7 @@ services: - "traefik.http.middlewares.nextcloudHeader.headers.stsPreload=true" - "traefik.http.middlewares.nextcloudHeader.headers.forceSTSHeader=true" - "traefik.http.routers.nextcloud.middlewares=nextcloudHeader" + - "docker.group=netxtcloud" app: image: nextcloud:20-fpm @@ -62,6 +63,8 @@ services: - MYSQL_USER=${MYSQL_USER} networks: - nextcloud + labels: + - "docker.group=netxtcloud" redis: image: redis:alpine @@ -71,6 +74,8 @@ services: - nextcloud volumes: - ${VOLUMES_PATH}/nextcloud_redis:/data + labels: + - "docker.group=netxtcloud" office: @@ -99,6 +104,7 @@ services: - "traefik.http.middlewares.officeHeader.headers.stsPreload=true" - "traefik.http.middlewares.officeHeader.headers.forceSTSHeader=true" - "traefik.http.routers.office.middlewares=officeHeader" + - "docker.group=netxtcloud" networks: diff --git a/proxy/docker-compose.yaml b/proxy/docker-compose.yaml index d8d7c51..d0bfec2 100644 --- a/proxy/docker-compose.yaml +++ b/proxy/docker-compose.yaml @@ -48,6 +48,7 @@ services: - "traefik.http.routers.dashboard.tls.options=intermediate@file" - "traefik.http.routers.dashboard.middlewares=auth" - "traefik.http.middlewares.auth.basicauth.users=${HTPASSWD}" + - "docker.group=proxy" # whoami: @@ -59,6 +60,7 @@ services: # - "traefik.http.routers.whoami.rule=Host(`whoami.${DOMAIN}`)" # - "traefik.http.routers.whoami.entrypoints=websecure" # - "traefik.http.routers.whoami.tls.certresolver=netcup" +# - "docker.group=proxy" # restart: unless-stopped diff --git a/wallabag/docker-compose.yaml b/wallabag/docker-compose.yaml index b059e7e..4093979 100644 --- a/wallabag/docker-compose.yaml +++ b/wallabag/docker-compose.yaml @@ -28,6 +28,7 @@ services: - "traefik.http.routers.wallabag.entrypoints=websecure" - "traefik.http.routers.wallabag.tls.certresolver=netcup" - "traefik.http.routers.wallabag.tls.options=intermediate@file" + - "docker.group=wallabag" depends_on: - db - redis @@ -41,6 +42,8 @@ services: - wallabag volumes: - /var/dockervolumes/wallabag_db:/var/lib/mysql + labels: + - "docker.group=wallabag" redis: image: redis:alpine diff --git a/www/docker-compose.yaml b/www/docker-compose.yaml index aebd24d..3882ca9 100644 --- a/www/docker-compose.yaml +++ b/www/docker-compose.yaml @@ -15,6 +15,7 @@ services: - "traefik.http.routers.web-secure.rule=Host(`${DOMAIN}`) || Host(`www.${DOMAIN}`)" - "traefik.http.routers.web-secure.tls.certresolver=netcup" - "traefik.http.routers.web-secure.tls.options=intermediate@file" + - "docker.group=www" networks: web: