From 4bd9e36ff45b2001b048a23a218e26605deac7d0 Mon Sep 17 00:00:00 2001 From: Georg Date: Fri, 6 Aug 2021 17:42:45 +0200 Subject: Mass Commit 06/08/2021 Signed-off-by: Georg --- Prometheus/alerts/lysergic/blackbox.yml | 94 ++++++++++++++++++++++ Prometheus/alerts/lysergic/nginx_exporters.yml | 13 +++ Prometheus/alerts/lysergic/node_exporters.yml | 13 +++ Prometheus/alerts/lysergic/wireguard_exporters.yml | 23 ++++++ Prometheus/alerts/tripsit/blackbox.yml | 21 +++++ Prometheus/alerts/tripsit/node_exporters.yml | 13 +++ Prometheus/prometheus.service | 13 +++ Prometheus/prometheus.yml | 69 ++++++++++++++++ Prometheus/targets/README.md | 1 + 9 files changed, 260 insertions(+) create mode 100644 Prometheus/alerts/lysergic/blackbox.yml create mode 100644 Prometheus/alerts/lysergic/nginx_exporters.yml create mode 100644 Prometheus/alerts/lysergic/node_exporters.yml create mode 100644 Prometheus/alerts/lysergic/wireguard_exporters.yml create mode 100644 Prometheus/alerts/tripsit/blackbox.yml create mode 100644 Prometheus/alerts/tripsit/node_exporters.yml create mode 100644 Prometheus/prometheus.service create mode 100644 Prometheus/prometheus.yml create mode 100644 Prometheus/targets/README.md (limited to 'Prometheus') diff --git a/Prometheus/alerts/lysergic/blackbox.yml b/Prometheus/alerts/lysergic/blackbox.yml new file mode 100644 index 0000000..49a553d --- /dev/null +++ b/Prometheus/alerts/lysergic/blackbox.yml @@ -0,0 +1,94 @@ +groups: +- name: lysergic-blackbox_exporters + rules: + - alert: LIBERTACASA-WEB-DOWN + expr: probe_success{instance=~"https://liberta.casa|https://www.lysergic.dev"} == 0 + for: 30s + labels: + severity: warning + project: LYSERGIC + annotations: + title: Node {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down. + - alert: XKEK-WEB-DOWN + expr: probe_success{instance=~"https://xkek.net|https://kekx.net"} == 0 + for: 30s + labels: + severity: warning + project: LYSERGIC + annotations: + title: Node {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down. + - alert: JITSI-WEB-DOWN + expr: probe_success{instance=~"https://meet.lysergic.dev"} == 0 + for: 30s + labels: + severity: warning + project: LYSERGIC + annotations: + title: Node {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down. + - alert: PASTA-WEB-DOWN + expr: probe_success{instance=~"https://pasta.lysergic.dev"} == 0 + for: 30s + labels: + severity: warning + project: LYSERGIC + annotations: + title: Node {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down. + - alert: GITEA-WEB-DOWN + expr: probe_success{instance=~"https://git.com.de"} == 0 + for: 30s + labels: + severity: warning + project: LYSERGIC + annotations: + title: Node {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down. + - alert: CGIT-WEB-DOWN + expr: probe_success{instance=~"https://git.casa"} == 0 + for: 30s + labels: + severity: warning + project: LYSERGIC + annotations: + title: Node {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down. + - alert: ELEMENT-WEB-DOWN + expr: probe_success{instance=~"https://element.liberta.casa"} == 0 + for: 30s + labels: + severity: warning + project: LYSERGIC + annotations: + title: Node {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down. + - alert: ETHERPAD-WEB-DOWN + expr: probe_success{instance=~"https://pad.hugz.io"} == 0 + for: 30s + labels: + severity: warning + project: LYSERGIC + annotations: + title: Node {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down. + - alert: SEARX-WEB-DOWN + expr: probe_success{instance=~"https://searx.xkek.net"} == 0 + for: 120s + labels: + severity: warning + project: LYSERGIC + annotations: + title: Node {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down. + - alert: YACY-WEB-DOWN + expr: probe_success{instance=~"https://yacy.xkek.net"} == 0 + for: 30s + labels: + severity: warning + project: LYSERGIC + annotations: + title: Node {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down. +/etc/prometheus/alerts/lysergic/blackbox.yml lines 29-93/93 (END) diff --git a/Prometheus/alerts/lysergic/nginx_exporters.yml b/Prometheus/alerts/lysergic/nginx_exporters.yml new file mode 100644 index 0000000..8a3777d --- /dev/null +++ b/Prometheus/alerts/lysergic/nginx_exporters.yml @@ -0,0 +1,13 @@ +groups: +- name: lysergic-nginx_exporters + rules: + - alert: lysergic-nginx-down + expr: nginx_up{project="LYSERGIC", job="nginx_exporters"} == 0 + for: 1m + labels: + job: nginx_exporters + severity: WARNING + project: LYSERGIC + annotations: + title: nginx {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 1 minute. nginx seems down. diff --git a/Prometheus/alerts/lysergic/node_exporters.yml b/Prometheus/alerts/lysergic/node_exporters.yml new file mode 100644 index 0000000..392b3b7 --- /dev/null +++ b/Prometheus/alerts/lysergic/node_exporters.yml @@ -0,0 +1,13 @@ +groups: +- name: lysergic-node_exporters + rules: + - alert: lysergic-node-down + expr: up{project="LYSERGIC", job="node_exporters"} == 0 + for: 1m + labels: + job: node_exporters + severity: warning + project: LYSERGIC + annotations: + title: Node {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 1 minute. Node seems down. diff --git a/Prometheus/alerts/lysergic/wireguard_exporters.yml b/Prometheus/alerts/lysergic/wireguard_exporters.yml new file mode 100644 index 0000000..87fecb6 --- /dev/null +++ b/Prometheus/alerts/lysergic/wireguard_exporters.yml @@ -0,0 +1,23 @@ +groups: +- name: lysergic-wireguard_exporters + rules: + - alert: mercury-wireguard-down + expr: wireguard_peer_last_handshake_seconds{device="wg0", instance="localhost:9586", job="wireguard-mercury", public_key="$WG0_PUBKEY"} - time() <- 125 + for: 10s + labels: + job: wireguard-mercury + severity: CRITICAL + project: LYSERGIC + annotations: + title: WireGuard {{ $labels.instance }} is down + description: 'Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 2 minutes. Uplink to Psyched seems down. Affected services: Global infrastructure monitoring. Other outages might no longer be tracked. Urgent investigation is advised.' + - alert: lysergic-wireguard-down + expr: wireguard_peer_last_handshake_seconds{project="LYSERGIC",device!="wg1"} - time() <- 130 + for: 10s + labels: + job: wireguard-lysergic + severity: CRITICAL + project: LYSERGIC + annotations: + title: WireGuard Incident + description: 'No handshakes for {{ $labels.device }} in {{ $labels.job }} on {{ $labels.instance}} have been exchanged for an extended amount of time. Critical services could be affected.' diff --git a/Prometheus/alerts/tripsit/blackbox.yml b/Prometheus/alerts/tripsit/blackbox.yml new file mode 100644 index 0000000..0390718 --- /dev/null +++ b/Prometheus/alerts/tripsit/blackbox.yml @@ -0,0 +1,21 @@ +groups: +- name: tripsit-blackbox_exporters + rules: + - alert: TRIPSIT.ME-WEB-INCIDENT + expr: probe_success{instance=~"https://tripsit.me|https://chat.tripsit.me|https://chat.tripsit.me/chat|https://drugs.tripsit.me|https://benzo.tripsit.me|https://dxm.tripsit.me|https://combo.tripsit.me|https://wiki.tripsit.me|https://tripbot.tripsit.me"} == 0 + for: 15s + labels: + severity: critical + project: TRIPSIT + annotations: + title: Node {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 15 seconds. Node seems down. + - alert: TRIPSIT.DEV-WEB-INCIDENT + expr: probe_success{instance=~"https://mail.tripsit.dev|https://dopamine.tripsit.dev"} == 0 + for: 1m + labels: + severity: warning + project: TRIPSIT + annotations: + title: Node {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 1 minute. Node seems down. diff --git a/Prometheus/alerts/tripsit/node_exporters.yml b/Prometheus/alerts/tripsit/node_exporters.yml new file mode 100644 index 0000000..ee82367 --- /dev/null +++ b/Prometheus/alerts/tripsit/node_exporters.yml @@ -0,0 +1,13 @@ +groups: +- name: tripsit-node_exporters + rules: + - alert: TRIPSIT-NODE-INCIDENT + expr: up{project="TRIPSIT", job="node_exporters"} == 0 + for: 30s + labels: + severity: warning + job: node_exporters + project: TRIPSIT + annotations: + title: Node {{ $labels.instance }} is down + description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 1 minute. Node seems down. diff --git a/Prometheus/prometheus.service b/Prometheus/prometheus.service new file mode 100644 index 0000000..38b063f --- /dev/null +++ b/Prometheus/prometheus.service @@ -0,0 +1,13 @@ +[Unit] +Description=Prometheus +After=network.target + +[Service] +User=prometheus +Group=prometheus +Type=simple +ExecStart=/opt/prometheus/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus --web.console.templates=/opt/prometheus/consoles --web.console.libraries=/opt/prometheus/console_libraries --web.external-url=https://prometheus.lysergic.dev:4433 +ExecReload=/usr/bin/kill -HUP $MAINPID + +[Install] +WantedBy=multi-user.target diff --git a/Prometheus/prometheus.yml b/Prometheus/prometheus.yml new file mode 100644 index 0000000..df902f6 --- /dev/null +++ b/Prometheus/prometheus.yml @@ -0,0 +1,69 @@ +# my global config +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 1m # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + - localhost:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + - "/etc/prometheus/alerts/lysergic/*.yml" +# - "/etc/prometheus/alerts/tripsit/*.yml" - Disabled on 22/07/2021 + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'node_exporters' + file_sd_configs: + - files: + - '/etc/prometheus/targets/node-*.json' +# - job_name: 'node_exporters_02' +# file_sd_configs: +# - files: +# - '/etc/prometheus/targets/node-tripsit.json' +# proxy_url: 'https://dopamine.tripsit.dev:9493/' + #params: + # _scheme: [https] + + - job_name: 'nginx_exporters' + file_sd_configs: + - files: + - '/etc/prometheus/targets/nginx-*.json' + + - job_name: 'etcd_metrics' + static_configs: + - targets: ['$ETCDNODE01:2379', '$ETCDNODE02:2379', '$ETCDNODE03:2379'] + scheme: https + tls_config: + ca_file: $ETCDCACRT + cert_file: $ETCDCLIENTCRT + key_file: $ETCDCLIENTKEY + + - job_name: 'blackbox' + metrics_path: /probe + params: + module: [http_2xx] + file_sd_configs: + - files: ['/etc/prometheus/targets/blackbox*.yml'] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9115 + + - job_name: 'wireguard_exporters' + file_sd_configs: + - files: + - '/etc/prometheus/targets/wireguard-*.json' diff --git a/Prometheus/targets/README.md b/Prometheus/targets/README.md new file mode 100644 index 0000000..46cd645 --- /dev/null +++ b/Prometheus/targets/README.md @@ -0,0 +1 @@ +These configuration files are currently not available publicly. -- cgit v1.2.3