9 files changed, 260 insertions, 0 deletions
diff --git a/Prometheus/alerts/lysergic/blackbox.yml b/Prometheus/alerts/lysergic/blackbox.yml
new file mode 100644
index 0000000..49a553d
--- /dev/null
+++ b/Prometheus/alerts/lysergic/blackbox.yml
@@ -0,0 +1,94 @@
+groups:
+- name: lysergic-blackbox_exporters
+  rules:
+  - alert: LIBERTACASA-WEB-DOWN
+    expr: probe_success{instance=~"https://liberta.casa|https://www.lysergic.dev"} == 0
+    for: 30s
+    labels:
+      severity: warning
+      project: LYSERGIC
+    annotations:
+      title: Node {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down.
+  - alert: XKEK-WEB-DOWN
+    expr: probe_success{instance=~"https://xkek.net|https://kekx.net"} == 0
+    for: 30s
+    labels:
+      severity: warning
+      project: LYSERGIC
+    annotations:
+      title: Node {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down.
+  - alert: JITSI-WEB-DOWN
+    expr: probe_success{instance=~"https://meet.lysergic.dev"} == 0
+    for: 30s
+    labels:
+      severity: warning
+      project: LYSERGIC
+    annotations:
+      title: Node {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down.
+  - alert: PASTA-WEB-DOWN
+    expr: probe_success{instance=~"https://pasta.lysergic.dev"} == 0
+    for: 30s
+    labels:
+      severity: warning
+      project: LYSERGIC
+    annotations:
+      title: Node {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down.
+  - alert: GITEA-WEB-DOWN
+    expr: probe_success{instance=~"https://git.com.de"} == 0
+    for: 30s
+    labels:
+      severity: warning
+      project: LYSERGIC
+    annotations:
+      title: Node {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down.
+  - alert: CGIT-WEB-DOWN
+    expr: probe_success{instance=~"https://git.casa"} == 0
+    for: 30s
+    labels:
+      severity: warning
+      project: LYSERGIC
+    annotations:
+      title: Node {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down.
+  - alert: ELEMENT-WEB-DOWN
+    expr: probe_success{instance=~"https://element.liberta.casa"} == 0
+    for: 30s
+    labels:
+      severity: warning
+      project: LYSERGIC
+    annotations:
+      title: Node {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down.
+  - alert: ETHERPAD-WEB-DOWN
+    expr: probe_success{instance=~"https://pad.hugz.io"} == 0
+    for: 30s
+    labels:
+      severity: warning
+      project: LYSERGIC
+    annotations:
+      title: Node {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down.
+  - alert: SEARX-WEB-DOWN
+    expr: probe_success{instance=~"https://searx.xkek.net"} == 0
+    for: 120s
+    labels:
+      severity: warning
+      project: LYSERGIC
+    annotations:
+      title: Node {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down.
+  - alert: YACY-WEB-DOWN
+    expr: probe_success{instance=~"https://yacy.xkek.net"} == 0
+    for: 30s
+    labels:
+      severity: warning
+      project: LYSERGIC
+    annotations:
+      title: Node {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 30 seconds. Node seems down.
+/etc/prometheus/alerts/lysergic/blackbox.yml lines 29-93/93 (END)
diff --git a/Prometheus/alerts/lysergic/nginx_exporters.yml b/Prometheus/alerts/lysergic/nginx_exporters.yml
new file mode 100644
index 0000000..8a3777d
--- /dev/null
+++ b/Prometheus/alerts/lysergic/nginx_exporters.yml
@@ -0,0 +1,13 @@
+groups:
+- name: lysergic-nginx_exporters
+  rules:
+  - alert: lysergic-nginx-down
+    expr: nginx_up{project="LYSERGIC", job="nginx_exporters"} == 0
+    for: 1m
+    labels:
+      job: nginx_exporters
+      severity: WARNING
+      project: LYSERGIC
+    annotations:
+      title: nginx {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 1 minute. nginx seems down.
diff --git a/Prometheus/alerts/lysergic/node_exporters.yml b/Prometheus/alerts/lysergic/node_exporters.yml
new file mode 100644
index 0000000..392b3b7
--- /dev/null
+++ b/Prometheus/alerts/lysergic/node_exporters.yml
@@ -0,0 +1,13 @@
+groups:
+- name: lysergic-node_exporters
+  rules:
+  - alert: lysergic-node-down
+    expr: up{project="LYSERGIC", job="node_exporters"} == 0
+    for: 1m
+    labels:
+      job: node_exporters
+      severity: warning
+      project: LYSERGIC
+    annotations:
+      title: Node {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 1 minute. Node seems down.
diff --git a/Prometheus/alerts/lysergic/wireguard_exporters.yml b/Prometheus/alerts/lysergic/wireguard_exporters.yml
new file mode 100644
index 0000000..87fecb6
--- /dev/null
+++ b/Prometheus/alerts/lysergic/wireguard_exporters.yml
@@ -0,0 +1,23 @@
+groups:
+- name: lysergic-wireguard_exporters
+  rules:
+  - alert: mercury-wireguard-down
+    expr: wireguard_peer_last_handshake_seconds{device="wg0", instance="localhost:9586", job="wireguard-mercury", public_key="$WG0_PUBKEY"} - time() <- 125
+    for: 10s
+    labels:
+      job: wireguard-mercury
+      severity: CRITICAL
+      project: LYSERGIC
+    annotations:
+      title: WireGuard {{ $labels.instance }} is down
+      description: 'Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 2 minutes. Uplink to Psyched seems down. Affected services: Global infrastructure monitoring. Other outages might no longer be tracked. Urgent investigation is advised.'
+  - alert: lysergic-wireguard-down
+    expr: wireguard_peer_last_handshake_seconds{project="LYSERGIC",device!="wg1"} - time() <- 130
+    for: 10s
+    labels:
+      job: wireguard-lysergic
+      severity: CRITICAL
+      project: LYSERGIC
+    annotations:
+      title: WireGuard Incident
+      description: 'No handshakes for {{ $labels.device }} in {{ $labels.job }} on {{ $labels.instance}} have been exchanged for an extended amount of time. Critical services could be affected.'
diff --git a/Prometheus/alerts/tripsit/blackbox.yml b/Prometheus/alerts/tripsit/blackbox.yml
new file mode 100644
index 0000000..0390718
--- /dev/null
+++ b/Prometheus/alerts/tripsit/blackbox.yml
@@ -0,0 +1,21 @@
+groups:
+- name: tripsit-blackbox_exporters
+  rules:
+  - alert: TRIPSIT.ME-WEB-INCIDENT
+    expr:  probe_success{instance=~"https://tripsit.me|https://chat.tripsit.me|https://chat.tripsit.me/chat|https://drugs.tripsit.me|https://benzo.tripsit.me|https://dxm.tripsit.me|https://combo.tripsit.me|https://wiki.tripsit.me|https://tripbot.tripsit.me"} == 0
+    for: 15s
+    labels:
+      severity: critical
+      project: TRIPSIT
+    annotations:
+      title: Node {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 15 seconds. Node seems down.
+  - alert: TRIPSIT.DEV-WEB-INCIDENT
+    expr: probe_success{instance=~"https://mail.tripsit.dev|https://dopamine.tripsit.dev"} == 0
+    for: 1m
+    labels:
+      severity: warning
+      project: TRIPSIT
+    annotations:
+      title: Node {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 1 minute. Node seems down.
diff --git a/Prometheus/alerts/tripsit/node_exporters.yml b/Prometheus/alerts/tripsit/node_exporters.yml
new file mode 100644
index 0000000..ee82367
--- /dev/null
+++ b/Prometheus/alerts/tripsit/node_exporters.yml
@@ -0,0 +1,13 @@
+groups:
+- name: tripsit-node_exporters
+  rules:
+  - alert: TRIPSIT-NODE-INCIDENT
+    expr: up{project="TRIPSIT", job="node_exporters"} == 0
+    for: 30s
+    labels:
+      severity: warning
+      job: node_exporters
+      project: TRIPSIT
+    annotations:
+      title: Node {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance}} for more than 1 minute. Node seems down.
diff --git a/Prometheus/prometheus.service b/Prometheus/prometheus.service
new file mode 100644
index 0000000..38b063f
--- /dev/null
+++ b/Prometheus/prometheus.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Prometheus
+After=network.target
+
+[Service]
+User=prometheus
+Group=prometheus
+Type=simple
+ExecStart=/opt/prometheus/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus --web.console.templates=/opt/prometheus/consoles --web.console.libraries=/opt/prometheus/console_libraries --web.external-url=https://prometheus.lysergic.dev:4433
+ExecReload=/usr/bin/kill -HUP $MAINPID
+
+[Install]
+WantedBy=multi-user.target
diff --git a/Prometheus/prometheus.yml b/Prometheus/prometheus.yml
new file mode 100644
index 0000000..df902f6
--- /dev/null
+++ b/Prometheus/prometheus.yml
@@ -0,0 +1,69 @@
+# my global config
+global:
+  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 1m # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets:
+       - localhost:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  - "/etc/prometheus/alerts/lysergic/*.yml"
+#  - "/etc/prometheus/alerts/tripsit/*.yml" - Disabled on 22/07/2021
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+    - job_name: 'prometheus'
+      static_configs:
+      - targets: ['localhost:9090']
+
+    - job_name: 'node_exporters'
+      file_sd_configs:
+      - files:
+        - '/etc/prometheus/targets/node-*.json'
+#    - job_name: 'node_exporters_02'
+#      file_sd_configs:
+#      - files:
+#        - '/etc/prometheus/targets/node-tripsit.json'
+#      proxy_url: 'https://dopamine.tripsit.dev:9493/'
+      #params:
+      #  _scheme: [https]
+
+    - job_name: 'nginx_exporters'
+      file_sd_configs:
+      - files:
+        - '/etc/prometheus/targets/nginx-*.json'
+
+    - job_name: 'etcd_metrics'
+      static_configs:
+      - targets: ['$ETCDNODE01:2379', '$ETCDNODE02:2379', '$ETCDNODE03:2379']
+      scheme: https
+      tls_config:
+        ca_file: $ETCDCACRT
+        cert_file: $ETCDCLIENTCRT
+        key_file: $ETCDCLIENTKEY
+
+    - job_name: 'blackbox'
+      metrics_path: /probe
+      params:
+        module: [http_2xx]
+      file_sd_configs:
+      - files: ['/etc/prometheus/targets/blackbox*.yml']
+      relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9115
+
+    - job_name: 'wireguard_exporters'
+      file_sd_configs:
+      - files:
+        - '/etc/prometheus/targets/wireguard-*.json'
diff --git a/Prometheus/targets/README.md b/Prometheus/targets/README.md
new file mode 100644
index 0000000..46cd645
--- /dev/null
+++ b/Prometheus/targets/README.md
@@ -0,0 +1 @@
+These configuration files are currently not available publicly.