Import Prometheus server configuration

* add new roles:
  - monitoring.prometheus
  - monitoring.prometheus-alertmanager
  - monitoring.prometheus-exporter-blackbox
* add common Prometheus and Prometheus Alertmanager pillar data
* add moni.lysergic.dev specific Prometheus pillar data

Signed-off-by: Georg Pfuetzenreuter <mail@georg-pfuetzenreuter.net>

4 files changed, 173 insertions, 0 deletions

diff --git a/pillar/id/moni_lysergic_dev.sls b/pillar/id/moni_lysergic_dev.sls
new file mode 100644
index 0000000..2607654
--- /dev/null
+++ b/pillar/id/moni_lysergic_dev.sls
@@ -0,0 +1,110 @@
+prometheus:
+  pkg:
+    component:
+      prometheus:
+        config:
+          alerting:
+            alertmanagers:
+              - static_configs:
+                - targets:
+                  - localhost:9093
+
+          rule_files:
+            - /etc/prometheus/alerts/lysergic/*.yml
+
+          scrape_configs:
+            - job_name: 'prometheus'
+              static_configs:
+              - targets: ['localhost:9090']
+
+            - job_name: 'node_exporters_lysergic'
+              scrape_timeout: 1m
+              scrape_interval: 5m
+              file_sd_configs:
+              - files:
+                - '/etc/prometheus/targets/node-lysergic.json'
+
+            - job_name: 'blackbox-2xx'
+              metrics_path: /probe
+              params:
+                module: [http_2xx]
+              file_sd_configs:
+              - files: ['/etc/prometheus/targets/blackbox-2xx*.yml']
+              relabel_configs:
+              - source_labels: [__address__]
+                target_label: __param_target
+              - source_labels: [__param_target]
+                target_label: instance
+              - target_label: __address__
+                replacement: 127.0.0.1:9115
+
+            - job_name: 'blackbox-3xx'
+              metrics_path: /probe
+              params:
+                module: [http_3xx]
+              file_sd_configs:
+              - files: ['/etc/prometheus/targets/blackbox-3xx*.yml']
+              relabel_configs:
+              - source_labels: [__address__]
+                target_label: __param_target
+              - source_labels: [__param_target]
+                target_label: instance
+              - target_label: __address__
+                replacement: 127.0.0.1:9115
+
+            - job_name: 'certificate_exporter'
+              static_configs:
+              - targets: ['therapon.rigel.lysergic.dev:9793']
+
+      alertmanager:
+        config:
+          route:
+            group_by: ['alertname']
+            group_wait: 10s
+            group_interval: 10s
+            repeat_interval: 1h
+            receiver: 'smtp-local'
+            routes:
+            - receiver: 'lysergic'
+          #    continue: false
+              match:
+               project: LYSERGIC
+            - receiver: 'chillnet'
+              match:
+               project: CHILLNET
+
+          receivers:
+          - name: 'smtp-local'
+            email_configs:
+            - to: 'system@lysergic.dev'
+              from: 'alertmanager@moni.lysergic.dev'
+              require_tls: false
+          # !!! TO-DO
+              smarthost: 'zz0.email:465'
+              send_resolved: yes
+
+          - name: 'irc-libertacasa'
+            webhook_configs:
+            - url: 'http://127.0.0.1:2410/universe'
+              send_resolved: yes
+
+          - name: 'lysergic'
+            webhook_configs:
+            - url: 'http://127.0.0.1:2410/universe'
+              send_resolved: yes
+            - url: http://127.0.0.2:8081/prometheus/webhook
+              send_resolved: yes
+            email_configs:
+            - to: 'system@lysergic.dev'
+              from: 'alertmanager@moni.lysergic.dev'
+              require_tls: false
+              smarthost: 'zz0.email:465'
+              send_resolved: yes
+
+          - name: 'chillnet'
+            email_configs:
+            - to: 'team@chillnet.org'
+              from: 'alertmanager@moni.lysergic.dev'
+              require_tls: false
+              smarthost: 'zz0.email:465'
+              send_resolved: yes
diff --git a/pillar/role/monitoring/prometheus-alertmanager.sls b/pillar/role/monitoring/prometheus-alertmanager.sls
new file mode 100644
index 0000000..33eb5a4
--- /dev/null
+++ b/pillar/role/monitoring/prometheus-alertmanager.sls
@@ -0,0 +1,11 @@
+prometheus:
+  wanted:
+    component:
+      - alertmanager
+  pkg:
+    component:
+      alertmanager:
+        config:
+          global:
+            resolve_timeout: 5m
+
diff --git a/pillar/role/monitoring/prometheus-exporter-blackbox.sls b/pillar/role/monitoring/prometheus-exporter-blackbox.sls
new file mode 100644
index 0000000..59b9945
--- /dev/null
+++ b/pillar/role/monitoring/prometheus-exporter-blackbox.sls
@@ -0,0 +1,41 @@
+prometheus:
+  wanted:
+    component:
+      - blackbox_exporter
+  pkg:
+    component:
+      blackbox_exporter:
+        config:
+          modules:
+            http_2xx:
+              prober: http
+              timeout: 15s
+            http_post_2xx:
+              prober: http
+              http:
+                method: POST
+            http_3xx:
+              prober: http
+              timeout: 5s
+              http:
+                method: HEAD
+                no_follow_redirects: true
+                valid_status_codes: [301, 302]
+            tcp_connect:
+              prober: tcp
+            ssh_banner:
+              prober: tcp
+              tcp:
+                query_response:
+                - expect: "^SSH-2.0-"
+            irc_banner:
+              prober: tcp
+              tcp:
+                query_response:
+                - send: "NICK prober"
+                - send: "USER prober prober prober :prober"
+                - expect: "PING :([^ ]+)"
+                  send: "PONG ${1}"
+                - expect: "^:[^ ]+ 001"
+            icmp:
+              prober: icmp
diff --git a/pillar/role/monitoring/prometheus.sls b/pillar/role/monitoring/prometheus.sls
new file mode 100644
index 0000000..6cd2235
--- /dev/null
+++ b/pillar/role/monitoring/prometheus.sls
@@ -0,0 +1,11 @@
+prometheus:
+  wanted:
+    component:
+      - prometheus
+  pkg:
+    component:
+      prometheus:
+        config:
+          global:
+            scrape_interval: 15s
+            evaluation_interval: 1m