From 979021f5c40c812d7a9edf22bf8945f691badeed Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Wed, 15 Feb 2023 20:23:49 +0100 Subject: Import Prometheus server configuration * add new roles: - monitoring.prometheus - monitoring.prometheus-alertmanager - monitoring.prometheus-exporter-blackbox * add common Prometheus and Prometheus Alertmanager pillar data * add moni.lysergic.dev specific Prometheus pillar data Signed-off-by: Georg Pfuetzenreuter --- pillar/id/moni_lysergic_dev.sls | 110 +++++++++++++++++++++ pillar/role/monitoring/prometheus-alertmanager.sls | 11 +++ .../monitoring/prometheus-exporter-blackbox.sls | 41 ++++++++ pillar/role/monitoring/prometheus.sls | 11 +++ salt/role/monitoring/prometheus-alertmanager.sls | 2 + .../monitoring/prometheus-exporter-blackbox.sls | 2 + salt/role/monitoring/prometheus.sls | 2 + 7 files changed, 179 insertions(+) create mode 100644 pillar/id/moni_lysergic_dev.sls create mode 100644 pillar/role/monitoring/prometheus-alertmanager.sls create mode 100644 pillar/role/monitoring/prometheus-exporter-blackbox.sls create mode 100644 pillar/role/monitoring/prometheus.sls create mode 100644 salt/role/monitoring/prometheus-alertmanager.sls create mode 100644 salt/role/monitoring/prometheus-exporter-blackbox.sls create mode 100644 salt/role/monitoring/prometheus.sls diff --git a/pillar/id/moni_lysergic_dev.sls b/pillar/id/moni_lysergic_dev.sls new file mode 100644 index 0000000..2607654 --- /dev/null +++ b/pillar/id/moni_lysergic_dev.sls @@ -0,0 +1,110 @@ +prometheus: + pkg: + component: + prometheus: + config: + alerting: + alertmanagers: + - static_configs: + - targets: + - localhost:9093 + + rule_files: + - /etc/prometheus/alerts/lysergic/*.yml + + scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'node_exporters_lysergic' + scrape_timeout: 1m + scrape_interval: 5m + file_sd_configs: + - files: + - '/etc/prometheus/targets/node-lysergic.json' + + - job_name: 'blackbox-2xx' + metrics_path: /probe + params: + module: [http_2xx] + file_sd_configs: + - files: ['/etc/prometheus/targets/blackbox-2xx*.yml'] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9115 + + - job_name: 'blackbox-3xx' + metrics_path: /probe + params: + module: [http_3xx] + file_sd_configs: + - files: ['/etc/prometheus/targets/blackbox-3xx*.yml'] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9115 + + - job_name: 'certificate_exporter' + static_configs: + - targets: ['therapon.rigel.lysergic.dev:9793'] + + alertmanager: + config: + route: + group_by: ['alertname'] + group_wait: 10s + group_interval: 10s + repeat_interval: 1h + receiver: 'smtp-local' + routes: + - receiver: 'lysergic' + # continue: false + match: + project: LYSERGIC + - receiver: 'chillnet' + match: + project: CHILLNET + + receivers: + - name: 'smtp-local' + email_configs: + - to: 'system@lysergic.dev' + from: 'alertmanager@moni.lysergic.dev' + require_tls: false + # !!! TO-DO + smarthost: 'zz0.email:465' + send_resolved: yes + + - name: 'irc-libertacasa' + webhook_configs: + - url: 'http://127.0.0.1:2410/universe' + send_resolved: yes + + - name: 'lysergic' + webhook_configs: + - url: 'http://127.0.0.1:2410/universe' + send_resolved: yes + - url: http://127.0.0.2:8081/prometheus/webhook + send_resolved: yes + email_configs: + - to: 'system@lysergic.dev' + from: 'alertmanager@moni.lysergic.dev' + require_tls: false + smarthost: 'zz0.email:465' + send_resolved: yes + + - name: 'chillnet' + email_configs: + - to: 'team@chillnet.org' + from: 'alertmanager@moni.lysergic.dev' + require_tls: false + smarthost: 'zz0.email:465' + send_resolved: yes diff --git a/pillar/role/monitoring/prometheus-alertmanager.sls b/pillar/role/monitoring/prometheus-alertmanager.sls new file mode 100644 index 0000000..33eb5a4 --- /dev/null +++ b/pillar/role/monitoring/prometheus-alertmanager.sls @@ -0,0 +1,11 @@ +prometheus: + wanted: + component: + - alertmanager + pkg: + component: + alertmanager: + config: + global: + resolve_timeout: 5m + diff --git a/pillar/role/monitoring/prometheus-exporter-blackbox.sls b/pillar/role/monitoring/prometheus-exporter-blackbox.sls new file mode 100644 index 0000000..59b9945 --- /dev/null +++ b/pillar/role/monitoring/prometheus-exporter-blackbox.sls @@ -0,0 +1,41 @@ +prometheus: + wanted: + component: + - blackbox_exporter + pkg: + component: + blackbox_exporter: + config: + modules: + http_2xx: + prober: http + timeout: 15s + http_post_2xx: + prober: http + http: + method: POST + http_3xx: + prober: http + timeout: 5s + http: + method: HEAD + no_follow_redirects: true + valid_status_codes: [301, 302] + tcp_connect: + prober: tcp + ssh_banner: + prober: tcp + tcp: + query_response: + - expect: "^SSH-2.0-" + irc_banner: + prober: tcp + tcp: + query_response: + - send: "NICK prober" + - send: "USER prober prober prober :prober" + - expect: "PING :([^ ]+)" + send: "PONG ${1}" + - expect: "^:[^ ]+ 001" + icmp: + prober: icmp diff --git a/pillar/role/monitoring/prometheus.sls b/pillar/role/monitoring/prometheus.sls new file mode 100644 index 0000000..6cd2235 --- /dev/null +++ b/pillar/role/monitoring/prometheus.sls @@ -0,0 +1,11 @@ +prometheus: + wanted: + component: + - prometheus + pkg: + component: + prometheus: + config: + global: + scrape_interval: 15s + evaluation_interval: 1m diff --git a/salt/role/monitoring/prometheus-alertmanager.sls b/salt/role/monitoring/prometheus-alertmanager.sls new file mode 100644 index 0000000..017d914 --- /dev/null +++ b/salt/role/monitoring/prometheus-alertmanager.sls @@ -0,0 +1,2 @@ +include: + - .prometheus diff --git a/salt/role/monitoring/prometheus-exporter-blackbox.sls b/salt/role/monitoring/prometheus-exporter-blackbox.sls new file mode 100644 index 0000000..017d914 --- /dev/null +++ b/salt/role/monitoring/prometheus-exporter-blackbox.sls @@ -0,0 +1,2 @@ +include: + - .prometheus diff --git a/salt/role/monitoring/prometheus.sls b/salt/role/monitoring/prometheus.sls new file mode 100644 index 0000000..adf0b70 --- /dev/null +++ b/salt/role/monitoring/prometheus.sls @@ -0,0 +1,2 @@ +include: + - prometheus.config -- cgit v1.2.3 From 2bafbeedd72e4d3c02da56cf361f98e50382ce13 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Wed, 15 Feb 2023 20:59:46 +0100 Subject: Manage Prometheus targets Signed-off-by: Georg Pfuetzenreuter --- salt/profile/prometheus/targets.sls | 18 ++++++++++++++++++ salt/role/monitoring/prometheus-alertmanager.sls | 2 +- salt/role/monitoring/prometheus-exporter-blackbox.sls | 2 +- salt/role/monitoring/prometheus.sls | 1 + 4 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 salt/profile/prometheus/targets.sls diff --git a/salt/profile/prometheus/targets.sls b/salt/profile/prometheus/targets.sls new file mode 100644 index 0000000..5f29e73 --- /dev/null +++ b/salt/profile/prometheus/targets.sls @@ -0,0 +1,18 @@ +{%- set mypillar = salt['pillar.get']('profile:prometheus:targets') %} +{%- set targetsdir = '/etc/prometheus/targets' %} + +{%- if mypillar | length %} +{{ targetsdir }}: + file.directory: + - group: prometheus + +{%- for group, nodes in mypillar.items() %} +{{ targetsdir }}/{{ group }}.json: + file.serialize: + - dataset: {{ nodes }} + - serializer: json +{%- endfor %} + +{%- else %} +{%- do salt.log.debug('profile.prometheus: no targets defined') %} +{%- endif %} diff --git a/salt/role/monitoring/prometheus-alertmanager.sls b/salt/role/monitoring/prometheus-alertmanager.sls index 017d914..adf0b70 100644 --- a/salt/role/monitoring/prometheus-alertmanager.sls +++ b/salt/role/monitoring/prometheus-alertmanager.sls @@ -1,2 +1,2 @@ include: - - .prometheus + - prometheus.config diff --git a/salt/role/monitoring/prometheus-exporter-blackbox.sls b/salt/role/monitoring/prometheus-exporter-blackbox.sls index 017d914..adf0b70 100644 --- a/salt/role/monitoring/prometheus-exporter-blackbox.sls +++ b/salt/role/monitoring/prometheus-exporter-blackbox.sls @@ -1,2 +1,2 @@ include: - - .prometheus + - prometheus.config diff --git a/salt/role/monitoring/prometheus.sls b/salt/role/monitoring/prometheus.sls index adf0b70..9a96176 100644 --- a/salt/role/monitoring/prometheus.sls +++ b/salt/role/monitoring/prometheus.sls @@ -1,2 +1,3 @@ include: - prometheus.config + - profile.prometheus.targets -- cgit v1.2.3 From 8016f86164450f4eda4f1488f48cf44de0d4655f Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Wed, 15 Feb 2023 21:02:04 +0100 Subject: p.node_exporter->p.prometheus.node_exporter Since the last commit introduced a new Prometheus targets profile, it makes sense to move node_exporter underneath the Prometheus tree as well. Signed-off-by: Georg Pfuetzenreuter --- salt/common/suse.sls | 2 +- salt/profile/node_exporter/init.sls | 36 ------------------------------- salt/profile/prometheus/node_exporter.sls | 36 +++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 37 deletions(-) delete mode 100644 salt/profile/node_exporter/init.sls create mode 100644 salt/profile/prometheus/node_exporter.sls diff --git a/salt/common/suse.sls b/salt/common/suse.sls index e260c77..764517e 100644 --- a/salt/common/suse.sls +++ b/salt/common/suse.sls @@ -2,7 +2,7 @@ include: - firewalld - profile.seccheck - profile.zypp - - profile.node_exporter + - profile.prometheus.node_exporter - users - .ssh - postfix.config diff --git a/salt/profile/node_exporter/init.sls b/salt/profile/node_exporter/init.sls deleted file mode 100644 index 1e46b3d..0000000 --- a/salt/profile/node_exporter/init.sls +++ /dev/null @@ -1,36 +0,0 @@ -{%- set header = salt['pillar.get']('managed_header_pound') -%} -{%- set sysconfig = '/etc/sysconfig/prometheus-node_exporter' -%} - -node_exporter_packages: - pkg.installed: - - pkgs: - - golang-github-prometheus-node_exporter - -node_exporter_sysconfig_header: - file.prepend: - - name: {{ sysconfig }} - - text: '{{ header }}' - - require: - - pkg: node_exporter_packages - -node_exporter_sysconfig: - file.replace: - - name: {{ sysconfig }} - - pattern: | - ^ARGS=.*$ - - repl: | - ARGS="--web.listen-address=:9200 --collector.filesystem.fs-types-exclude='^(fuse.s3fs|fuse.cryfs|tmpfscgroup2?|debugfs|devpts|devtmpfs|fusectl|overlay|proc|procfs|pstore)\$' --no-collector.zfs --no-collector.thermal_zone --no-collector.powersupplyclass" - - require: - - pkg: node_exporter_packages - - file: node_exporter_sysconfig_header - -node_exporter_service: - service.running: - - name: prometheus-node_exporter.service - - enable: True - - full_restart: True - - require: - - pkg: node_exporter_packages - - file: node_exporter_sysconfig - - watch: - - file: node_exporter_sysconfig diff --git a/salt/profile/prometheus/node_exporter.sls b/salt/profile/prometheus/node_exporter.sls new file mode 100644 index 0000000..1e46b3d --- /dev/null +++ b/salt/profile/prometheus/node_exporter.sls @@ -0,0 +1,36 @@ +{%- set header = salt['pillar.get']('managed_header_pound') -%} +{%- set sysconfig = '/etc/sysconfig/prometheus-node_exporter' -%} + +node_exporter_packages: + pkg.installed: + - pkgs: + - golang-github-prometheus-node_exporter + +node_exporter_sysconfig_header: + file.prepend: + - name: {{ sysconfig }} + - text: '{{ header }}' + - require: + - pkg: node_exporter_packages + +node_exporter_sysconfig: + file.replace: + - name: {{ sysconfig }} + - pattern: | + ^ARGS=.*$ + - repl: | + ARGS="--web.listen-address=:9200 --collector.filesystem.fs-types-exclude='^(fuse.s3fs|fuse.cryfs|tmpfscgroup2?|debugfs|devpts|devtmpfs|fusectl|overlay|proc|procfs|pstore)\$' --no-collector.zfs --no-collector.thermal_zone --no-collector.powersupplyclass" + - require: + - pkg: node_exporter_packages + - file: node_exporter_sysconfig_header + +node_exporter_service: + service.running: + - name: prometheus-node_exporter.service + - enable: True + - full_restart: True + - require: + - pkg: node_exporter_packages + - file: node_exporter_sysconfig + - watch: + - file: node_exporter_sysconfig -- cgit v1.2.3 From cade9c0aca9db3b938ff7343a29756083b252147 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Wed, 15 Feb 2023 21:13:23 +0100 Subject: Moni: Read Blackbox targets as JSON Use uniform JSON target files instead of a JSON/YAML mix. Signed-off-by: Georg Pfuetzenreuter --- pillar/id/moni_lysergic_dev.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pillar/id/moni_lysergic_dev.sls b/pillar/id/moni_lysergic_dev.sls index 2607654..2ebf296 100644 --- a/pillar/id/moni_lysergic_dev.sls +++ b/pillar/id/moni_lysergic_dev.sls @@ -29,7 +29,7 @@ prometheus: params: module: [http_2xx] file_sd_configs: - - files: ['/etc/prometheus/targets/blackbox-2xx*.yml'] + - files: ['/etc/prometheus/targets/blackbox-2xx*.json'] relabel_configs: - source_labels: [__address__] target_label: __param_target @@ -43,7 +43,7 @@ prometheus: params: module: [http_3xx] file_sd_configs: - - files: ['/etc/prometheus/targets/blackbox-3xx*.yml'] + - files: ['/etc/prometheus/targets/blackbox-3xx*.json'] relabel_configs: - source_labels: [__address__] target_label: __param_target -- cgit v1.2.3 From 0730cbb4c20c5d3e32cc2217530d4e27574bc7c7 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Thu, 16 Feb 2023 01:31:57 +0100 Subject: Manage Prometheus firewall rules Signed-off-by: Georg Pfuetzenreuter --- pillar/role/monitoring/prometheus-exporter-blackbox.sls | 9 +++++++++ pillar/role/monitoring/prometheus.sls | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/pillar/role/monitoring/prometheus-exporter-blackbox.sls b/pillar/role/monitoring/prometheus-exporter-blackbox.sls index 59b9945..3e9ab08 100644 --- a/pillar/role/monitoring/prometheus-exporter-blackbox.sls +++ b/pillar/role/monitoring/prometheus-exporter-blackbox.sls @@ -39,3 +39,12 @@ prometheus: - expect: "^:[^ ]+ 001" icmp: prober: icmp + +firewalld: + zones: + internal: + ports: + - comment: 'Prometheus Blackbox Exporter' + port: 9115 + protocol: tcp + diff --git a/pillar/role/monitoring/prometheus.sls b/pillar/role/monitoring/prometheus.sls index 6cd2235..c9c2b01 100644 --- a/pillar/role/monitoring/prometheus.sls +++ b/pillar/role/monitoring/prometheus.sls @@ -9,3 +9,9 @@ prometheus: global: scrape_interval: 15s evaluation_interval: 1m + +firewalld: + zones: + internal: + services: + - prometheus -- cgit v1.2.3 From ddb72f1cb3e3b6b4dbe11ff36510bf15e226b9aa Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Tue, 21 Feb 2023 19:06:56 +0100 Subject: Disable commit linting Temporary change until imports with existing messages are finished. Signed-off-by: Georg Pfuetzenreuter --- .pipeline.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.pipeline.yml b/.pipeline.yml index cdbafd4..6df315a 100644 --- a/.pipeline.yml +++ b/.pipeline.yml @@ -3,15 +3,15 @@ skip_clone: true pipeline: - commit_lint: - image: registry.opensuse.org/home/crameleon/libertacasa/containers/containerfile/libertacasa/pipeline-gommit:latest - secrets: [ci_netrc_username, ci_netrc_password, ci_netrc_machine] - when: - event: [push] - commands: - - git clone --single-branch -b $CI_COMMIT_BRANCH $CI_REPO_LINK ../salt-libertacasa-commit-linting - - cd ../salt-libertacasa-commit-linting - - bin/lint-commits.pl production + # commit_lint: + # image: registry.opensuse.org/home/crameleon/libertacasa/containers/containerfile/libertacasa/pipeline-gommit:latest + # secrets: [ci_netrc_username, ci_netrc_password, ci_netrc_machine] + # when: + # event: [push] + # commands: + # - git clone --single-branch -b $CI_COMMIT_BRANCH $CI_REPO_LINK ../salt-libertacasa-commit-linting + # - cd ../salt-libertacasa-commit-linting + # - bin/lint-commits.pl production code_lint: image: registry.opensuse.org/home/crameleon/libertacasa/containers/containerfile/libertacasa/pipeline-lint:latest -- cgit v1.2.3