diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 824ae2f0a46f..9e21689604a4 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1115,7 +1115,7 @@ in private-gpt = handleTest ./private-gpt.nix { }; privatebin = runTest ./privatebin.nix; privoxy = handleTest ./privoxy.nix { }; - prometheus = handleTest ./prometheus { }; + prometheus = import ./prometheus { inherit runTest; }; prometheus-exporters = handleTest ./prometheus-exporters.nix { }; prosody = handleTest ./xmpp/prosody.nix { }; prosody-mysql = handleTest ./xmpp/prosody-mysql.nix { }; diff --git a/nixos/tests/prometheus/alertmanager.nix b/nixos/tests/prometheus/alertmanager.nix index 51151cbab534..794bbb8aab90 100644 --- a/nixos/tests/prometheus/alertmanager.nix +++ b/nixos/tests/prometheus/alertmanager.nix @@ -1,160 +1,146 @@ -import ../make-test-python.nix ( - { lib, pkgs, ... }: +{ pkgs, ... }: - { - name = "prometheus-alertmanager"; +{ + name = "prometheus-alertmanager"; - nodes = { - prometheus = - { config, pkgs, ... }: - { - environment.systemPackages = [ pkgs.jq ]; + nodes = { + prometheus = + { config, pkgs, ... }: + { + environment.systemPackages = [ pkgs.jq ]; - networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; - services.prometheus = { - enable = true; - globalConfig.scrape_interval = "2s"; + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "2s"; - alertmanagers = [ - { - scheme = "http"; - static_configs = [ - { - targets = [ - "alertmanager:${toString config.services.prometheus.alertmanager.port}" - ]; - } - ]; - } - ]; - - rules = [ - '' - groups: - - name: test - rules: - - alert: InstanceDown - expr: up == 0 - for: 5s - labels: - severity: page - annotations: - summary: "Instance {{ $labels.instance }} down" - '' - ]; - - scrapeConfigs = [ - { - job_name = "alertmanager"; - static_configs = [ - { - targets = [ - "alertmanager:${toString config.services.prometheus.alertmanager.port}" - ]; - } - ]; - } - { - job_name = "node"; - static_configs = [ - { - targets = [ - "node:${toString config.services.prometheus.exporters.node.port}" - ]; - } - ]; - } - ]; - }; - }; - - alertmanager = - { config, pkgs, ... }: - { - services.prometheus.alertmanager = { - enable = true; - openFirewall = true; - - configuration = { - global = { - resolve_timeout = "1m"; - }; - - route = { - # Root route node - receiver = "test"; - group_by = [ "..." ]; - continue = false; - group_wait = "1s"; - group_interval = "15s"; - repeat_interval = "24h"; - }; - - receivers = [ - { - name = "test"; - webhook_configs = [ - { - url = "http://logger:6725"; - send_resolved = true; - max_alerts = 0; - } - ]; - } + alertmanagers = [ + { + scheme = "http"; + static_configs = [ + { targets = [ "alertmanager:${toString config.services.prometheus.alertmanager.port}" ]; } ]; + } + ]; + + rules = [ + '' + groups: + - name: test + rules: + - alert: InstanceDown + expr: up == 0 + for: 5s + labels: + severity: page + annotations: + summary: "Instance {{ $labels.instance }} down" + '' + ]; + + scrapeConfigs = [ + { + job_name = "alertmanager"; + static_configs = [ + { targets = [ "alertmanager:${toString config.services.prometheus.alertmanager.port}" ]; } + ]; + } + { + job_name = "node"; + static_configs = [ + { targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ]; } + ]; + } + ]; + }; + }; + + alertmanager = + { config, pkgs, ... }: + { + services.prometheus.alertmanager = { + enable = true; + openFirewall = true; + + configuration = { + global = { + resolve_timeout = "1m"; }; + + route = { + # Root route node + receiver = "test"; + group_by = [ "..." ]; + continue = false; + group_wait = "1s"; + group_interval = "15s"; + repeat_interval = "24h"; + }; + + receivers = [ + { + name = "test"; + webhook_configs = [ + { + url = "http://logger:6725"; + send_resolved = true; + max_alerts = 0; + } + ]; + } + ]; }; }; + }; - logger = - { config, pkgs, ... }: - { - networking.firewall.allowedTCPPorts = [ 6725 ]; + logger = + { config, pkgs, ... }: + { + networking.firewall.allowedTCPPorts = [ 6725 ]; - services.prometheus.alertmanagerWebhookLogger.enable = true; - }; - }; + services.prometheus.alertmanagerWebhookLogger.enable = true; + }; + }; - testScript = '' - alertmanager.wait_for_unit("alertmanager") - alertmanager.wait_for_open_port(9093) - alertmanager.wait_until_succeeds("curl -s http://127.0.0.1:9093/-/ready") - #alertmanager.wait_until_succeeds("journalctl -o cat -u alertmanager.service | grep 'version=${pkgs.prometheus-alertmanager.version}'") + testScript = '' + alertmanager.wait_for_unit("alertmanager") + alertmanager.wait_for_open_port(9093) + alertmanager.wait_until_succeeds("curl -s http://127.0.0.1:9093/-/ready") + #alertmanager.wait_until_succeeds("journalctl -o cat -u alertmanager.service | grep 'version=${pkgs.prometheus-alertmanager.version}'") - logger.wait_for_unit("alertmanager-webhook-logger") - logger.wait_for_open_port(6725) + logger.wait_for_unit("alertmanager-webhook-logger") + logger.wait_for_open_port(6725) - prometheus.wait_for_unit("prometheus") - prometheus.wait_for_open_port(9090) + prometheus.wait_for_unit("prometheus") + prometheus.wait_for_open_port(9090) - prometheus.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"alertmanager\"\}==1)' | " - + "jq '.data.result[0].value[1]' | grep '\"1\"'" - ) + prometheus.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"alertmanager\"\}==1)' | " + + "jq '.data.result[0].value[1]' | grep '\"1\"'" + ) - prometheus.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(alertmanager_build_info)%20by%20(version)' | " - + "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-alertmanager.version}\"'" - ) + prometheus.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(alertmanager_build_info)%20by%20(version)' | " + + "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-alertmanager.version}\"'" + ) - prometheus.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\}!=1)' | " - + "jq '.data.result[0].value[1]' | grep '\"1\"'" - ) + prometheus.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\}!=1)' | " + + "jq '.data.result[0].value[1]' | grep '\"1\"'" + ) - prometheus.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=alertmanager_notifications_total\{integration=\"webhook\"\}' | " - + "jq '.data.result[0].value[1]' | grep -v '\"0\"'" - ) + prometheus.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=alertmanager_notifications_total\{integration=\"webhook\"\}' | " + + "jq '.data.result[0].value[1]' | grep -v '\"0\"'" + ) - logger.wait_until_succeeds( - "journalctl -o cat -u alertmanager-webhook-logger.service | grep '\"alertname\":\"InstanceDown\"'" - ) + logger.wait_until_succeeds( + "journalctl -o cat -u alertmanager-webhook-logger.service | grep '\"alertname\":\"InstanceDown\"'" + ) - logger.log(logger.succeed("systemd-analyze security alertmanager-webhook-logger.service | grep -v '✓'")) + logger.log(logger.succeed("systemd-analyze security alertmanager-webhook-logger.service | grep -v '✓'")) - alertmanager.log(alertmanager.succeed("systemd-analyze security alertmanager.service | grep -v '✓'")) - ''; - } -) + alertmanager.log(alertmanager.succeed("systemd-analyze security alertmanager.service | grep -v '✓'")) + ''; +} diff --git a/nixos/tests/prometheus/config-reload.nix b/nixos/tests/prometheus/config-reload.nix index 60f5bc6a1d23..bb20d249fe9c 100644 --- a/nixos/tests/prometheus/config-reload.nix +++ b/nixos/tests/prometheus/config-reload.nix @@ -1,120 +1,108 @@ -import ../make-test-python.nix ( - { lib, pkgs, ... }: +{ + name = "prometheus-config-reload"; - { - name = "prometheus-config-reload"; + nodes = { + prometheus = + { config, pkgs, ... }: + { + environment.systemPackages = [ pkgs.jq ]; - nodes = { - prometheus = - { config, pkgs, ... }: - { - environment.systemPackages = [ pkgs.jq ]; + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; - networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + services.prometheus = { + enable = true; + enableReload = true; + globalConfig.scrape_interval = "2s"; + scrapeConfigs = [ + { + job_name = "prometheus"; + static_configs = [ { targets = [ "prometheus:${toString config.services.prometheus.port}" ]; } ]; + } + ]; + }; - services.prometheus = { - enable = true; - enableReload = true; - globalConfig.scrape_interval = "2s"; - scrapeConfigs = [ - { - job_name = "prometheus"; - static_configs = [ + specialisation = { + "prometheus-config-change" = { + configuration = { + environment.systemPackages = [ pkgs.yq ]; + + # This configuration just adds a new prometheus job + # to scrape the node_exporter metrics of the s3 machine. + services.prometheus = { + scrapeConfigs = [ { - targets = [ - "prometheus:${toString config.services.prometheus.port}" + job_name = "node"; + static_configs = [ + { targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ]; } ]; } ]; - } - ]; - }; - - specialisation = { - "prometheus-config-change" = { - configuration = { - environment.systemPackages = [ pkgs.yq ]; - - # This configuration just adds a new prometheus job - # to scrape the node_exporter metrics of the s3 machine. - services.prometheus = { - scrapeConfigs = [ - { - job_name = "node"; - static_configs = [ - { - targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ]; - } - ]; - } - ]; - }; }; }; }; }; - }; + }; + }; - testScript = '' - prometheus.wait_for_unit("prometheus") - prometheus.wait_for_open_port(9090) + testScript = '' + prometheus.wait_for_unit("prometheus") + prometheus.wait_for_open_port(9090) - # Check if switching to a NixOS configuration that changes the prometheus - # configuration reloads (instead of restarts) prometheus before the switch - # finishes successfully: - with subtest("config change reloads prometheus"): - import json - # We check if prometheus has finished reloading by looking for the message - # "Completed loading of configuration file" in the journal between the start - # and finish of switching to the new NixOS configuration. - # - # To mark the start we record the journal cursor before starting the switch: - cursor_before_switching = json.loads( - prometheus.succeed("journalctl -n1 -o json --output-fields=__CURSOR") - )["__CURSOR"] + # Check if switching to a NixOS configuration that changes the prometheus + # configuration reloads (instead of restarts) prometheus before the switch + # finishes successfully: + with subtest("config change reloads prometheus"): + import json + # We check if prometheus has finished reloading by looking for the message + # "Completed loading of configuration file" in the journal between the start + # and finish of switching to the new NixOS configuration. + # + # To mark the start we record the journal cursor before starting the switch: + cursor_before_switching = json.loads( + prometheus.succeed("journalctl -n1 -o json --output-fields=__CURSOR") + )["__CURSOR"] - # Now we switch: - prometheus_config_change = prometheus.succeed( - "readlink /run/current-system/specialisation/prometheus-config-change" - ).strip() - prometheus.succeed(prometheus_config_change + "/bin/switch-to-configuration test") + # Now we switch: + prometheus_config_change = prometheus.succeed( + "readlink /run/current-system/specialisation/prometheus-config-change" + ).strip() + prometheus.succeed(prometheus_config_change + "/bin/switch-to-configuration test") - # Next we retrieve all logs since the start of switching: - logs_after_starting_switching = prometheus.succeed( - """ - journalctl --after-cursor='{cursor_before_switching}' -o json --output-fields=MESSAGE - """.format( - cursor_before_switching=cursor_before_switching - ) - ) - - # Finally we check if the message "Completed loading of configuration file" - # occurs before the "finished switching to system configuration" message: - finished_switching_msg = ( - "finished switching to system configuration " + prometheus_config_change - ) - reloaded_before_switching_finished = False - finished_switching = False - for log_line in logs_after_starting_switching.split("\n"): - msg = json.loads(log_line)["MESSAGE"] - if "Completed loading of configuration file" in msg: - reloaded_before_switching_finished = True - if msg == finished_switching_msg: - finished_switching = True - break - - assert reloaded_before_switching_finished - assert finished_switching - - # Check if the reloaded config includes the new node job: - prometheus.succeed( + # Next we retrieve all logs since the start of switching: + logs_after_starting_switching = prometheus.succeed( """ - curl -sf http://127.0.0.1:9090/api/v1/status/config \ - | jq -r .data.yaml \ - | yq '.scrape_configs | any(.job_name == "node")' \ - | grep true - """ - ) - ''; - } -) + journalctl --after-cursor='{cursor_before_switching}' -o json --output-fields=MESSAGE + """.format( + cursor_before_switching=cursor_before_switching + ) + ) + + # Finally we check if the message "Completed loading of configuration file" + # occurs before the "finished switching to system configuration" message: + finished_switching_msg = ( + "finished switching to system configuration " + prometheus_config_change + ) + reloaded_before_switching_finished = False + finished_switching = False + for log_line in logs_after_starting_switching.split("\n"): + msg = json.loads(log_line)["MESSAGE"] + if "Completed loading of configuration file" in msg: + reloaded_before_switching_finished = True + if msg == finished_switching_msg: + finished_switching = True + break + + assert reloaded_before_switching_finished + assert finished_switching + + # Check if the reloaded config includes the new node job: + prometheus.succeed( + """ + curl -sf http://127.0.0.1:9090/api/v1/status/config \ + | jq -r .data.yaml \ + | yq '.scrape_configs | any(.job_name == "node")' \ + | grep true + """ + ) + ''; +} diff --git a/nixos/tests/prometheus/default.nix b/nixos/tests/prometheus/default.nix index ea6c61c85b80..a06f708f184e 100644 --- a/nixos/tests/prometheus/default.nix +++ b/nixos/tests/prometheus/default.nix @@ -1,14 +1,10 @@ -{ - system ? builtins.currentSystem, - config ? { }, - pkgs ? import ../../.. { inherit system config; }, -}: +{ runTest }: { - alertmanager = import ./alertmanager.nix { inherit system pkgs; }; - config-reload = import ./config-reload.nix { inherit system pkgs; }; - federation = import ./federation.nix { inherit system pkgs; }; - prometheus-pair = import ./prometheus-pair.nix { inherit system pkgs; }; - pushgateway = import ./pushgateway.nix { inherit system pkgs; }; - remote-write = import ./remote-write.nix { inherit system pkgs; }; + alertmanager = runTest ./alertmanager.nix; + config-reload = runTest ./config-reload.nix; + federation = runTest ./federation.nix; + prometheus-pair = runTest ./prometheus-pair.nix; + pushgateway = runTest ./pushgateway.nix; + remote-write = runTest ./remote-write.nix; } diff --git a/nixos/tests/prometheus/federation.nix b/nixos/tests/prometheus/federation.nix index 15f84b2d88af..e81418dfba93 100644 --- a/nixos/tests/prometheus/federation.nix +++ b/nixos/tests/prometheus/federation.nix @@ -1,227 +1,203 @@ -import ../make-test-python.nix ( - { lib, pkgs, ... }: +{ + name = "prometheus-federation"; - { - name = "prometheus-federation"; + nodes = { + global1 = + { config, pkgs, ... }: + { + environment.systemPackages = [ pkgs.jq ]; - nodes = { - global1 = - { config, pkgs, ... }: - { - environment.systemPackages = [ pkgs.jq ]; + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; - networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "2s"; - services.prometheus = { - enable = true; - globalConfig.scrape_interval = "2s"; + scrapeConfigs = [ + { + job_name = "federate"; + honor_labels = true; + metrics_path = "/federate"; - scrapeConfigs = [ - { - job_name = "federate"; - honor_labels = true; - metrics_path = "/federate"; + params = { + "match[]" = [ + "{job=\"node\"}" + "{job=\"prometheus\"}" + ]; + }; - params = { - "match[]" = [ - "{job=\"node\"}" - "{job=\"prometheus\"}" + static_configs = [ + { + targets = [ + "prometheus1:${toString config.services.prometheus.port}" + "prometheus2:${toString config.services.prometheus.port}" ]; - }; - - static_configs = [ - { - targets = [ - "prometheus1:${toString config.services.prometheus.port}" - "prometheus2:${toString config.services.prometheus.port}" - ]; - } - ]; - } - { - job_name = "prometheus"; - static_configs = [ - { - targets = [ - "global1:${toString config.services.prometheus.port}" - "global2:${toString config.services.prometheus.port}" - ]; - } - ]; - } - ]; - }; - }; - - global2 = - { config, pkgs, ... }: - { - environment.systemPackages = [ pkgs.jq ]; - - networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; - - services.prometheus = { - enable = true; - globalConfig.scrape_interval = "2s"; - - scrapeConfigs = [ - { - job_name = "federate"; - honor_labels = true; - metrics_path = "/federate"; - - params = { - "match[]" = [ - "{job=\"node\"}" - "{job=\"prometheus\"}" + } + ]; + } + { + job_name = "prometheus"; + static_configs = [ + { + targets = [ + "global1:${toString config.services.prometheus.port}" + "global2:${toString config.services.prometheus.port}" ]; - }; - - static_configs = [ - { - targets = [ - "prometheus1:${toString config.services.prometheus.port}" - "prometheus2:${toString config.services.prometheus.port}" - ]; - } - ]; - } - { - job_name = "prometheus"; - static_configs = [ - { - targets = [ - "global1:${toString config.services.prometheus.port}" - "global2:${toString config.services.prometheus.port}" - ]; - } - ]; - } - ]; - }; + } + ]; + } + ]; }; + }; - prometheus1 = - { config, pkgs, ... }: - { - environment.systemPackages = [ pkgs.jq ]; + global2 = + { config, pkgs, ... }: + { + environment.systemPackages = [ pkgs.jq ]; - networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; - services.prometheus = { - enable = true; - globalConfig.scrape_interval = "2s"; + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "2s"; - scrapeConfigs = [ - { - job_name = "node"; - static_configs = [ - { - targets = [ - "node1:${toString config.services.prometheus.exporters.node.port}" - ]; - } + scrapeConfigs = [ + { + job_name = "federate"; + honor_labels = true; + metrics_path = "/federate"; + + params = { + "match[]" = [ + "{job=\"node\"}" + "{job=\"prometheus\"}" ]; - } - { - job_name = "prometheus"; - static_configs = [ - { - targets = [ - "prometheus1:${toString config.services.prometheus.port}" - ]; - } - ]; - } - ]; - }; + }; + + static_configs = [ + { + targets = [ + "prometheus1:${toString config.services.prometheus.port}" + "prometheus2:${toString config.services.prometheus.port}" + ]; + } + ]; + } + { + job_name = "prometheus"; + static_configs = [ + { + targets = [ + "global1:${toString config.services.prometheus.port}" + "global2:${toString config.services.prometheus.port}" + ]; + } + ]; + } + ]; }; + }; - prometheus2 = - { config, pkgs, ... }: - { - environment.systemPackages = [ pkgs.jq ]; + prometheus1 = + { config, pkgs, ... }: + { + environment.systemPackages = [ pkgs.jq ]; - networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; - services.prometheus = { - enable = true; - globalConfig.scrape_interval = "2s"; + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "2s"; - scrapeConfigs = [ - { - job_name = "node"; - static_configs = [ - { - targets = [ - "node2:${toString config.services.prometheus.exporters.node.port}" - ]; - } - ]; - } - { - job_name = "prometheus"; - static_configs = [ - { - targets = [ - "prometheus2:${toString config.services.prometheus.port}" - ]; - } - ]; - } - ]; - }; + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [ + { targets = [ "node1:${toString config.services.prometheus.exporters.node.port}" ]; } + ]; + } + { + job_name = "prometheus"; + static_configs = [ { targets = [ "prometheus1:${toString config.services.prometheus.port}" ]; } ]; + } + ]; }; + }; - node1 = - { config, pkgs, ... }: - { - services.prometheus.exporters.node = { - enable = true; - openFirewall = true; - }; + prometheus2 = + { config, pkgs, ... }: + { + environment.systemPackages = [ pkgs.jq ]; + + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "2s"; + + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [ + { targets = [ "node2:${toString config.services.prometheus.exporters.node.port}" ]; } + ]; + } + { + job_name = "prometheus"; + static_configs = [ { targets = [ "prometheus2:${toString config.services.prometheus.port}" ]; } ]; + } + ]; }; + }; - node2 = - { config, pkgs, ... }: - { - services.prometheus.exporters.node = { - enable = true; - openFirewall = true; - }; + node1 = + { config, pkgs, ... }: + { + services.prometheus.exporters.node = { + enable = true; + openFirewall = true; }; - }; + }; - testScript = '' - for machine in node1, node2: - machine.wait_for_unit("prometheus-node-exporter") - machine.wait_for_open_port(9100) + node2 = + { config, pkgs, ... }: + { + services.prometheus.exporters.node = { + enable = true; + openFirewall = true; + }; + }; + }; - for machine in prometheus1, prometheus2, global1, global2: - machine.wait_for_unit("prometheus") - machine.wait_for_open_port(9090) + testScript = '' + for machine in node1, node2: + machine.wait_for_unit("prometheus-node-exporter") + machine.wait_for_open_port(9100) - # Verify both servers got the same data from the exporter - for machine in prometheus1, prometheus2: - machine.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | " - + "jq '.data.result[0].value[1]' | grep '\"1\"'" - ) - machine.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | " - + "jq '.data.result[0].value[1]' | grep '\"1\"'" - ) + for machine in prometheus1, prometheus2, global1, global2: + machine.wait_for_unit("prometheus") + machine.wait_for_open_port(9090) - for machine in global1, global2: - machine.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | " - + "jq '.data.result[0].value[1]' | grep '\"2\"'" - ) + # Verify both servers got the same data from the exporter + for machine in prometheus1, prometheus2: + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | " + + "jq '.data.result[0].value[1]' | grep '\"1\"'" + ) + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | " + + "jq '.data.result[0].value[1]' | grep '\"1\"'" + ) - machine.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | " - + "jq '.data.result[0].value[1]' | grep '\"4\"'" - ) - ''; - } -) + for machine in global1, global2: + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | " + + "jq '.data.result[0].value[1]' | grep '\"2\"'" + ) + + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | " + + "jq '.data.result[0].value[1]' | grep '\"4\"'" + ) + ''; +} diff --git a/nixos/tests/prometheus/prometheus-pair.nix b/nixos/tests/prometheus/prometheus-pair.nix index 98860fa6bf95..6d8d5900480f 100644 --- a/nixos/tests/prometheus/prometheus-pair.nix +++ b/nixos/tests/prometheus/prometheus-pair.nix @@ -1,93 +1,91 @@ -import ../make-test-python.nix ( - { lib, pkgs, ... }: +{ pkgs, ... }: - { - name = "prometheus-pair"; +{ + name = "prometheus-pair"; - nodes = { - prometheus1 = - { config, pkgs, ... }: - { - environment.systemPackages = [ pkgs.jq ]; + nodes = { + prometheus1 = + { config, pkgs, ... }: + { + environment.systemPackages = [ pkgs.jq ]; - networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; - services.prometheus = { - enable = true; - globalConfig.scrape_interval = "2s"; - scrapeConfigs = [ - { - job_name = "prometheus"; - static_configs = [ - { - targets = [ - "prometheus1:${toString config.services.prometheus.port}" - "prometheus2:${toString config.services.prometheus.port}" - ]; - } - ]; - } - ]; - }; + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "2s"; + scrapeConfigs = [ + { + job_name = "prometheus"; + static_configs = [ + { + targets = [ + "prometheus1:${toString config.services.prometheus.port}" + "prometheus2:${toString config.services.prometheus.port}" + ]; + } + ]; + } + ]; }; + }; - prometheus2 = - { config, pkgs, ... }: - { - environment.systemPackages = [ pkgs.jq ]; + prometheus2 = + { config, pkgs, ... }: + { + environment.systemPackages = [ pkgs.jq ]; - networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; - services.prometheus = { - enable = true; - globalConfig.scrape_interval = "2s"; - scrapeConfigs = [ - { - job_name = "prometheus"; - static_configs = [ - { - targets = [ - "prometheus1:${toString config.services.prometheus.port}" - "prometheus2:${toString config.services.prometheus.port}" - ]; - } - ]; - } - ]; - }; + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "2s"; + scrapeConfigs = [ + { + job_name = "prometheus"; + static_configs = [ + { + targets = [ + "prometheus1:${toString config.services.prometheus.port}" + "prometheus2:${toString config.services.prometheus.port}" + ]; + } + ]; + } + ]; }; - }; + }; + }; - testScript = '' - for machine in prometheus1, prometheus2: - machine.wait_for_unit("prometheus") - machine.wait_for_open_port(9090) - machine.wait_until_succeeds("journalctl -o cat -u prometheus.service | grep 'version=${pkgs.prometheus.version}'") - machine.wait_until_succeeds("curl -sSf http://localhost:9090/-/healthy") + testScript = '' + for machine in prometheus1, prometheus2: + machine.wait_for_unit("prometheus") + machine.wait_for_open_port(9090) + machine.wait_until_succeeds("journalctl -o cat -u prometheus.service | grep 'version=${pkgs.prometheus.version}'") + machine.wait_until_succeeds("curl -sSf http://localhost:9090/-/healthy") - # Prometheii ready - run some queries - for machine in prometheus1, prometheus2: - machine.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_build_info\{instance=\"prometheus1:9090\",version=\"${pkgs.prometheus.version}\"\}' | " - + "jq '.data.result[0].value[1]' | grep '\"1\"'" - ) + # Prometheii ready - run some queries + for machine in prometheus1, prometheus2: + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_build_info\{instance=\"prometheus1:9090\",version=\"${pkgs.prometheus.version}\"\}' | " + + "jq '.data.result[0].value[1]' | grep '\"1\"'" + ) - machine.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_build_info\{instance=\"prometheus1:9090\"\}' | " - + "jq '.data.result[0].value[1]' | grep '\"1\"'" - ) + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_build_info\{instance=\"prometheus1:9090\"\}' | " + + "jq '.data.result[0].value[1]' | grep '\"1\"'" + ) - machine.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(prometheus_build_info)%20by%20(version)' | " - + "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus.version}\"'" - ) + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(prometheus_build_info)%20by%20(version)' | " + + "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus.version}\"'" + ) - machine.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(prometheus_build_info)%20by%20(version)' | " - + "jq '.data.result[0].value[1]' | grep '\"2\"'" - ) + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(prometheus_build_info)%20by%20(version)' | " + + "jq '.data.result[0].value[1]' | grep '\"2\"'" + ) - prometheus1.log(prometheus1.succeed("systemd-analyze security prometheus.service | grep -v '✓'")) - ''; - } -) + prometheus1.log(prometheus1.succeed("systemd-analyze security prometheus.service | grep -v '✓'")) + ''; +} diff --git a/nixos/tests/prometheus/pushgateway.nix b/nixos/tests/prometheus/pushgateway.nix index fb8b6c091aa1..6b13447085d9 100644 --- a/nixos/tests/prometheus/pushgateway.nix +++ b/nixos/tests/prometheus/pushgateway.nix @@ -1,102 +1,91 @@ -import ../make-test-python.nix ( - { lib, pkgs, ... }: +{ pkgs, ... }: - { - name = "prometheus-pushgateway"; +{ + name = "prometheus-pushgateway"; - nodes = { - prometheus = - { config, pkgs, ... }: - { - environment.systemPackages = [ pkgs.jq ]; + nodes = { + prometheus = + { config, pkgs, ... }: + { + environment.systemPackages = [ pkgs.jq ]; - networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; - services.prometheus = { - enable = true; - globalConfig.scrape_interval = "2s"; + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "2s"; - scrapeConfigs = [ - { - job_name = "pushgateway"; - static_configs = [ - { - targets = [ - "pushgateway:9091" - ]; - } - ]; - } - ]; - }; + scrapeConfigs = [ + { + job_name = "pushgateway"; + static_configs = [ { targets = [ "pushgateway:9091" ]; } ]; + } + ]; }; + }; - pushgateway = - { config, pkgs, ... }: - { - networking.firewall.allowedTCPPorts = [ 9091 ]; + pushgateway = + { config, pkgs, ... }: + { + networking.firewall.allowedTCPPorts = [ 9091 ]; - services.prometheus.pushgateway = { - enable = true; - }; + services.prometheus.pushgateway = { + enable = true; }; + }; - client = - { config, pkgs, ... }: - { - }; - }; + client = { config, pkgs, ... }: { }; + }; - testScript = '' - pushgateway.wait_for_unit("pushgateway") - pushgateway.wait_for_open_port(9091) - pushgateway.wait_until_succeeds("curl -s http://127.0.0.1:9091/-/ready") - pushgateway.wait_until_succeeds("journalctl -o cat -u pushgateway.service | grep 'version=${pkgs.prometheus-pushgateway.version}'") + testScript = '' + pushgateway.wait_for_unit("pushgateway") + pushgateway.wait_for_open_port(9091) + pushgateway.wait_until_succeeds("curl -s http://127.0.0.1:9091/-/ready") + pushgateway.wait_until_succeeds("journalctl -o cat -u pushgateway.service | grep 'version=${pkgs.prometheus-pushgateway.version}'") - prometheus.wait_for_unit("prometheus") - prometheus.wait_for_open_port(9090) + prometheus.wait_for_unit("prometheus") + prometheus.wait_for_open_port(9090) - prometheus.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"pushgateway\"\})' | " - + "jq '.data.result[0].value[1]' | grep '\"1\"'" - ) + prometheus.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"pushgateway\"\})' | " + + "jq '.data.result[0].value[1]' | grep '\"1\"'" + ) - prometheus.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(pushgateway_build_info)%20by%20(version)' | " - + "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-pushgateway.version}\"'" - ) + prometheus.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(pushgateway_build_info)%20by%20(version)' | " + + "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-pushgateway.version}\"'" + ) - # Add a metric and check in Prometheus - client.wait_until_succeeds( - "echo 'some_metric 3.14' | curl --data-binary @- http://pushgateway:9091/metrics/job/some_job" - ) + # Add a metric and check in Prometheus + client.wait_until_succeeds( + "echo 'some_metric 3.14' | curl --data-binary @- http://pushgateway:9091/metrics/job/some_job" + ) - prometheus.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' | " - + "jq '.data.result[0].value[1]' | grep '\"3.14\"'" - ) + prometheus.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' | " + + "jq '.data.result[0].value[1]' | grep '\"3.14\"'" + ) - prometheus.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=absent(some_metric)' | " - + "jq '.data.result[0].value[1]' | grep 'null'" - ) + prometheus.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=absent(some_metric)' | " + + "jq '.data.result[0].value[1]' | grep 'null'" + ) - # Delete the metric, check not in Prometheus - client.wait_until_succeeds( - "curl -X DELETE http://pushgateway:9091/metrics/job/some_job" - ) + # Delete the metric, check not in Prometheus + client.wait_until_succeeds( + "curl -X DELETE http://pushgateway:9091/metrics/job/some_job" + ) - prometheus.wait_until_fails( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' | " - + "jq '.data.result[0].value[1]' | grep '\"3.14\"'" - ) + prometheus.wait_until_fails( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' | " + + "jq '.data.result[0].value[1]' | grep '\"3.14\"'" + ) - prometheus.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=absent(some_metric)' | " - + "jq '.data.result[0].value[1]' | grep '\"1\"'" - ) + prometheus.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=absent(some_metric)' | " + + "jq '.data.result[0].value[1]' | grep '\"1\"'" + ) - pushgateway.log(pushgateway.succeed("systemd-analyze security pushgateway.service | grep -v '✓'")) - ''; - } -) + pushgateway.log(pushgateway.succeed("systemd-analyze security pushgateway.service | grep -v '✓'")) + ''; +} diff --git a/nixos/tests/prometheus/remote-write.nix b/nixos/tests/prometheus/remote-write.nix index 86c14901a6b0..23458c737e1a 100644 --- a/nixos/tests/prometheus/remote-write.nix +++ b/nixos/tests/prometheus/remote-write.nix @@ -1,81 +1,69 @@ -import ../make-test-python.nix ( - { lib, pkgs, ... }: +{ + name = "prometheus-remote-write"; - { - name = "prometheus-remote-write"; + nodes = { + receiver = + { config, pkgs, ... }: + { + environment.systemPackages = [ pkgs.jq ]; - nodes = { - receiver = - { config, pkgs, ... }: - { - environment.systemPackages = [ pkgs.jq ]; + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; - networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "2s"; - services.prometheus = { - enable = true; - globalConfig.scrape_interval = "2s"; - - extraFlags = [ "--web.enable-remote-write-receiver" ]; - }; + extraFlags = [ "--web.enable-remote-write-receiver" ]; }; + }; - prometheus = - { config, pkgs, ... }: - { - environment.systemPackages = [ pkgs.jq ]; + prometheus = + { config, pkgs, ... }: + { + environment.systemPackages = [ pkgs.jq ]; - networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; - services.prometheus = { - enable = true; - globalConfig.scrape_interval = "2s"; + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "2s"; - remoteWrite = [ - { - url = "http://receiver:9090/api/v1/write"; - } - ]; + remoteWrite = [ { url = "http://receiver:9090/api/v1/write"; } ]; - scrapeConfigs = [ - { - job_name = "node"; - static_configs = [ - { - targets = [ - "node:${toString config.services.prometheus.exporters.node.port}" - ]; - } - ]; - } - ]; - }; + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [ + { targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ]; } + ]; + } + ]; }; + }; - node = - { config, pkgs, ... }: - { - services.prometheus.exporters.node = { - enable = true; - openFirewall = true; - }; + node = + { config, pkgs, ... }: + { + services.prometheus.exporters.node = { + enable = true; + openFirewall = true; }; - }; + }; + }; - testScript = '' - node.wait_for_unit("prometheus-node-exporter") - node.wait_for_open_port(9100) + testScript = '' + node.wait_for_unit("prometheus-node-exporter") + node.wait_for_open_port(9100) - for machine in prometheus, receiver: - machine.wait_for_unit("prometheus") - machine.wait_for_open_port(9090) + for machine in prometheus, receiver: + machine.wait_for_unit("prometheus") + machine.wait_for_open_port(9090) - # Verify both servers got the same data from the exporter - for machine in prometheus, receiver: - machine.wait_until_succeeds( - "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=node_exporter_build_info\{instance=\"node:9100\"\}' | " - + "jq '.data.result[0].value[1]' | grep '\"1\"'" - ) - ''; - } -) + # Verify both servers got the same data from the exporter + for machine in prometheus, receiver: + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=node_exporter_build_info\{instance=\"node:9100\"\}' | " + + "jq '.data.result[0].value[1]' | grep '\"1\"'" + ) + ''; +}