nixos/tests/prometheus: migrate to runTest

nixos/tests/all-tests.nix

@@ -1115,7 +1115,7 @@ in
   private-gpt = handleTest ./private-gpt.nix { };
   privatebin = runTest ./privatebin.nix;
   privoxy = handleTest ./privoxy.nix { };
-  prometheus = handleTest ./prometheus { };
+  prometheus = import ./prometheus { inherit runTest; };
   prometheus-exporters = handleTest ./prometheus-exporters.nix { };
   prosody = handleTest ./xmpp/prosody.nix { };
   prosody-mysql = handleTest ./xmpp/prosody-mysql.nix { };

nixos/tests/prometheus/alertmanager.nix

@@ -1,160 +1,146 @@
-import ../make-test-python.nix (
-  { lib, pkgs, ... }:
+{ pkgs, ... }:
 
-  {
-    name = "prometheus-alertmanager";
+{
+  name = "prometheus-alertmanager";
 
-    nodes = {
-      prometheus =
-        { config, pkgs, ... }:
-        {
-          environment.systemPackages = [ pkgs.jq ];
+  nodes = {
+    prometheus =
+      { config, pkgs, ... }:
+      {
+        environment.systemPackages = [ pkgs.jq ];
 
-          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
+        networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
 
-          services.prometheus = {
-            enable = true;
-            globalConfig.scrape_interval = "2s";
+        services.prometheus = {
+          enable = true;
+          globalConfig.scrape_interval = "2s";
 
-            alertmanagers = [
-              {
-                scheme = "http";
-                static_configs = [
-                  {
-                    targets = [
-                      "alertmanager:${toString config.services.prometheus.alertmanager.port}"
-                    ];
-                  }
-                ];
-              }
-            ];
-
-            rules = [
-              ''
-                groups:
-                  - name: test
-                    rules:
-                      - alert: InstanceDown
-                        expr: up == 0
-                        for: 5s
-                        labels:
-                          severity: page
-                        annotations:
-                          summary: "Instance {{ $labels.instance }} down"
-              ''
-            ];
-
-            scrapeConfigs = [
-              {
-                job_name = "alertmanager";
-                static_configs = [
-                  {
-                    targets = [
-                      "alertmanager:${toString config.services.prometheus.alertmanager.port}"
-                    ];
-                  }
-                ];
-              }
-              {
-                job_name = "node";
-                static_configs = [
-                  {
-                    targets = [
-                      "node:${toString config.services.prometheus.exporters.node.port}"
-                    ];
-                  }
-                ];
-              }
-            ];
-          };
-        };
-
-      alertmanager =
-        { config, pkgs, ... }:
-        {
-          services.prometheus.alertmanager = {
-            enable = true;
-            openFirewall = true;
-
-            configuration = {
-              global = {
-                resolve_timeout = "1m";
-              };
-
-              route = {
-                # Root route node
-                receiver = "test";
-                group_by = [ "..." ];
-                continue = false;
-                group_wait = "1s";
-                group_interval = "15s";
-                repeat_interval = "24h";
-              };
-
-              receivers = [
-                {
-                  name = "test";
-                  webhook_configs = [
-                    {
-                      url = "http://logger:6725";
-                      send_resolved = true;
-                      max_alerts = 0;
-                    }
-                  ];
-                }
+          alertmanagers = [
+            {
+              scheme = "http";
+              static_configs = [
+                { targets = [ "alertmanager:${toString config.services.prometheus.alertmanager.port}" ]; }
               ];
+            }
+          ];
+
+          rules = [
+            ''
+              groups:
+                - name: test
+                  rules:
+                    - alert: InstanceDown
+                      expr: up == 0
+                      for: 5s
+                      labels:
+                        severity: page
+                      annotations:
+                        summary: "Instance {{ $labels.instance }} down"
+            ''
+          ];
+
+          scrapeConfigs = [
+            {
+              job_name = "alertmanager";
+              static_configs = [
+                { targets = [ "alertmanager:${toString config.services.prometheus.alertmanager.port}" ]; }
+              ];
+            }
+            {
+              job_name = "node";
+              static_configs = [
+                { targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ]; }
+              ];
+            }
+          ];
+        };
+      };
+
+    alertmanager =
+      { config, pkgs, ... }:
+      {
+        services.prometheus.alertmanager = {
+          enable = true;
+          openFirewall = true;
+
+          configuration = {
+            global = {
+              resolve_timeout = "1m";
             };
+
+            route = {
+              # Root route node
+              receiver = "test";
+              group_by = [ "..." ];
+              continue = false;
+              group_wait = "1s";
+              group_interval = "15s";
+              repeat_interval = "24h";
+            };
+
+            receivers = [
+              {
+                name = "test";
+                webhook_configs = [
+                  {
+                    url = "http://logger:6725";
+                    send_resolved = true;
+                    max_alerts = 0;
+                  }
+                ];
+              }
+            ];
           };
         };
+      };
 
-      logger =
-        { config, pkgs, ... }:
-        {
-          networking.firewall.allowedTCPPorts = [ 6725 ];
+    logger =
+      { config, pkgs, ... }:
+      {
+        networking.firewall.allowedTCPPorts = [ 6725 ];
 
-          services.prometheus.alertmanagerWebhookLogger.enable = true;
-        };
-    };
+        services.prometheus.alertmanagerWebhookLogger.enable = true;
+      };
+  };
 
-    testScript = ''
-      alertmanager.wait_for_unit("alertmanager")
-      alertmanager.wait_for_open_port(9093)
-      alertmanager.wait_until_succeeds("curl -s http://127.0.0.1:9093/-/ready")
-      #alertmanager.wait_until_succeeds("journalctl -o cat -u alertmanager.service | grep 'version=${pkgs.prometheus-alertmanager.version}'")
+  testScript = ''
+    alertmanager.wait_for_unit("alertmanager")
+    alertmanager.wait_for_open_port(9093)
+    alertmanager.wait_until_succeeds("curl -s http://127.0.0.1:9093/-/ready")
+    #alertmanager.wait_until_succeeds("journalctl -o cat -u alertmanager.service | grep 'version=${pkgs.prometheus-alertmanager.version}'")
 
-      logger.wait_for_unit("alertmanager-webhook-logger")
-      logger.wait_for_open_port(6725)
+    logger.wait_for_unit("alertmanager-webhook-logger")
+    logger.wait_for_open_port(6725)
 
-      prometheus.wait_for_unit("prometheus")
-      prometheus.wait_for_open_port(9090)
+    prometheus.wait_for_unit("prometheus")
+    prometheus.wait_for_open_port(9090)
 
-      prometheus.wait_until_succeeds(
-        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"alertmanager\"\}==1)' | "
-        + "jq '.data.result[0].value[1]' | grep '\"1\"'"
-      )
+    prometheus.wait_until_succeeds(
+      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"alertmanager\"\}==1)' | "
+      + "jq '.data.result[0].value[1]' | grep '\"1\"'"
+    )
 
-      prometheus.wait_until_succeeds(
-        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(alertmanager_build_info)%20by%20(version)' | "
-        + "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-alertmanager.version}\"'"
-      )
+    prometheus.wait_until_succeeds(
+      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(alertmanager_build_info)%20by%20(version)' | "
+      + "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-alertmanager.version}\"'"
+    )
 
-      prometheus.wait_until_succeeds(
-        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\}!=1)' | "
-        + "jq '.data.result[0].value[1]' | grep '\"1\"'"
-      )
+    prometheus.wait_until_succeeds(
+      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\}!=1)' | "
+      + "jq '.data.result[0].value[1]' | grep '\"1\"'"
+    )
 
-      prometheus.wait_until_succeeds(
-        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=alertmanager_notifications_total\{integration=\"webhook\"\}' | "
-        + "jq '.data.result[0].value[1]' | grep -v '\"0\"'"
-      )
+    prometheus.wait_until_succeeds(
+      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=alertmanager_notifications_total\{integration=\"webhook\"\}' | "
+      + "jq '.data.result[0].value[1]' | grep -v '\"0\"'"
+    )
 
-      logger.wait_until_succeeds(
-        "journalctl -o cat -u alertmanager-webhook-logger.service | grep '\"alertname\":\"InstanceDown\"'"
-      )
+    logger.wait_until_succeeds(
+      "journalctl -o cat -u alertmanager-webhook-logger.service | grep '\"alertname\":\"InstanceDown\"'"
+    )
 
-      logger.log(logger.succeed("systemd-analyze security alertmanager-webhook-logger.service | grep -v '✓'"))
+    logger.log(logger.succeed("systemd-analyze security alertmanager-webhook-logger.service | grep -v '✓'"))
 
-      alertmanager.log(alertmanager.succeed("systemd-analyze security alertmanager.service | grep -v '✓'"))
-    '';
-  }
-)
+    alertmanager.log(alertmanager.succeed("systemd-analyze security alertmanager.service | grep -v '✓'"))
+  '';
+}

nixos/tests/prometheus/config-reload.nix

@@ -1,120 +1,108 @@
-import ../make-test-python.nix (
-  { lib, pkgs, ... }:
+{
+  name = "prometheus-config-reload";
 
-  {
-    name = "prometheus-config-reload";
+  nodes = {
+    prometheus =
+      { config, pkgs, ... }:
+      {
+        environment.systemPackages = [ pkgs.jq ];
 
-    nodes = {
-      prometheus =
-        { config, pkgs, ... }:
-        {
-          environment.systemPackages = [ pkgs.jq ];
+        networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
 
-          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
+        services.prometheus = {
+          enable = true;
+          enableReload = true;
+          globalConfig.scrape_interval = "2s";
+          scrapeConfigs = [
+            {
+              job_name = "prometheus";
+              static_configs = [ { targets = [ "prometheus:${toString config.services.prometheus.port}" ]; } ];
+            }
+          ];
+        };
 
-          services.prometheus = {
-            enable = true;
-            enableReload = true;
-            globalConfig.scrape_interval = "2s";
-            scrapeConfigs = [
-              {
-                job_name = "prometheus";
-                static_configs = [
+        specialisation = {
+          "prometheus-config-change" = {
+            configuration = {
+              environment.systemPackages = [ pkgs.yq ];
+
+              # This configuration just adds a new prometheus job
+              # to scrape the node_exporter metrics of the s3 machine.
+              services.prometheus = {
+                scrapeConfigs = [
                   {
-                    targets = [
-                      "prometheus:${toString config.services.prometheus.port}"
+                    job_name = "node";
+                    static_configs = [
+                      { targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ]; }
                     ];
                   }
                 ];
-              }
-            ];
-          };
-
-          specialisation = {
-            "prometheus-config-change" = {
-              configuration = {
-                environment.systemPackages = [ pkgs.yq ];
-
-                # This configuration just adds a new prometheus job
-                # to scrape the node_exporter metrics of the s3 machine.
-                services.prometheus = {
-                  scrapeConfigs = [
-                    {
-                      job_name = "node";
-                      static_configs = [
-                        {
-                          targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ];
-                        }
-                      ];
-                    }
-                  ];
-                };
               };
             };
           };
         };
-    };
+      };
+  };
 
-    testScript = ''
-      prometheus.wait_for_unit("prometheus")
-      prometheus.wait_for_open_port(9090)
+  testScript = ''
+    prometheus.wait_for_unit("prometheus")
+    prometheus.wait_for_open_port(9090)
 
-      # Check if switching to a NixOS configuration that changes the prometheus
-      # configuration reloads (instead of restarts) prometheus before the switch
-      # finishes successfully:
-      with subtest("config change reloads prometheus"):
-        import json
-        # We check if prometheus has finished reloading by looking for the message
-        # "Completed loading of configuration file" in the journal between the start
-        # and finish of switching to the new NixOS configuration.
-        #
-        # To mark the start we record the journal cursor before starting the switch:
-        cursor_before_switching = json.loads(
-            prometheus.succeed("journalctl -n1 -o json --output-fields=__CURSOR")
-        )["__CURSOR"]
+    # Check if switching to a NixOS configuration that changes the prometheus
+    # configuration reloads (instead of restarts) prometheus before the switch
+    # finishes successfully:
+    with subtest("config change reloads prometheus"):
+      import json
+      # We check if prometheus has finished reloading by looking for the message
+      # "Completed loading of configuration file" in the journal between the start
+      # and finish of switching to the new NixOS configuration.
+      #
+      # To mark the start we record the journal cursor before starting the switch:
+      cursor_before_switching = json.loads(
+          prometheus.succeed("journalctl -n1 -o json --output-fields=__CURSOR")
+      )["__CURSOR"]
 
-        # Now we switch:
-        prometheus_config_change = prometheus.succeed(
-            "readlink /run/current-system/specialisation/prometheus-config-change"
-        ).strip()
-        prometheus.succeed(prometheus_config_change + "/bin/switch-to-configuration test")
+      # Now we switch:
+      prometheus_config_change = prometheus.succeed(
+          "readlink /run/current-system/specialisation/prometheus-config-change"
+      ).strip()
+      prometheus.succeed(prometheus_config_change + "/bin/switch-to-configuration test")
 
-        # Next we retrieve all logs since the start of switching:
-        logs_after_starting_switching = prometheus.succeed(
-            """
-              journalctl --after-cursor='{cursor_before_switching}' -o json --output-fields=MESSAGE
-            """.format(
-                cursor_before_switching=cursor_before_switching
-            )
-        )
-
-        # Finally we check if the message "Completed loading of configuration file"
-        # occurs before the "finished switching to system configuration" message:
-        finished_switching_msg = (
-            "finished switching to system configuration " + prometheus_config_change
-        )
-        reloaded_before_switching_finished = False
-        finished_switching = False
-        for log_line in logs_after_starting_switching.split("\n"):
-            msg = json.loads(log_line)["MESSAGE"]
-            if "Completed loading of configuration file" in msg:
-                reloaded_before_switching_finished = True
-            if msg == finished_switching_msg:
-                finished_switching = True
-                break
-
-        assert reloaded_before_switching_finished
-        assert finished_switching
-
-        # Check if the reloaded config includes the new node job:
-        prometheus.succeed(
+      # Next we retrieve all logs since the start of switching:
+      logs_after_starting_switching = prometheus.succeed(
           """
-            curl -sf http://127.0.0.1:9090/api/v1/status/config \
-              | jq -r .data.yaml \
-              | yq '.scrape_configs | any(.job_name == "node")' \
-              | grep true
-          """
-        )
-    '';
-  }
-)
+            journalctl --after-cursor='{cursor_before_switching}' -o json --output-fields=MESSAGE
+          """.format(
+              cursor_before_switching=cursor_before_switching
+          )
+      )
+
+      # Finally we check if the message "Completed loading of configuration file"
+      # occurs before the "finished switching to system configuration" message:
+      finished_switching_msg = (
+          "finished switching to system configuration " + prometheus_config_change
+      )
+      reloaded_before_switching_finished = False
+      finished_switching = False
+      for log_line in logs_after_starting_switching.split("\n"):
+          msg = json.loads(log_line)["MESSAGE"]
+          if "Completed loading of configuration file" in msg:
+              reloaded_before_switching_finished = True
+          if msg == finished_switching_msg:
+              finished_switching = True
+              break
+
+      assert reloaded_before_switching_finished
+      assert finished_switching
+
+      # Check if the reloaded config includes the new node job:
+      prometheus.succeed(
+        """
+          curl -sf http://127.0.0.1:9090/api/v1/status/config \
+            | jq -r .data.yaml \
+            | yq '.scrape_configs | any(.job_name == "node")' \
+            | grep true
+        """
+      )
+  '';
+}

nixos/tests/prometheus/default.nix

@@ -1,14 +1,10 @@
-{
-  system ? builtins.currentSystem,
-  config ? { },
-  pkgs ? import ../../.. { inherit system config; },
-}:
+{ runTest }:
 
 {
-  alertmanager = import ./alertmanager.nix { inherit system pkgs; };
-  config-reload = import ./config-reload.nix { inherit system pkgs; };
-  federation = import ./federation.nix { inherit system pkgs; };
-  prometheus-pair = import ./prometheus-pair.nix { inherit system pkgs; };
-  pushgateway = import ./pushgateway.nix { inherit system pkgs; };
-  remote-write = import ./remote-write.nix { inherit system pkgs; };
+  alertmanager = runTest ./alertmanager.nix;
+  config-reload = runTest ./config-reload.nix;
+  federation = runTest ./federation.nix;
+  prometheus-pair = runTest ./prometheus-pair.nix;
+  pushgateway = runTest ./pushgateway.nix;
+  remote-write = runTest ./remote-write.nix;
 }

nixos/tests/prometheus/federation.nix

@@ -1,227 +1,203 @@
-import ../make-test-python.nix (
-  { lib, pkgs, ... }:
+{
+  name = "prometheus-federation";
 
-  {
-    name = "prometheus-federation";
+  nodes = {
+    global1 =
+      { config, pkgs, ... }:
+      {
+        environment.systemPackages = [ pkgs.jq ];
 
-    nodes = {
-      global1 =
-        { config, pkgs, ... }:
-        {
-          environment.systemPackages = [ pkgs.jq ];
+        networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
 
-          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
+        services.prometheus = {
+          enable = true;
+          globalConfig.scrape_interval = "2s";
 
-          services.prometheus = {
-            enable = true;
-            globalConfig.scrape_interval = "2s";
+          scrapeConfigs = [
+            {
+              job_name = "federate";
+              honor_labels = true;
+              metrics_path = "/federate";
 
-            scrapeConfigs = [
-              {
-                job_name = "federate";
-                honor_labels = true;
-                metrics_path = "/federate";
+              params = {
+                "match[]" = [
+                  "{job=\"node\"}"
+                  "{job=\"prometheus\"}"
+                ];
+              };
 
-                params = {
-                  "match[]" = [
-                    "{job=\"node\"}"
-                    "{job=\"prometheus\"}"
+              static_configs = [
+                {
+                  targets = [
+                    "prometheus1:${toString config.services.prometheus.port}"
+                    "prometheus2:${toString config.services.prometheus.port}"
                   ];
-                };
-
-                static_configs = [
-                  {
-                    targets = [
-                      "prometheus1:${toString config.services.prometheus.port}"
-                      "prometheus2:${toString config.services.prometheus.port}"
-                    ];
-                  }
-                ];
-              }
-              {
-                job_name = "prometheus";
-                static_configs = [
-                  {
-                    targets = [
-                      "global1:${toString config.services.prometheus.port}"
-                      "global2:${toString config.services.prometheus.port}"
-                    ];
-                  }
-                ];
-              }
-            ];
-          };
-        };
-
-      global2 =
-        { config, pkgs, ... }:
-        {
-          environment.systemPackages = [ pkgs.jq ];
-
-          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
-
-          services.prometheus = {
-            enable = true;
-            globalConfig.scrape_interval = "2s";
-
-            scrapeConfigs = [
-              {
-                job_name = "federate";
-                honor_labels = true;
-                metrics_path = "/federate";
-
-                params = {
-                  "match[]" = [
-                    "{job=\"node\"}"
-                    "{job=\"prometheus\"}"
+                }
+              ];
+            }
+            {
+              job_name = "prometheus";
+              static_configs = [
+                {
+                  targets = [
+                    "global1:${toString config.services.prometheus.port}"
+                    "global2:${toString config.services.prometheus.port}"
                   ];
-                };
-
-                static_configs = [
-                  {
-                    targets = [
-                      "prometheus1:${toString config.services.prometheus.port}"
-                      "prometheus2:${toString config.services.prometheus.port}"
-                    ];
-                  }
-                ];
-              }
-              {
-                job_name = "prometheus";
-                static_configs = [
-                  {
-                    targets = [
-                      "global1:${toString config.services.prometheus.port}"
-                      "global2:${toString config.services.prometheus.port}"
-                    ];
-                  }
-                ];
-              }
-            ];
-          };
+                }
+              ];
+            }
+          ];
         };
+      };
 
-      prometheus1 =
-        { config, pkgs, ... }:
-        {
-          environment.systemPackages = [ pkgs.jq ];
+    global2 =
+      { config, pkgs, ... }:
+      {
+        environment.systemPackages = [ pkgs.jq ];
 
-          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
+        networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
 
-          services.prometheus = {
-            enable = true;
-            globalConfig.scrape_interval = "2s";
+        services.prometheus = {
+          enable = true;
+          globalConfig.scrape_interval = "2s";
 
-            scrapeConfigs = [
-              {
-                job_name = "node";
-                static_configs = [
-                  {
-                    targets = [
-                      "node1:${toString config.services.prometheus.exporters.node.port}"
-                    ];
-                  }
+          scrapeConfigs = [
+            {
+              job_name = "federate";
+              honor_labels = true;
+              metrics_path = "/federate";
+
+              params = {
+                "match[]" = [
+                  "{job=\"node\"}"
+                  "{job=\"prometheus\"}"
                 ];
-              }
-              {
-                job_name = "prometheus";
-                static_configs = [
-                  {
-                    targets = [
-                      "prometheus1:${toString config.services.prometheus.port}"
-                    ];
-                  }
-                ];
-              }
-            ];
-          };
+              };
+
+              static_configs = [
+                {
+                  targets = [
+                    "prometheus1:${toString config.services.prometheus.port}"
+                    "prometheus2:${toString config.services.prometheus.port}"
+                  ];
+                }
+              ];
+            }
+            {
+              job_name = "prometheus";
+              static_configs = [
+                {
+                  targets = [
+                    "global1:${toString config.services.prometheus.port}"
+                    "global2:${toString config.services.prometheus.port}"
+                  ];
+                }
+              ];
+            }
+          ];
         };
+      };
 
-      prometheus2 =
-        { config, pkgs, ... }:
-        {
-          environment.systemPackages = [ pkgs.jq ];
+    prometheus1 =
+      { config, pkgs, ... }:
+      {
+        environment.systemPackages = [ pkgs.jq ];
 
-          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
+        networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
 
-          services.prometheus = {
-            enable = true;
-            globalConfig.scrape_interval = "2s";
+        services.prometheus = {
+          enable = true;
+          globalConfig.scrape_interval = "2s";
 
-            scrapeConfigs = [
-              {
-                job_name = "node";
-                static_configs = [
-                  {
-                    targets = [
-                      "node2:${toString config.services.prometheus.exporters.node.port}"
-                    ];
-                  }
-                ];
-              }
-              {
-                job_name = "prometheus";
-                static_configs = [
-                  {
-                    targets = [
-                      "prometheus2:${toString config.services.prometheus.port}"
-                    ];
-                  }
-                ];
-              }
-            ];
-          };
+          scrapeConfigs = [
+            {
+              job_name = "node";
+              static_configs = [
+                { targets = [ "node1:${toString config.services.prometheus.exporters.node.port}" ]; }
+              ];
+            }
+            {
+              job_name = "prometheus";
+              static_configs = [ { targets = [ "prometheus1:${toString config.services.prometheus.port}" ]; } ];
+            }
+          ];
         };
+      };
 
-      node1 =
-        { config, pkgs, ... }:
-        {
-          services.prometheus.exporters.node = {
-            enable = true;
-            openFirewall = true;
-          };
+    prometheus2 =
+      { config, pkgs, ... }:
+      {
+        environment.systemPackages = [ pkgs.jq ];
+
+        networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
+
+        services.prometheus = {
+          enable = true;
+          globalConfig.scrape_interval = "2s";
+
+          scrapeConfigs = [
+            {
+              job_name = "node";
+              static_configs = [
+                { targets = [ "node2:${toString config.services.prometheus.exporters.node.port}" ]; }
+              ];
+            }
+            {
+              job_name = "prometheus";
+              static_configs = [ { targets = [ "prometheus2:${toString config.services.prometheus.port}" ]; } ];
+            }
+          ];
         };
+      };
 
-      node2 =
-        { config, pkgs, ... }:
-        {
-          services.prometheus.exporters.node = {
-            enable = true;
-            openFirewall = true;
-          };
+    node1 =
+      { config, pkgs, ... }:
+      {
+        services.prometheus.exporters.node = {
+          enable = true;
+          openFirewall = true;
         };
-    };
+      };
 
-    testScript = ''
-      for machine in node1, node2:
-        machine.wait_for_unit("prometheus-node-exporter")
-        machine.wait_for_open_port(9100)
+    node2 =
+      { config, pkgs, ... }:
+      {
+        services.prometheus.exporters.node = {
+          enable = true;
+          openFirewall = true;
+        };
+      };
+  };
 
-      for machine in prometheus1, prometheus2, global1, global2:
-        machine.wait_for_unit("prometheus")
-        machine.wait_for_open_port(9090)
+  testScript = ''
+    for machine in node1, node2:
+      machine.wait_for_unit("prometheus-node-exporter")
+      machine.wait_for_open_port(9100)
 
-      # Verify both servers got the same data from the exporter
-      for machine in prometheus1, prometheus2:
-        machine.wait_until_succeeds(
-          "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
-          + "jq '.data.result[0].value[1]' | grep '\"1\"'"
-        )
-        machine.wait_until_succeeds(
-          "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
-          + "jq '.data.result[0].value[1]' | grep '\"1\"'"
-        )
+    for machine in prometheus1, prometheus2, global1, global2:
+      machine.wait_for_unit("prometheus")
+      machine.wait_for_open_port(9090)
 
-      for machine in global1, global2:
-        machine.wait_until_succeeds(
-          "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
-          + "jq '.data.result[0].value[1]' | grep '\"2\"'"
-        )
+    # Verify both servers got the same data from the exporter
+    for machine in prometheus1, prometheus2:
+      machine.wait_until_succeeds(
+        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
+        + "jq '.data.result[0].value[1]' | grep '\"1\"'"
+      )
+      machine.wait_until_succeeds(
+        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
+        + "jq '.data.result[0].value[1]' | grep '\"1\"'"
+      )
 
-        machine.wait_until_succeeds(
-          "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
-          + "jq '.data.result[0].value[1]' | grep '\"4\"'"
-        )
-    '';
-  }
-)
+    for machine in global1, global2:
+      machine.wait_until_succeeds(
+        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
+        + "jq '.data.result[0].value[1]' | grep '\"2\"'"
+      )
+
+      machine.wait_until_succeeds(
+        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
+        + "jq '.data.result[0].value[1]' | grep '\"4\"'"
+      )
+  '';
+}

nixos/tests/prometheus/prometheus-pair.nix

@@ -1,93 +1,91 @@
-import ../make-test-python.nix (
-  { lib, pkgs, ... }:
+{ pkgs, ... }:
 
-  {
-    name = "prometheus-pair";
+{
+  name = "prometheus-pair";
 
-    nodes = {
-      prometheus1 =
-        { config, pkgs, ... }:
-        {
-          environment.systemPackages = [ pkgs.jq ];
+  nodes = {
+    prometheus1 =
+      { config, pkgs, ... }:
+      {
+        environment.systemPackages = [ pkgs.jq ];
 
-          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
+        networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
 
-          services.prometheus = {
-            enable = true;
-            globalConfig.scrape_interval = "2s";
-            scrapeConfigs = [
-              {
-                job_name = "prometheus";
-                static_configs = [
-                  {
-                    targets = [
-                      "prometheus1:${toString config.services.prometheus.port}"
-                      "prometheus2:${toString config.services.prometheus.port}"
-                    ];
-                  }
-                ];
-              }
-            ];
-          };
+        services.prometheus = {
+          enable = true;
+          globalConfig.scrape_interval = "2s";
+          scrapeConfigs = [
+            {
+              job_name = "prometheus";
+              static_configs = [
+                {
+                  targets = [
+                    "prometheus1:${toString config.services.prometheus.port}"
+                    "prometheus2:${toString config.services.prometheus.port}"
+                  ];
+                }
+              ];
+            }
+          ];
         };
+      };
 
-      prometheus2 =
-        { config, pkgs, ... }:
-        {
-          environment.systemPackages = [ pkgs.jq ];
+    prometheus2 =
+      { config, pkgs, ... }:
+      {
+        environment.systemPackages = [ pkgs.jq ];
 
-          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
+        networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
 
-          services.prometheus = {
-            enable = true;
-            globalConfig.scrape_interval = "2s";
-            scrapeConfigs = [
-              {
-                job_name = "prometheus";
-                static_configs = [
-                  {
-                    targets = [
-                      "prometheus1:${toString config.services.prometheus.port}"
-                      "prometheus2:${toString config.services.prometheus.port}"
-                    ];
-                  }
-                ];
-              }
-            ];
-          };
+        services.prometheus = {
+          enable = true;
+          globalConfig.scrape_interval = "2s";
+          scrapeConfigs = [
+            {
+              job_name = "prometheus";
+              static_configs = [
+                {
+                  targets = [
+                    "prometheus1:${toString config.services.prometheus.port}"
+                    "prometheus2:${toString config.services.prometheus.port}"
+                  ];
+                }
+              ];
+            }
+          ];
         };
-    };
+      };
+  };
 
-    testScript = ''
-      for machine in prometheus1, prometheus2:
-        machine.wait_for_unit("prometheus")
-        machine.wait_for_open_port(9090)
-        machine.wait_until_succeeds("journalctl -o cat -u prometheus.service | grep 'version=${pkgs.prometheus.version}'")
-        machine.wait_until_succeeds("curl -sSf http://localhost:9090/-/healthy")
+  testScript = ''
+    for machine in prometheus1, prometheus2:
+      machine.wait_for_unit("prometheus")
+      machine.wait_for_open_port(9090)
+      machine.wait_until_succeeds("journalctl -o cat -u prometheus.service | grep 'version=${pkgs.prometheus.version}'")
+      machine.wait_until_succeeds("curl -sSf http://localhost:9090/-/healthy")
 
-      # Prometheii ready - run some queries
-      for machine in prometheus1, prometheus2:
-        machine.wait_until_succeeds(
-          "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_build_info\{instance=\"prometheus1:9090\",version=\"${pkgs.prometheus.version}\"\}' | "
-          + "jq '.data.result[0].value[1]' | grep '\"1\"'"
-        )
+    # Prometheii ready - run some queries
+    for machine in prometheus1, prometheus2:
+      machine.wait_until_succeeds(
+        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_build_info\{instance=\"prometheus1:9090\",version=\"${pkgs.prometheus.version}\"\}' | "
+        + "jq '.data.result[0].value[1]' | grep '\"1\"'"
+      )
 
-        machine.wait_until_succeeds(
-          "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_build_info\{instance=\"prometheus1:9090\"\}' | "
-          + "jq '.data.result[0].value[1]' | grep '\"1\"'"
-        )
+      machine.wait_until_succeeds(
+        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_build_info\{instance=\"prometheus1:9090\"\}' | "
+        + "jq '.data.result[0].value[1]' | grep '\"1\"'"
+      )
 
-        machine.wait_until_succeeds(
-          "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(prometheus_build_info)%20by%20(version)' | "
-          + "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus.version}\"'"
-        )
+      machine.wait_until_succeeds(
+        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(prometheus_build_info)%20by%20(version)' | "
+        + "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus.version}\"'"
+      )
 
-        machine.wait_until_succeeds(
-          "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(prometheus_build_info)%20by%20(version)' | "
-          + "jq '.data.result[0].value[1]' | grep '\"2\"'"
-        )
+      machine.wait_until_succeeds(
+        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(prometheus_build_info)%20by%20(version)' | "
+        + "jq '.data.result[0].value[1]' | grep '\"2\"'"
+      )
 
-      prometheus1.log(prometheus1.succeed("systemd-analyze security prometheus.service | grep -v '✓'"))
-    '';
-  }
-)
+    prometheus1.log(prometheus1.succeed("systemd-analyze security prometheus.service | grep -v '✓'"))
+  '';
+}

nixos/tests/prometheus/pushgateway.nix

@@ -1,102 +1,91 @@
-import ../make-test-python.nix (
-  { lib, pkgs, ... }:
+{ pkgs, ... }:
 
-  {
-    name = "prometheus-pushgateway";
+{
+  name = "prometheus-pushgateway";
 
-    nodes = {
-      prometheus =
-        { config, pkgs, ... }:
-        {
-          environment.systemPackages = [ pkgs.jq ];
+  nodes = {
+    prometheus =
+      { config, pkgs, ... }:
+      {
+        environment.systemPackages = [ pkgs.jq ];
 
-          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
+        networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
 
-          services.prometheus = {
-            enable = true;
-            globalConfig.scrape_interval = "2s";
+        services.prometheus = {
+          enable = true;
+          globalConfig.scrape_interval = "2s";
 
-            scrapeConfigs = [
-              {
-                job_name = "pushgateway";
-                static_configs = [
-                  {
-                    targets = [
-                      "pushgateway:9091"
-                    ];
-                  }
-                ];
-              }
-            ];
-          };
+          scrapeConfigs = [
+            {
+              job_name = "pushgateway";
+              static_configs = [ { targets = [ "pushgateway:9091" ]; } ];
+            }
+          ];
         };
+      };
 
-      pushgateway =
-        { config, pkgs, ... }:
-        {
-          networking.firewall.allowedTCPPorts = [ 9091 ];
+    pushgateway =
+      { config, pkgs, ... }:
+      {
+        networking.firewall.allowedTCPPorts = [ 9091 ];
 
-          services.prometheus.pushgateway = {
-            enable = true;
-          };
+        services.prometheus.pushgateway = {
+          enable = true;
         };
+      };
 
-      client =
-        { config, pkgs, ... }:
-        {
-        };
-    };
+    client = { config, pkgs, ... }: { };
+  };
 
-    testScript = ''
-      pushgateway.wait_for_unit("pushgateway")
-      pushgateway.wait_for_open_port(9091)
-      pushgateway.wait_until_succeeds("curl -s http://127.0.0.1:9091/-/ready")
-      pushgateway.wait_until_succeeds("journalctl -o cat -u pushgateway.service | grep 'version=${pkgs.prometheus-pushgateway.version}'")
+  testScript = ''
+    pushgateway.wait_for_unit("pushgateway")
+    pushgateway.wait_for_open_port(9091)
+    pushgateway.wait_until_succeeds("curl -s http://127.0.0.1:9091/-/ready")
+    pushgateway.wait_until_succeeds("journalctl -o cat -u pushgateway.service | grep 'version=${pkgs.prometheus-pushgateway.version}'")
 
-      prometheus.wait_for_unit("prometheus")
-      prometheus.wait_for_open_port(9090)
+    prometheus.wait_for_unit("prometheus")
+    prometheus.wait_for_open_port(9090)
 
-      prometheus.wait_until_succeeds(
-        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"pushgateway\"\})' | "
-        + "jq '.data.result[0].value[1]' | grep '\"1\"'"
-      )
+    prometheus.wait_until_succeeds(
+      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"pushgateway\"\})' | "
+      + "jq '.data.result[0].value[1]' | grep '\"1\"'"
+    )
 
-      prometheus.wait_until_succeeds(
-        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(pushgateway_build_info)%20by%20(version)' | "
-        + "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-pushgateway.version}\"'"
-      )
+    prometheus.wait_until_succeeds(
+      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(pushgateway_build_info)%20by%20(version)' | "
+      + "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-pushgateway.version}\"'"
+    )
 
-      # Add a metric and check in Prometheus
-      client.wait_until_succeeds(
-        "echo 'some_metric 3.14' | curl --data-binary @- http://pushgateway:9091/metrics/job/some_job"
-      )
+    # Add a metric and check in Prometheus
+    client.wait_until_succeeds(
+      "echo 'some_metric 3.14' | curl --data-binary @- http://pushgateway:9091/metrics/job/some_job"
+    )
 
-      prometheus.wait_until_succeeds(
-        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' | "
-        + "jq '.data.result[0].value[1]' | grep '\"3.14\"'"
-      )
+    prometheus.wait_until_succeeds(
+      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' | "
+      + "jq '.data.result[0].value[1]' | grep '\"3.14\"'"
+    )
 
-      prometheus.wait_until_succeeds(
-        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=absent(some_metric)' | "
-        + "jq '.data.result[0].value[1]' | grep 'null'"
-      )
+    prometheus.wait_until_succeeds(
+      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=absent(some_metric)' | "
+      + "jq '.data.result[0].value[1]' | grep 'null'"
+    )
 
-      # Delete the metric, check not in Prometheus
-      client.wait_until_succeeds(
-        "curl -X DELETE http://pushgateway:9091/metrics/job/some_job"
-      )
+    # Delete the metric, check not in Prometheus
+    client.wait_until_succeeds(
+      "curl -X DELETE http://pushgateway:9091/metrics/job/some_job"
+    )
 
-      prometheus.wait_until_fails(
-        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' | "
-        + "jq '.data.result[0].value[1]' | grep '\"3.14\"'"
-      )
+    prometheus.wait_until_fails(
+      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' | "
+      + "jq '.data.result[0].value[1]' | grep '\"3.14\"'"
+    )
 
-      prometheus.wait_until_succeeds(
-        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=absent(some_metric)' | "
-        + "jq '.data.result[0].value[1]' | grep '\"1\"'"
-      )
+    prometheus.wait_until_succeeds(
+      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=absent(some_metric)' | "
+      + "jq '.data.result[0].value[1]' | grep '\"1\"'"
+    )
 
-      pushgateway.log(pushgateway.succeed("systemd-analyze security pushgateway.service | grep -v '✓'"))
-    '';
-  }
-)
+    pushgateway.log(pushgateway.succeed("systemd-analyze security pushgateway.service | grep -v '✓'"))
+  '';
+}

nixos/tests/prometheus/remote-write.nix

@@ -1,81 +1,69 @@
-import ../make-test-python.nix (
-  { lib, pkgs, ... }:
+{
+  name = "prometheus-remote-write";
 
-  {
-    name = "prometheus-remote-write";
+  nodes = {
+    receiver =
+      { config, pkgs, ... }:
+      {
+        environment.systemPackages = [ pkgs.jq ];
 
-    nodes = {
-      receiver =
-        { config, pkgs, ... }:
-        {
-          environment.systemPackages = [ pkgs.jq ];
+        networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
 
-          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
+        services.prometheus = {
+          enable = true;
+          globalConfig.scrape_interval = "2s";
 
-          services.prometheus = {
-            enable = true;
-            globalConfig.scrape_interval = "2s";
-
-            extraFlags = [ "--web.enable-remote-write-receiver" ];
-          };
+          extraFlags = [ "--web.enable-remote-write-receiver" ];
         };
+      };
 
-      prometheus =
-        { config, pkgs, ... }:
-        {
-          environment.systemPackages = [ pkgs.jq ];
+    prometheus =
+      { config, pkgs, ... }:
+      {
+        environment.systemPackages = [ pkgs.jq ];
 
-          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
+        networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
 
-          services.prometheus = {
-            enable = true;
-            globalConfig.scrape_interval = "2s";
+        services.prometheus = {
+          enable = true;
+          globalConfig.scrape_interval = "2s";
 
-            remoteWrite = [
-              {
-                url = "http://receiver:9090/api/v1/write";
-              }
-            ];
+          remoteWrite = [ { url = "http://receiver:9090/api/v1/write"; } ];
 
-            scrapeConfigs = [
-              {
-                job_name = "node";
-                static_configs = [
-                  {
-                    targets = [
-                      "node:${toString config.services.prometheus.exporters.node.port}"
-                    ];
-                  }
-                ];
-              }
-            ];
-          };
+          scrapeConfigs = [
+            {
+              job_name = "node";
+              static_configs = [
+                { targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ]; }
+              ];
+            }
+          ];
         };
+      };
 
-      node =
-        { config, pkgs, ... }:
-        {
-          services.prometheus.exporters.node = {
-            enable = true;
-            openFirewall = true;
-          };
+    node =
+      { config, pkgs, ... }:
+      {
+        services.prometheus.exporters.node = {
+          enable = true;
+          openFirewall = true;
         };
-    };
+      };
+  };
 
-    testScript = ''
-      node.wait_for_unit("prometheus-node-exporter")
-      node.wait_for_open_port(9100)
+  testScript = ''
+    node.wait_for_unit("prometheus-node-exporter")
+    node.wait_for_open_port(9100)
 
-      for machine in prometheus, receiver:
-        machine.wait_for_unit("prometheus")
-        machine.wait_for_open_port(9090)
+    for machine in prometheus, receiver:
+      machine.wait_for_unit("prometheus")
+      machine.wait_for_open_port(9090)
 
-      # Verify both servers got the same data from the exporter
-      for machine in prometheus, receiver:
-        machine.wait_until_succeeds(
-          "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=node_exporter_build_info\{instance=\"node:9100\"\}' | "
-          + "jq '.data.result[0].value[1]' | grep '\"1\"'"
-        )
-    '';
-  }
-)
+    # Verify both servers got the same data from the exporter
+    for machine in prometheus, receiver:
+      machine.wait_until_succeeds(
+        "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=node_exporter_build_info\{instance=\"node:9100\"\}' | "
+        + "jq '.data.result[0].value[1]' | grep '\"1\"'"
+      )
+  '';
+}