nixos/grafana: add alerting

nixos/doc/manual/from_md/release-notes/rl-2211.section.xml

@@ -844,6 +844,15 @@
           option.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          The <literal>services.grafana.provision.alerting</literal>
+          option was added. It includes suboptions for every
+          alerting-related objects (with the exception of
+          <literal>notifiers</literal>), which means it’s now possible
+          to configure modern Grafana alerting declaratively.
+        </para>
+      </listitem>
       <listitem>
         <para>
           Matrix Synapse now requires entries in the

nixos/doc/manual/release-notes/rl-2211.section.md

@@ -274,6 +274,8 @@ Available as [services.patroni](options.html#opt-services.patroni.enable).
 
 - The `services.grafana.provision.datasources` and `services.grafana.provision.dashboards` options were converted to a [RFC 0042](https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md) configuration. They also now support specifying the provisioning YAML file with `path` option.
 
+- The `services.grafana.provision.alerting` option was added. It includes suboptions for every alerting-related objects (with the exception of `notifiers`), which means it's now possible to configure modern Grafana alerting declaratively.
+
 - Matrix Synapse now requires entries in the `state_group_edges` table to be unique, in order to prevent accidentally introducing duplicate information (for example, because a database backup was restored multiple times). If your Synapse database already has duplicate rows in this table, this could fail with an error and require manual remediation.
 
 - The `diamond` package has been update from 0.8.36 to 2.0.15. See the [upstream release notes](https://github.com/bbuchfink/diamond/releases) for more details.

nixos/modules/services/monitoring/grafana.nix

@@ -96,11 +96,25 @@ let
 
   notifierFile = pkgs.writeText "notifier.yaml" (builtins.toJSON notifierConfiguration);
 
+  generateAlertingProvisioningYaml = x: if (cfg.provision.alerting."${x}".path == null)
+                                        then provisioningSettingsFormat.generate "${x}.yaml" cfg.provision.alerting."${x}".settings
+                                        else cfg.provision.alerting."${x}".path;
+  rulesFile = generateAlertingProvisioningYaml "rules";
+  contactPointsFile = generateAlertingProvisioningYaml "contactPoints";
+  policiesFile = generateAlertingProvisioningYaml "policies";
+  templatesFile = generateAlertingProvisioningYaml "templates";
+  muteTimingsFile = generateAlertingProvisioningYaml "muteTimings";
+
   provisionConfDir =  pkgs.runCommand "grafana-provisioning" { } ''
-    mkdir -p $out/{datasources,dashboards,notifiers}
+    mkdir -p $out/{datasources,dashboards,notifiers,alerting}
     ln -sf ${datasourceFile} $out/datasources/datasource.yaml
     ln -sf ${dashboardFile} $out/dashboards/dashboard.yaml
     ln -sf ${notifierFile} $out/notifiers/notifier.yaml
+    ln -sf ${rulesFile} $out/alerting/rules.yaml
+    ln -sf ${contactPointsFile} $out/alerting/contactPoints.yaml
+    ln -sf ${policiesFile} $out/alerting/policies.yaml
+    ln -sf ${templatesFile} $out/alerting/templates.yaml
+    ln -sf ${muteTimingsFile} $out/alerting/muteTimings.yaml
   '';
 
   # Get a submodule without any embedded metadata:
@@ -544,6 +558,461 @@ in {
         type = types.listOf grafanaTypes.notifierConfig;
         apply = x: map _filter x;
       };
+
+
+      alerting = {
+        rules = {
+          path = mkOption {
+            description = lib.mdDoc ''
+              Path to YAML rules configuration. Can't be used with
+              `services.grafana.provision.alerting.rules.settings` simultaneously.
+            '';
+            default = null;
+            type = types.nullOr types.path;
+          };
+
+          settings = mkOption {
+            description = lib.mdDoc ''
+              Grafana rules configuration in Nix. Can't be used with
+              `services.grafana.provision.alerting.rules.path` simultaneously. See
+              <link xlink:href="https://grafana.com/docs/grafana/latest/administration/provisioning/#rules"/>
+              for supported options.
+            '';
+            default = null;
+            type = types.nullOr (types.submodule {
+              options = {
+                apiVersion = mkOption {
+                  description = lib.mdDoc "Config file version.";
+                  default = 1;
+                  type = types.int;
+                };
+
+                groups = mkOption {
+                  description = lib.mdDoc "List of rule groups to import or update.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    freeformType = provisioningSettingsFormat.type;
+
+                    options.name = mkOption {
+                      description = lib.mdDoc "Name of the rule group. Required.";
+                      type = types.str;
+                    };
+
+                    options.folder = mkOption {
+                      description = lib.mdDoc "Name of the folder the rule group will be stored in. Required.";
+                      type = types.str;
+                    };
+
+                    options.interval = mkOption {
+                      description = lib.mdDoc "Interval that the rule group should be evaluated at. Required.";
+                      type = types.str;
+                    };
+                  });
+                };
+
+                deleteRules = mkOption {
+                  description = lib.mdDoc "List of alert rule UIDs that should be deleted.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    options.orgId = mkOption {
+                      description = lib.mdDoc "Organization ID, default = 1";
+                      default = 1;
+                      type = types.int;
+                    };
+
+                    options.uid = mkOption {
+                      description = lib.mdDoc "Unique identifier for the rule. Required.";
+                      type = types.str;
+                    };
+                  });
+                };
+              };
+            });
+            example = literalExpression ''
+              {
+                apiVersion = 1;
+
+                groups = [{
+                  orgId = 1;
+                  name = "my_rule_group";
+                  folder = "my_first_folder";
+                  interval = "60s";
+                  rules = [{
+                    uid = "my_id_1";
+                    title = "my_first_rule";
+                    condition = "A";
+                    data = [{
+                      refId = "A";
+                      datasourceUid = "-100";
+                      model = {
+                        conditions = [{
+                          evaluator = {
+                            params = [ 3 ];
+                            type = "git";
+                          };
+                          operator.type = "and";
+                          query.params = [ "A" ];
+                          reducer.type = "last";
+                          type = "query";
+                        }];
+                        datasource = {
+                          type = "__expr__";
+                          uid = "-100";
+                        };
+                        expression = "1==0";
+                        intervalMs = 1000;
+                        maxDataPoints = 43200;
+                        refId = "A";
+                        type = "math";
+                      };
+                    }];
+                    dashboardUid = "my_dashboard";
+                    panelId = 123;
+                    noDataState = "Alerting";
+                    for = "60s";
+                    annotations.some_key = "some_value";
+                    labels.team = "sre_team1";
+                  }];
+                }];
+
+                deleteRules = [{
+                  orgId = 1;
+                  uid = "my_id_1";
+                }];
+              }
+            '';
+          };
+        };
+
+        contactPoints = {
+          path = mkOption {
+            description = lib.mdDoc ''
+              Path to YAML contact points configuration. Can't be used with
+              `services.grafana.provision.alerting.contactPoints.settings` simultaneously.
+            '';
+            default = null;
+            type = types.nullOr types.path;
+          };
+
+          settings = mkOption {
+            description = lib.mdDoc ''
+              Grafana contact points configuration in Nix. Can't be used with
+              `services.grafana.provision.alerting.contactPoints.path` simultaneously. See
+              <link xlink:href="https://grafana.com/docs/grafana/latest/administration/provisioning/#contact-points"/>
+              for supported options.
+            '';
+            default = null;
+            type = types.nullOr (types.submodule {
+              options = {
+                apiVersion = mkOption {
+                  description = lib.mdDoc "Config file version.";
+                  default = 1;
+                  type = types.int;
+                };
+
+                contactPoints = mkOption {
+                  description = lib.mdDoc "List of contact points to import or update.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    freeformType = provisioningSettingsFormat.type;
+
+                    options.name = mkOption {
+                      description = lib.mdDoc "Name of the contact point. Required.";
+                      type = types.str;
+                    };
+                  });
+                };
+
+                deleteContactPoints = mkOption {
+                  description = lib.mdDoc "List of receivers that should be deleted.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    options.orgId = mkOption {
+                      description = lib.mdDoc "Organization ID, default = 1.";
+                      default = 1;
+                      type = types.int;
+                    };
+
+                    options.uid = mkOption {
+                      description = lib.mdDoc "Unique identifier for the receiver. Required.";
+                      type = types.str;
+                    };
+                  });
+                };
+              };
+            });
+            example = literalExpression ''
+              {
+                apiVersion = 1;
+
+                contactPoints = [{
+                  orgId = 1;
+                  name = "cp_1";
+                  receivers = [{
+                    uid = "first_uid";
+                    type = "prometheus-alertmanager";
+                    settings.url = "http://test:9000";
+                  }];
+                }];
+
+                deleteContactPoints = [{
+                  orgId = 1;
+                  uid = "first_uid";
+                }];
+              }
+            '';
+          };
+        };
+
+        policies = {
+          path = mkOption {
+            description = lib.mdDoc ''
+              Path to YAML notification policies configuration. Can't be used with
+              `services.grafana.provision.alerting.policies.settings` simultaneously.
+            '';
+            default = null;
+            type = types.nullOr types.path;
+          };
+
+          settings = mkOption {
+            description = lib.mdDoc ''
+              Grafana notification policies configuration in Nix. Can't be used with
+              `services.grafana.provision.alerting.policies.path` simultaneously. See
+              <link xlink:href="https://grafana.com/docs/grafana/latest/administration/provisioning/#notification-policies"/>
+              for supported options.
+            '';
+            default = null;
+            type = types.nullOr (types.submodule {
+              options = {
+                apiVersion = mkOption {
+                  description = lib.mdDoc "Config file version.";
+                  default = 1;
+                  type = types.int;
+                };
+
+                policies = mkOption {
+                  description = lib.mdDoc "List of contact points to import or update.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    freeformType = provisioningSettingsFormat.type;
+                  });
+                };
+
+                resetPolicies = mkOption {
+                  description = lib.mdDoc "List of orgIds that should be reset to the default policy.";
+                  default = [];
+                  type = types.listOf types.int;
+                };
+              };
+            });
+            example = literalExpression ''
+              {
+                apiVersion = 1;
+
+                policies = [{
+                  orgId = 1;
+                  receiver = "grafana-default-email";
+                  group_by = [ "..." ];
+                  matchers = [
+                    "alertname = Watchdog"
+                    "severity =~ \"warning|critical\""
+                  ];
+                  mute_time_intervals = [
+                    "abc"
+                  ];
+                  group_wait = "30s";
+                  group_interval = "5m";
+                  repeat_interval = "4h";
+                }];
+
+                resetPolicies = [
+                  1
+                ];
+              }
+            '';
+          };
+        };
+
+        templates = {
+          path = mkOption {
+            description = lib.mdDoc ''
+              Path to YAML templates configuration. Can't be used with
+              `services.grafana.provision.alerting.templates.settings` simultaneously.
+            '';
+            default = null;
+            type = types.nullOr types.path;
+          };
+
+          settings = mkOption {
+            description = lib.mdDoc ''
+              Grafana templates configuration in Nix. Can't be used with
+              `services.grafana.provision.alerting.templates.path` simultaneously. See
+              <link xlink:href="https://grafana.com/docs/grafana/latest/administration/provisioning/#templates"/>
+              for supported options.
+            '';
+            default = null;
+            type = types.nullOr (types.submodule {
+              options = {
+                apiVersion = mkOption {
+                  description = lib.mdDoc "Config file version.";
+                  default = 1;
+                  type = types.int;
+                };
+
+                templates = mkOption {
+                  description = lib.mdDoc "List of templates to import or update.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    freeformType = provisioningSettingsFormat.type;
+
+                    options.name = mkOption {
+                      description = lib.mdDoc "Name of the template, must be unique. Required.";
+                      type = types.str;
+                    };
+
+                    options.template = mkOption {
+                      description = lib.mdDoc "Alerting with a custom text template";
+                      type = types.str;
+                    };
+                  });
+                };
+
+                deleteTemplates = mkOption {
+                  description = lib.mdDoc "List of alert rule UIDs that should be deleted.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    options.orgId = mkOption {
+                      description = lib.mdDoc "Organization ID, default = 1.";
+                      default = 1;
+                      type = types.int;
+                    };
+
+                    options.name = mkOption {
+                      description = lib.mdDoc "Name of the template, must be unique. Required.";
+                      type = types.str;
+                    };
+                  });
+                };
+              };
+            });
+            example = literalExpression ''
+              {
+                apiVersion = 1;
+
+                templates = [{
+                  orgId = 1;
+                  name = "my_first_template";
+                  template = "Alerting with a custom text template";
+                }];
+
+                deleteTemplates = [{
+                  orgId = 1;
+                  name = "my_first_template";
+                }];
+              }
+            '';
+          };
+        };
+
+        muteTimings = {
+          path = mkOption {
+            description = lib.mdDoc ''
+              Path to YAML mute timings configuration. Can't be used with
+              `services.grafana.provision.alerting.muteTimings.settings` simultaneously.
+            '';
+            default = null;
+            type = types.nullOr types.path;
+          };
+
+          settings = mkOption {
+            description = lib.mdDoc ''
+              Grafana mute timings configuration in Nix. Can't be used with
+              `services.grafana.provision.alerting.muteTimings.path` simultaneously. See
+              <link xlink:href="https://grafana.com/docs/grafana/latest/administration/provisioning/#mute-timings"/>
+              for supported options.
+            '';
+            default = null;
+            type = types.nullOr (types.submodule {
+              options = {
+                apiVersion = mkOption {
+                  description = lib.mdDoc "Config file version.";
+                  default = 1;
+                  type = types.int;
+                };
+
+                muteTimes = mkOption {
+                  description = lib.mdDoc "List of mute time intervals to import or update.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    freeformType = provisioningSettingsFormat.type;
+
+                    options.name = mkOption {
+                      description = lib.mdDoc "Name of the mute time interval, must be unique. Required.";
+                      type = types.str;
+                    };
+                  });
+                };
+
+                deleteMuteTimes = mkOption {
+                  description = lib.mdDoc "List of mute time intervals that should be deleted.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    options.orgId = mkOption {
+                      description = lib.mdDoc "Organization ID, default = 1.";
+                      default = 1;
+                      type = types.int;
+                    };
+
+                    options.name = mkOption {
+                      description = lib.mdDoc "Name of the mute time interval, must be unique. Required.";
+                      type = types.str;
+                    };
+                  });
+                };
+              };
+            });
+            example = literalExpression ''
+              {
+                apiVersion = 1;
+
+                muteTimes = [{
+                  orgId = 1;
+                  name = "mti_1";
+                  time_intervals = [{
+                    times = [{
+                      start_time = "06:00";
+                      end_time = "23:59";
+                    }];
+                    weekdays = [
+                      "monday:wednesday"
+                      "saturday"
+                      "sunday"
+                    ];
+                    months = [
+                      "1:3"
+                      "may:august"
+                      "december"
+                    ];
+                    years = [
+                      "2020:2022"
+                      "2030"
+                    ];
+                    days_of_month = [
+                      "1:5"
+                      "-3:-1"
+                    ];
+                  }];
+                }];
+
+                deleteMuteTimes = [{
+                  orgId = 1;
+                  name = "mti_1";
+                }];
+              }
+            '';
+          };
+        };
+      };
     };
 
     security = {
@@ -841,6 +1310,26 @@ in {
         assertion = if (builtins.isList cfg.provision.dashboards) then true else cfg.provision.dashboards.settings == null || cfg.provision.dashboards.path == null;
         message = "Cannot set both dashboards settings and dashboards path";
       }
+      {
+        assertion = cfg.provision.alerting.rules.settings == null || cfg.provision.alerting.rules.path == null;
+        message = "Cannot set both rules settings and rules path";
+      }
+      {
+        assertion = cfg.provision.alerting.contactPoints.settings == null || cfg.provision.alerting.contactPoints.path == null;
+        message = "Cannot set both contact points settings and contact points path";
+      }
+      {
+        assertion = cfg.provision.alerting.policies.settings == null || cfg.provision.alerting.policies.path == null;
+        message = "Cannot set both policies settings and policies path";
+      }
+      {
+        assertion = cfg.provision.alerting.templates.settings == null || cfg.provision.alerting.templates.path == null;
+        message = "Cannot set both templates settings and templates path";
+      }
+      {
+        assertion = cfg.provision.alerting.muteTimings.settings == null || cfg.provision.alerting.muteTimings.path == null;
+        message = "Cannot set both mute timings settings and mute timings path";
+      }
     ];
 
     systemd.services.grafana = {

nixos/tests/grafana/default.nix

@@ -8,4 +8,9 @@
   provision-datasources = import ./provision-datasources { inherit system pkgs; };
   provision-dashboards = import ./provision-dashboards { inherit system pkgs; };
   provision-notifiers = import ./provision-notifiers.nix { inherit system pkgs; };
+  provision-rules = import ./provision-rules { inherit system pkgs; };
+  provision-contact-points = import ./provision-contact-points { inherit system pkgs; };
+  provision-policies = import ./provision-policies { inherit system pkgs; };
+  provision-templates = import ./provision-templates { inherit system pkgs; };
+  provision-mute-timings = import ./provision-mute-timings { inherit system pkgs; };
 }

nixos/tests/grafana/provision-contact-points/default.nix

@@ -0,0 +1,76 @@
+args@{ pkgs, ... }:
+
+(import ../../make-test-python.nix ({ lib, pkgs, ... }:
+
+let
+  inherit (lib) mkMerge nameValuePair maintainers;
+
+  baseGrafanaConf = {
+    services.grafana = {
+      enable = true;
+      addr = "localhost";
+      analytics.reporting.enable = false;
+      domain = "localhost";
+      security = {
+        adminUser = "testadmin";
+        adminPassword = "snakeoilpwd";
+      };
+      provision.enable = true;
+    };
+  };
+
+  extraNodeConfs = {
+    provisionContactPointsNix = {
+      services.grafana.provision = {
+        alerting.contactPoints.settings = {
+          contactPoints = [{
+            name = "Test Contact Point";
+            receivers = [{
+              uid = "test_contact_point";
+              type = "prometheus-alertmanager";
+              settings.url = "http://localhost:9000";
+            }];
+          }];
+        };
+      };
+    };
+
+    provisionContactPointsYaml = {
+      services.grafana.provision.alerting.contactPoints.path = ./provision-contact-points.yaml;
+    };
+  };
+
+  nodes = builtins.listToAttrs (map (provisionType:
+    nameValuePair provisionType (mkMerge [
+    baseGrafanaConf
+    (extraNodeConfs.${provisionType} or {})
+  ])) [ "provisionContactPointsNix" "provisionContactPointsYaml" ]);
+
+in {
+  name = "grafana-provision-contact-points";
+
+  meta = with maintainers; {
+    maintainers = [ kfears willibutz ];
+  };
+
+  inherit nodes;
+
+  testScript = ''
+    start_all()
+    with subtest("Successful contact point provision with Nix"):
+        provisionContactPointsNix.wait_for_unit("grafana.service")
+        provisionContactPointsNix.wait_for_open_port(3000)
+        provisionContactPointsNix.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/contact-points | grep Test\ Contact\ Point"
+        )
+        provisionContactPointsNix.shutdown()
+
+    with subtest("Successful contact point provision with YAML"):
+        provisionContactPointsYaml.wait_for_unit("grafana.service")
+        provisionContactPointsYaml.wait_for_open_port(3000)
+        provisionContactPointsYaml.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/contact-points | grep Test\ Contact\ Point"
+        )
+        provisionContactPointsYaml.shutdown()
+  '';
+})) args

nixos/tests/grafana/provision-contact-points/provision-contact-points.yaml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+contactPoints:
+  - name: "Test Contact Point"
+    receivers:
+      - uid: "test_contact_point"
+        type: prometheus-alertmanager
+        settings:
+          url: http://localhost:9000

nixos/tests/grafana/provision-mute-timings/default.nix

@@ -0,0 +1,71 @@
+args@{ pkgs, ... }:
+
+(import ../../make-test-python.nix ({ lib, pkgs, ... }:
+
+let
+  inherit (lib) mkMerge nameValuePair maintainers;
+
+  baseGrafanaConf = {
+    services.grafana = {
+      enable = true;
+      addr = "localhost";
+      analytics.reporting.enable = false;
+      domain = "localhost";
+      security = {
+        adminUser = "testadmin";
+        adminPassword = "snakeoilpwd";
+      };
+      provision.enable = true;
+    };
+  };
+
+  extraNodeConfs = {
+    provisionMuteTimingsNix = {
+      services.grafana.provision = {
+        alerting.muteTimings.settings = {
+          muteTimes = [{
+            name = "Test Mute Timing";
+          }];
+        };
+      };
+    };
+
+    provisionMuteTimingsYaml = {
+      services.grafana.provision.alerting.muteTimings.path = ./provision-mute-timings.yaml;
+    };
+  };
+
+  nodes = builtins.listToAttrs (map (provisionType:
+    nameValuePair provisionType (mkMerge [
+    baseGrafanaConf
+    (extraNodeConfs.${provisionType} or {})
+  ])) [ "provisionMuteTimingsNix" "provisionMuteTimingsYaml" ]);
+
+in {
+  name = "grafana-provision-mute-timings";
+
+  meta = with maintainers; {
+    maintainers = [ kfears willibutz ];
+  };
+
+  inherit nodes;
+
+  testScript = ''
+    start_all()
+    with subtest("Successful mute timings provision with Nix"):
+        provisionMuteTimingsNix.wait_for_unit("grafana.service")
+        provisionMuteTimingsNix.wait_for_open_port(3000)
+        provisionMuteTimingsNix.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/mute-timings | grep Test\ Mute\ Timing"
+        )
+        provisionMuteTimingsNix.shutdown()
+
+    with subtest("Successful mute timings provision with YAML"):
+        provisionMuteTimingsYaml.wait_for_unit("grafana.service")
+        provisionMuteTimingsYaml.wait_for_open_port(3000)
+        provisionMuteTimingsYaml.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/mute-timings | grep Test\ Mute\ Timing"
+        )
+        provisionMuteTimingsYaml.shutdown()
+  '';
+})) args

nixos/tests/grafana/provision-mute-timings/provision-mute-timings.yaml

@@ -0,0 +1,4 @@
+apiVersion: 1
+
+muteTimes:
+  - name: "Test Mute Timing"

nixos/tests/grafana/provision-policies/default.nix

@@ -0,0 +1,82 @@
+args@{ pkgs, ... }:
+
+(import ../../make-test-python.nix ({ lib, pkgs, ... }:
+
+let
+  inherit (lib) mkMerge nameValuePair maintainers;
+
+  baseGrafanaConf = {
+    services.grafana = {
+      enable = true;
+      addr = "localhost";
+      analytics.reporting.enable = false;
+      domain = "localhost";
+      security = {
+        adminUser = "testadmin";
+        adminPassword = "snakeoilpwd";
+      };
+      provision.enable = true;
+    };
+  };
+
+  extraNodeConfs = {
+    provisionPoliciesNix = {
+      services.grafana.provision = {
+        alerting.policies.settings = {
+          policies = [{
+            receiver = "Test Contact Point";
+          }];
+        };
+        alerting.contactPoints.settings = {
+          contactPoints = [{
+            name = "Test Contact Point";
+            receivers = [{
+              uid = "test_contact_point";
+              type = "prometheus-alertmanager";
+              settings.url = "http://localhost:9000";
+            }];
+          }];
+        };
+      };
+    };
+
+    provisionPoliciesYaml = {
+      services.grafana.provision.alerting.policies.path = ./provision-policies.yaml;
+      services.grafana.provision.alerting.contactPoints.path = ./provision-contact-points.yaml;
+    };
+  };
+
+  nodes = builtins.listToAttrs (map (provisionType:
+    nameValuePair provisionType (mkMerge [
+    baseGrafanaConf
+    (extraNodeConfs.${provisionType} or {})
+  ])) [ "provisionPoliciesNix" "provisionPoliciesYaml" ]);
+
+in {
+  name = "grafana-provision-policies";
+
+  meta = with maintainers; {
+    maintainers = [ kfears willibutz ];
+  };
+
+  inherit nodes;
+
+  testScript = ''
+    start_all()
+    with subtest("Successful policy provision with Nix"):
+        provisionPoliciesNix.wait_for_unit("grafana.service")
+        provisionPoliciesNix.wait_for_open_port(3000)
+        provisionPoliciesNix.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/contact-points | grep Test\ Contact\ Point"
+        )
+        provisionPoliciesNix.shutdown()
+
+    with subtest("Successful policy provision with YAML"):
+        provisionPoliciesYaml.wait_for_unit("grafana.service")
+        provisionPoliciesYaml.wait_for_open_port(3000)
+        provisionPoliciesYaml.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/contact-points | grep Test\ Contact\ Point"
+        )
+        provisionPoliciesYaml.shutdown()
+  '';
+})) args

nixos/tests/grafana/provision-policies/provision-contact-points.yaml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+contactPoints:
+  - name: "Test Contact Point"
+    receivers:
+      - uid: "test_contact_point"
+        type: prometheus-alertmanager
+        settings:
+          url: http://localhost:9000

nixos/tests/grafana/provision-policies/provision-policies.yaml

@@ -0,0 +1,4 @@
+apiVersion: 1
+
+policies:
+  - receiver: "Test Contact Point"

nixos/tests/grafana/provision-rules/default.nix

@@ -0,0 +1,104 @@
+args@{ pkgs, ... }:
+
+(import ../../make-test-python.nix ({ lib, pkgs, ... }:
+
+let
+  inherit (lib) mkMerge nameValuePair maintainers;
+
+  baseGrafanaConf = {
+    services.grafana = {
+      enable = true;
+      addr = "localhost";
+      analytics.reporting.enable = false;
+      domain = "localhost";
+      security = {
+        adminUser = "testadmin";
+        adminPassword = "snakeoilpwd";
+      };
+      provision.enable = true;
+    };
+  };
+
+  extraNodeConfs = {
+    provisionRulesNix = {
+      services.grafana.provision = {
+        alerting.rules.settings = {
+          groups = [{
+            name = "test_rule_group";
+            folder = "test_folder";
+            interval = "60s";
+            rules = [{
+              uid = "test_rule";
+              title = "Test Rule";
+              condition = "A";
+              data = [{
+                refId = "A";
+                datasourceUid = "-100";
+                model = {
+                  conditions = [{
+                    evaluator = {
+                      params = [ 3 ];
+                      type = "git";
+                    };
+                    operator.type = "and";
+                    query.params = [ "A" ];
+                    reducer.type = "last";
+                    type = "query";
+                  }];
+                  datasource = {
+                    type = "__expr__";
+                    uid = "-100";
+                  };
+                  expression = "1==0";
+                  intervalMs = 1000;
+                  maxDataPoints = 43200;
+                  refId = "A";
+                  type = "math";
+                };
+              }];
+              for = "60s";
+            }];
+          }];
+        };
+      };
+    };
+
+    provisionRulesYaml = {
+      services.grafana.provision.alerting.rules.path = ./provision-rules.yaml;
+    };
+  };
+
+  nodes = builtins.listToAttrs (map (provisionType:
+    nameValuePair provisionType (mkMerge [
+    baseGrafanaConf
+    (extraNodeConfs.${provisionType} or {})
+  ])) [ "provisionRulesNix" "provisionRulesYaml" ]);
+
+in {
+  name = "grafana-provision-rules";
+
+  meta = with maintainers; {
+    maintainers = [ kfears willibutz ];
+  };
+
+  inherit nodes;
+
+  testScript = ''
+    start_all()
+    with subtest("Successful rule provision with Nix"):
+        provisionRulesNix.wait_for_unit("grafana.service")
+        provisionRulesNix.wait_for_open_port(3000)
+        provisionRulesNix.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/alert-rules/test_rule | grep Test\ Rule"
+        )
+        provisionRulesNix.shutdown()
+
+    with subtest("Successful rule provision with YAML"):
+        provisionRulesYaml.wait_for_unit("grafana.service")
+        provisionRulesYaml.wait_for_open_port(3000)
+        provisionRulesYaml.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/alert-rules/test_rule | grep Test\ Rule"
+        )
+        provisionRulesYaml.shutdown()
+  '';
+})) args

nixos/tests/grafana/provision-rules/provision-rules.yaml

@@ -0,0 +1,36 @@
+apiVersion: 1
+
+groups:
+  - name: "test_rule_group"
+    folder: "test_group"
+    interval: 60s
+    rules:
+      - uid: "test_rule"
+        title: "Test Rule"
+        condition: A
+        data:
+          - refId: A
+            datasourceUid: '-100'
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 3
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params:
+                      - A
+                  reducer:
+                    type: last
+                  type: query
+              datasource:
+                type: __expr__
+                uid: '-100'
+              expression: 1==0
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: A
+              type: math
+        for: 60s

nixos/tests/grafana/provision-templates/default.nix

@@ -0,0 +1,72 @@
+args@{ pkgs, ... }:
+
+(import ../../make-test-python.nix ({ lib, pkgs, ... }:
+
+let
+  inherit (lib) mkMerge nameValuePair maintainers;
+
+  baseGrafanaConf = {
+    services.grafana = {
+      enable = true;
+      addr = "localhost";
+      analytics.reporting.enable = false;
+      domain = "localhost";
+      security = {
+        adminUser = "testadmin";
+        adminPassword = "snakeoilpwd";
+      };
+      provision.enable = true;
+    };
+  };
+
+  extraNodeConfs = {
+    provisionTemplatesNix = {
+      services.grafana.provision = {
+        alerting.templates.settings = {
+          templates = [{
+            name = "Test Template";
+            template = "Test message";
+          }];
+        };
+      };
+    };
+
+    provisionTemplatesYaml = {
+      services.grafana.provision.alerting.templates.path = ./provision-templates.yaml;
+    };
+  };
+
+  nodes = builtins.listToAttrs (map (provisionType:
+    nameValuePair provisionType (mkMerge [
+    baseGrafanaConf
+    (extraNodeConfs.${provisionType} or {})
+  ])) [ "provisionTemplatesNix" "provisionTemplatesYaml" ]);
+
+in {
+  name = "grafana-provision-rules";
+
+  meta = with maintainers; {
+    maintainers = [ kfears willibutz ];
+  };
+
+  inherit nodes;
+
+  testScript = ''
+    start_all()
+    with subtest("Successful template provision with Nix"):
+        provisionTemplatesNix.wait_for_unit("grafana.service")
+        provisionTemplatesNix.wait_for_open_port(3000)
+        provisionTemplatesNix.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/templates | grep Test\ Template"
+        )
+        provisionTemplatesNix.shutdown()
+
+    with subtest("Successful template provision with YAML"):
+        provisionTemplatesYaml.wait_for_unit("grafana.service")
+        provisionTemplatesYaml.wait_for_open_port(3000)
+        provisionTemplatesYaml.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/templates | grep Test\ Template"
+        )
+        provisionTemplatesYaml.shutdown()
+  '';
+})) args

nixos/tests/grafana/provision-templates/provision-templates.yaml

@@ -0,0 +1,5 @@
+apiVersion: 1
+
+templates:
+  - name: "Test Template"
+    template: "Test message"